diff --git a/internal/framework/status/setters.go b/internal/framework/status/setters.go new file mode 100644 index 0000000000..40effc14c9 --- /dev/null +++ b/internal/framework/status/setters.go @@ -0,0 +1,226 @@ +package status + +import ( + "slices" + + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/gateway-api/apis/v1beta1" + + ngfAPI "github.com/nginxinc/nginx-gateway-fabric/apis/v1alpha1" +) + +// setter is a function that takes an object and sets the status on that object if the status has changed. +// If the status has not changed, and the setter does not set the status, it returns false. +type setter func(client.Object) bool + +func newNginxGatewayStatusSetter(clock Clock, status NginxGatewayStatus) func(client.Object) bool { + return func(object client.Object) bool { + ng := object.(*ngfAPI.NginxGateway) + conds := convertConditions( + status.Conditions, + status.ObservedGeneration, + clock.Now(), + ) + + if conditionsEqual(ng.Status.Conditions, conds) { + return false + } + + ng.Status = ngfAPI.NginxGatewayStatus{ + Conditions: conds, + } + + return true + } +} + +func newGatewayClassStatusSetter(clock Clock, gcs GatewayClassStatus) func(client.Object) bool { + return func(object client.Object) bool { + gc := object.(*v1beta1.GatewayClass) + status := prepareGatewayClassStatus(gcs, clock.Now()) + + if conditionsEqual(gc.Status.Conditions, status.Conditions) { + return false + } + + gc.Status = status + + return true + } +} + +func newGatewayStatusSetter(clock Clock, gs GatewayStatus) func(client.Object) bool { + return func(object client.Object) bool { + gw := object.(*v1beta1.Gateway) + status := prepareGatewayStatus(gs, clock.Now()) + + if gwStatusEqual(gw.Status, status) { + return false + } + + gw.Status = status + + return true + } +} + +func newHTTPRouteStatusSetter(gatewayCtlrName string, clock Clock, rs HTTPRouteStatus) func(client.Object) bool { + return func(object client.Object) bool { + hr := object.(*v1beta1.HTTPRoute) + status := prepareHTTPRouteStatus( + rs, + gatewayCtlrName, + clock.Now(), + ) + + if hrStatusEqual(gatewayCtlrName, hr.Status, status) { + return false + } + + hr.Status = status + + return true + } +} + +func gwStatusEqual(prev, cur v1beta1.GatewayStatus) bool { + addressesEqual := slices.EqualFunc(prev.Addresses, cur.Addresses, func(a1, a2 v1beta1.GatewayStatusAddress) bool { + if !equalPointers[v1beta1.AddressType](a1.Type, a2.Type) { + return false + } + + return a1.Value == a2.Value + }) + + if !addressesEqual { + return false + } + + if !conditionsEqual(prev.Conditions, cur.Conditions) { + return false + } + + return slices.EqualFunc(prev.Listeners, cur.Listeners, func(s1, s2 v1beta1.ListenerStatus) bool { + if s1.Name != s2.Name { + return false + } + + if s1.AttachedRoutes != s2.AttachedRoutes { + return false + } + + if !conditionsEqual(s1.Conditions, s2.Conditions) { + return false + } + + return slices.EqualFunc(s1.SupportedKinds, s2.SupportedKinds, func(k1, k2 v1beta1.RouteGroupKind) bool { + if k1.Kind != k2.Kind { + return false + } + + return equalPointers(k1.Group, k2.Group) + }) + }) +} + +func hrStatusEqual(gatewayCtlrName string, prev, cur v1beta1.HTTPRouteStatus) bool { + // Since other controllers may update HTTPRoute status we can't assume anything about the order of the statuses, + // and we have to ignore statuses written by other controllers when checking for equality. + // Therefore, we can't use slices.EqualFunc here because it cares about the order. + + // First, we check if the prev status has any RouteParentStatuses that are no longer present in the cur status. + for _, prevParent := range prev.Parents { + if prevParent.ControllerName != v1beta1.GatewayController(gatewayCtlrName) { + continue + } + + exists := slices.ContainsFunc(cur.Parents, func(curParent v1beta1.RouteParentStatus) bool { + return routeParentStatusEqual(prevParent, curParent) + }) + + if !exists { + return false + } + } + + // Then, we check if the cur status has any RouteParentStatuses that are no longer present in the prev status. + for _, curParent := range cur.Parents { + exists := slices.ContainsFunc(prev.Parents, func(prevParent v1beta1.RouteParentStatus) bool { + return routeParentStatusEqual(curParent, prevParent) + }) + + if !exists { + return false + } + } + + return true +} + +func routeParentStatusEqual(p1, p2 v1beta1.RouteParentStatus) bool { + if p1.ControllerName != p2.ControllerName { + return false + } + + if p1.ParentRef.Name != p2.ParentRef.Name { + return false + } + + if !equalPointers(p1.ParentRef.Namespace, p2.ParentRef.Namespace) { + return false + } + + if !equalPointers(p1.ParentRef.SectionName, p2.ParentRef.SectionName) { + return false + } + + // we ignore the rest of the ParentRef fields because we do not set them + + return conditionsEqual(p1.Conditions, p2.Conditions) +} + +func conditionsEqual(prev, cur []v1.Condition) bool { + return slices.EqualFunc(prev, cur, func(c1, c2 v1.Condition) bool { + if c1.ObservedGeneration != c2.ObservedGeneration { + return false + } + + if c1.Type != c2.Type { + return false + } + + if c1.Status != c2.Status { + return false + } + + if c1.Message != c2.Message { + return false + } + + return c1.Reason == c2.Reason + }) +} + +// equalPointers returns whether two pointers are equal. +// Pointers are considered equal if one of the following is true: +// - They are both nil. +// - One is nil and the other is empty (e.g. nil string and ""). +// - They are both non-nil, and their values are the same. +func equalPointers[T comparable](p1, p2 *T) bool { + if p1 == nil && p2 == nil { + return true + } + + var p1Val, p2Val T + + if p1 != nil { + p1Val = *p1 + } + + if p2 != nil { + p2Val = *p2 + } + + return p1Val == p2Val +} diff --git a/internal/framework/status/setters_test.go b/internal/framework/status/setters_test.go new file mode 100644 index 0000000000..f379e369ee --- /dev/null +++ b/internal/framework/status/setters_test.go @@ -0,0 +1,846 @@ +package status + +import ( + "testing" + "time" + + . "github.com/onsi/gomega" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/gateway-api/apis/v1beta1" + + ngfAPI "github.com/nginxinc/nginx-gateway-fabric/apis/v1alpha1" + "github.com/nginxinc/nginx-gateway-fabric/internal/framework/conditions" + "github.com/nginxinc/nginx-gateway-fabric/internal/framework/helpers" +) + +func TestNewNginxGatewayStatusSetter(t *testing.T) { + tests := []struct { + name string + status ngfAPI.NginxGatewayStatus + newStatus NginxGatewayStatus + expStatusSet bool + }{ + { + name: "NginxGateway has no status", + expStatusSet: true, + newStatus: NginxGatewayStatus{ + Conditions: []conditions.Condition{{Message: "new condition"}}, + }, + }, + { + name: "NginxGateway has old status", + expStatusSet: true, + newStatus: NginxGatewayStatus{ + Conditions: []conditions.Condition{{Message: "new condition"}}, + }, + status: ngfAPI.NginxGatewayStatus{ + Conditions: []v1.Condition{{Message: "old condition"}}, + }, + }, + { + name: "NginxGateway has same status", + expStatusSet: false, + newStatus: NginxGatewayStatus{ + Conditions: []conditions.Condition{{Message: "same condition"}}, + }, + status: ngfAPI.NginxGatewayStatus{ + Conditions: []v1.Condition{{Message: "same condition"}}, + }, + }, + } + + clock := &RealClock{} + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + g := NewWithT(t) + + setter := newNginxGatewayStatusSetter(clock, test.newStatus) + + statusSet := setter(&ngfAPI.NginxGateway{Status: test.status}) + g.Expect(statusSet).To(Equal(test.expStatusSet)) + }) + } +} + +func TestNewGatewayClassStatusSetter(t *testing.T) { + tests := []struct { + name string + status v1beta1.GatewayClassStatus + newStatus GatewayClassStatus + expStatusSet bool + }{ + { + name: "GatewayClass has no status", + newStatus: GatewayClassStatus{ + Conditions: []conditions.Condition{{Message: "new condition"}}, + }, + expStatusSet: true, + }, + { + name: "GatewayClass has old status", + newStatus: GatewayClassStatus{ + Conditions: []conditions.Condition{{Message: "new condition"}}, + }, + status: v1beta1.GatewayClassStatus{ + Conditions: []v1.Condition{{Message: "old condition"}}, + }, + expStatusSet: true, + }, + { + name: "GatewayClass has same status", + newStatus: GatewayClassStatus{ + Conditions: []conditions.Condition{{Message: "same condition"}}, + }, + status: v1beta1.GatewayClassStatus{ + Conditions: []v1.Condition{{Message: "same condition"}}, + }, + expStatusSet: false, + }, + } + + clock := &RealClock{} + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + g := NewWithT(t) + + setter := newGatewayClassStatusSetter(clock, test.newStatus) + statusSet := setter(&v1beta1.GatewayClass{Status: test.status}) + g.Expect(statusSet).To(Equal(test.expStatusSet)) + }) + } +} + +func TestNewGatewayStatusSetter(t *testing.T) { + expAddress := v1beta1.GatewayStatusAddress{ + Type: helpers.GetPointer(v1beta1.IPAddressType), + Value: "10.0.0.0", + } + + tests := []struct { + name string + status v1beta1.GatewayStatus + newStatus GatewayStatus + expStatusSet bool + }{ + { + name: "Gateway has no status", + newStatus: GatewayStatus{ + Conditions: []conditions.Condition{{Message: "new condition"}}, + Addresses: []v1beta1.GatewayStatusAddress{expAddress}, + }, + expStatusSet: true, + }, + { + name: "Gateway has old status", + newStatus: GatewayStatus{ + Conditions: []conditions.Condition{{Message: "new condition"}}, + Addresses: []v1beta1.GatewayStatusAddress{expAddress}, + }, + status: v1beta1.GatewayStatus{ + Conditions: []v1.Condition{{Message: "old condition"}}, + Addresses: []v1beta1.GatewayStatusAddress{expAddress}, + }, + expStatusSet: true, + }, + { + name: "Gateway has same status", + newStatus: GatewayStatus{ + Conditions: []conditions.Condition{{Message: "same condition"}}, + Addresses: []v1beta1.GatewayStatusAddress{expAddress}, + }, + status: v1beta1.GatewayStatus{ + Conditions: []v1.Condition{{Message: "same condition"}}, + Addresses: []v1beta1.GatewayStatusAddress{expAddress}, + }, + expStatusSet: false, + }, + } + + clock := &RealClock{} + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + g := NewWithT(t) + + setter := newGatewayStatusSetter(clock, test.newStatus) + + statusSet := setter(&v1beta1.Gateway{Status: test.status}) + g.Expect(statusSet).To(Equal(test.expStatusSet)) + }) + } +} + +func TestNewHTTPRouteStatusSetter(t *testing.T) { + controllerName := "controller" + + tests := []struct { + name string + status v1beta1.HTTPRouteStatus + newStatus HTTPRouteStatus + expStatusSet bool + }{ + { + name: "HTTPRoute has no status", + newStatus: HTTPRouteStatus{ + ParentStatuses: []ParentStatus{ + { + Conditions: []conditions.Condition{{Message: "new condition"}}, + }, + }, + }, + expStatusSet: true, + }, + { + name: "HTTPRoute has old status", + newStatus: HTTPRouteStatus{ + ParentStatuses: []ParentStatus{ + { + Conditions: []conditions.Condition{{Message: "new condition"}}, + }, + }, + }, + status: v1beta1.HTTPRouteStatus{ + RouteStatus: v1beta1.RouteStatus{ + Parents: []v1beta1.RouteParentStatus{ + { + ParentRef: v1beta1.ParentReference{}, + ControllerName: v1beta1.GatewayController(controllerName), + Conditions: []v1.Condition{{Message: "old condition"}}, + }, + }, + }, + }, + expStatusSet: true, + }, + { + name: "HTTPRoute has same status", + newStatus: HTTPRouteStatus{ + ParentStatuses: []ParentStatus{ + { + Conditions: []conditions.Condition{{Message: "same condition"}}, + }, + }, + }, + status: v1beta1.HTTPRouteStatus{ + RouteStatus: v1beta1.RouteStatus{ + Parents: []v1beta1.RouteParentStatus{ + { + ParentRef: v1beta1.ParentReference{}, + ControllerName: v1beta1.GatewayController(controllerName), + Conditions: []v1.Condition{{Message: "same condition"}}, + }, + }, + }, + }, + expStatusSet: false, + }, + } + + clock := &RealClock{} + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + g := NewWithT(t) + + setter := newHTTPRouteStatusSetter(controllerName, clock, test.newStatus) + + statusSet := setter(&v1beta1.HTTPRoute{Status: test.status}) + g.Expect(statusSet).To(Equal(test.expStatusSet)) + }) + } +} + +func TestGWStatusEqual(t *testing.T) { + getDefaultStatus := func() v1beta1.GatewayStatus { + return v1beta1.GatewayStatus{ + Addresses: []v1beta1.GatewayStatusAddress{ + { + Type: helpers.GetPointer(v1beta1.IPAddressType), + Value: "10.0.0.0", + }, + { + Type: helpers.GetPointer(v1beta1.IPAddressType), + Value: "11.0.0.0", + }, + }, + Conditions: []v1.Condition{ + { + Type: "type", /* conditions are covered by another test*/ + }, + }, + Listeners: []v1beta1.ListenerStatus{ + { + Name: "listener1", + SupportedKinds: []v1beta1.RouteGroupKind{ + { + Group: helpers.GetPointer[v1beta1.Group](v1beta1.GroupName), + Kind: "HTTPRoute", + }, + { + Group: helpers.GetPointer[v1beta1.Group](v1beta1.GroupName), + Kind: "TCPRoute", + }, + }, + AttachedRoutes: 1, + Conditions: []v1.Condition{ + { + Type: "type", /* conditions are covered by another test*/ + }, + }, + }, + { + Name: "listener2", + SupportedKinds: []v1beta1.RouteGroupKind{ + { + Group: helpers.GetPointer[v1beta1.Group](v1beta1.GroupName), + Kind: "HTTPRoute", + }, + }, + AttachedRoutes: 1, + Conditions: []v1.Condition{ + { + Type: "type", /* conditions are covered by another test*/ + }, + }, + }, + { + Name: "listener3", + SupportedKinds: []v1beta1.RouteGroupKind{ + { + Group: helpers.GetPointer[v1beta1.Group](v1beta1.GroupName), + Kind: "HTTPRoute", + }, + }, + AttachedRoutes: 1, + Conditions: []v1.Condition{ + { + Type: "type", /* conditions are covered by another test*/ + }, + }, + }, + }, + } + } + + getModifiedStatus := func(mod func(v1beta1.GatewayStatus) v1beta1.GatewayStatus) v1beta1.GatewayStatus { + return mod(getDefaultStatus()) + } + + tests := []struct { + name string + prevStatus v1beta1.GatewayStatus + curStatus v1beta1.GatewayStatus + expEqual bool + }{ + { + name: "different number of addresses", + prevStatus: getDefaultStatus(), + curStatus: getModifiedStatus(func(status v1beta1.GatewayStatus) v1beta1.GatewayStatus { + status.Addresses = status.Addresses[:1] + return status + }), + expEqual: false, + }, + { + name: "different address type", + prevStatus: getDefaultStatus(), + curStatus: getModifiedStatus(func(status v1beta1.GatewayStatus) v1beta1.GatewayStatus { + status.Addresses[1].Type = helpers.GetPointer(v1beta1.HostnameAddressType) + return status + }), + expEqual: false, + }, + { + name: "different address value", + prevStatus: getDefaultStatus(), + curStatus: getModifiedStatus(func(status v1beta1.GatewayStatus) v1beta1.GatewayStatus { + status.Addresses[0].Value = "12.0.0.0" + return status + }), + expEqual: false, + }, + { + name: "different conditions", + prevStatus: getDefaultStatus(), + curStatus: getModifiedStatus(func(status v1beta1.GatewayStatus) v1beta1.GatewayStatus { + status.Conditions[0].Type = "different" + return status + }), + expEqual: false, + }, + { + name: "different number of listener statuses", + prevStatus: getDefaultStatus(), + curStatus: getModifiedStatus(func(status v1beta1.GatewayStatus) v1beta1.GatewayStatus { + status.Listeners = status.Listeners[:2] + return status + }), + expEqual: false, + }, + { + name: "different listener status name", + prevStatus: getDefaultStatus(), + curStatus: getModifiedStatus(func(status v1beta1.GatewayStatus) v1beta1.GatewayStatus { + status.Listeners[2].Name = "different" + return status + }), + expEqual: false, + }, + { + name: "different listener status attached routes", + prevStatus: getDefaultStatus(), + curStatus: getModifiedStatus(func(status v1beta1.GatewayStatus) v1beta1.GatewayStatus { + status.Listeners[1].AttachedRoutes++ + return status + }), + expEqual: false, + }, + { + name: "different listener status conditions", + prevStatus: getDefaultStatus(), + curStatus: getModifiedStatus(func(status v1beta1.GatewayStatus) v1beta1.GatewayStatus { + status.Listeners[0].Conditions[0].Type = "different" + return status + }), + expEqual: false, + }, + { + name: "different listener status supported kinds (different number)", + prevStatus: getDefaultStatus(), + curStatus: getModifiedStatus(func(status v1beta1.GatewayStatus) v1beta1.GatewayStatus { + status.Listeners[0].SupportedKinds = status.Listeners[0].SupportedKinds[:1] + return status + }), + expEqual: false, + }, + { + name: "different listener status supported kinds (different kind)", + prevStatus: getDefaultStatus(), + curStatus: getModifiedStatus(func(status v1beta1.GatewayStatus) v1beta1.GatewayStatus { + status.Listeners[1].SupportedKinds[0].Kind = "TCPRoute" + return status + }), + expEqual: false, + }, + { + name: "different listener status supported kinds (different group)", + prevStatus: getDefaultStatus(), + curStatus: getModifiedStatus(func(status v1beta1.GatewayStatus) v1beta1.GatewayStatus { + status.Listeners[1].SupportedKinds[0].Group = helpers.GetPointer[v1beta1.Group]("different") + return status + }), + expEqual: false, + }, + { + name: "equal", + prevStatus: getDefaultStatus(), + curStatus: getDefaultStatus(), + expEqual: true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + g := NewWithT(t) + + equal := gwStatusEqual(test.prevStatus, test.curStatus) + g.Expect(equal).To(Equal(test.expEqual)) + }) + } +} + +func TestHRStatusEqual(t *testing.T) { + testConds := []v1.Condition{ + { + Type: "type", /* conditions are covered by another test*/ + }, + } + + previousStatus := v1beta1.HTTPRouteStatus{ + RouteStatus: v1beta1.RouteStatus{ + Parents: []v1beta1.RouteParentStatus{ + { + ParentRef: v1beta1.ParentReference{ + Namespace: helpers.GetPointer[v1beta1.Namespace]("test"), + Name: "our-parent", + SectionName: helpers.GetPointer[v1beta1.SectionName]("section1"), + }, + ControllerName: "ours", + Conditions: testConds, + }, + { + ParentRef: v1beta1.ParentReference{ + Namespace: helpers.GetPointer[v1beta1.Namespace]("test"), + Name: "not-our-parent", + SectionName: helpers.GetPointer[v1beta1.SectionName]("section1"), + }, + ControllerName: "not-ours", + Conditions: testConds, + }, + { + ParentRef: v1beta1.ParentReference{ + Namespace: helpers.GetPointer[v1beta1.Namespace]("test"), + Name: "our-parent", + SectionName: helpers.GetPointer[v1beta1.SectionName]("section2"), + }, + ControllerName: "ours", + Conditions: testConds, + }, + { + ParentRef: v1beta1.ParentReference{ + Namespace: helpers.GetPointer[v1beta1.Namespace]("test"), + Name: "not-our-parent", + SectionName: helpers.GetPointer[v1beta1.SectionName]("section2"), + }, + ControllerName: "not-ours", + Conditions: testConds, + }, + }, + }, + } + + getDefaultStatus := func() v1beta1.HTTPRouteStatus { + return v1beta1.HTTPRouteStatus{ + RouteStatus: v1beta1.RouteStatus{ + Parents: []v1beta1.RouteParentStatus{ + { + ParentRef: v1beta1.ParentReference{ + Namespace: helpers.GetPointer[v1beta1.Namespace]("test"), + Name: "our-parent", + SectionName: helpers.GetPointer[v1beta1.SectionName]("section1"), + }, + ControllerName: "ours", + Conditions: testConds, + }, + { + ParentRef: v1beta1.ParentReference{ + Namespace: helpers.GetPointer[v1beta1.Namespace]("test"), + Name: "our-parent", + SectionName: helpers.GetPointer[v1beta1.SectionName]("section2"), + }, + ControllerName: "ours", + Conditions: testConds, + }, + }, + }, + } + } + + newParentStatus := v1beta1.RouteParentStatus{ + ParentRef: v1beta1.ParentReference{ + Namespace: helpers.GetPointer[v1beta1.Namespace]("test"), + Name: "our-parent", + SectionName: helpers.GetPointer[v1beta1.SectionName]("section3"), + }, + ControllerName: "ours", + Conditions: testConds, + } + + getModifiedStatus := func(mod func(status v1beta1.HTTPRouteStatus) v1beta1.HTTPRouteStatus) v1beta1.HTTPRouteStatus { + return mod(getDefaultStatus()) + } + + tests := []struct { + name string + prevStatus v1beta1.HTTPRouteStatus + curStatus v1beta1.HTTPRouteStatus + expEqual bool + }{ + { + name: "stale status", + prevStatus: previousStatus, + curStatus: getModifiedStatus(func(status v1beta1.HTTPRouteStatus) v1beta1.HTTPRouteStatus { + // remove last parent status + status.Parents = status.Parents[:1] + return status + }), + expEqual: false, + }, + { + name: "new status", + prevStatus: previousStatus, + curStatus: getModifiedStatus(func(status v1beta1.HTTPRouteStatus) v1beta1.HTTPRouteStatus { + // add another parent status + status.Parents = append(status.Parents, newParentStatus) + return status + }), + expEqual: false, + }, + { + name: "equal", + prevStatus: previousStatus, + curStatus: getDefaultStatus(), + expEqual: true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + g := NewWithT(t) + equal := hrStatusEqual("ours", test.prevStatus, test.curStatus) + g.Expect(equal).To(Equal(test.expEqual)) + }) + } +} + +func TestRouteParentStatusEqual(t *testing.T) { + getDefaultStatus := func() v1beta1.RouteParentStatus { + return v1beta1.RouteParentStatus{ + ParentRef: v1beta1.ParentReference{ + Namespace: helpers.GetPointer[v1beta1.Namespace]("test"), + Name: "parent", + SectionName: helpers.GetPointer[v1beta1.SectionName]("section"), + }, + ControllerName: "controller", + Conditions: []v1.Condition{ + { + Type: "type", /* conditions are covered by another test*/ + }, + }, + } + } + + getModifiedStatus := func(mod func(v1beta1.RouteParentStatus) v1beta1.RouteParentStatus) v1beta1.RouteParentStatus { + return mod(getDefaultStatus()) + } + + tests := []struct { + name string + p1 v1beta1.RouteParentStatus + p2 v1beta1.RouteParentStatus + expEqual bool + }{ + { + name: "different controller name", + p1: getDefaultStatus(), + p2: getModifiedStatus(func(status v1beta1.RouteParentStatus) v1beta1.RouteParentStatus { + status.ControllerName = "different" + return status + }), + expEqual: false, + }, + { + name: "different parentRef name", + p1: getDefaultStatus(), + p2: getModifiedStatus(func(status v1beta1.RouteParentStatus) v1beta1.RouteParentStatus { + status.ParentRef.Name = "different" + return status + }), + expEqual: false, + }, + { + name: "different parentRef namespace", + p1: getDefaultStatus(), + p2: getModifiedStatus(func(status v1beta1.RouteParentStatus) v1beta1.RouteParentStatus { + status.ParentRef.Namespace = helpers.GetPointer[v1beta1.Namespace]("different") + return status + }), + expEqual: false, + }, + { + name: "different parentRef section name", + p1: getDefaultStatus(), + p2: getModifiedStatus(func(status v1beta1.RouteParentStatus) v1beta1.RouteParentStatus { + status.ParentRef.SectionName = helpers.GetPointer[v1beta1.SectionName]("different") + return status + }), + expEqual: false, + }, + { + name: "different conditions", + p1: getDefaultStatus(), + p2: getModifiedStatus(func(status v1beta1.RouteParentStatus) v1beta1.RouteParentStatus { + status.Conditions[0].Type = "different" + return status + }), + expEqual: false, + }, + { + name: "equal", + p1: getDefaultStatus(), + p2: getDefaultStatus(), + expEqual: true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + g := NewWithT(t) + equal := routeParentStatusEqual(test.p1, test.p2) + g.Expect(equal).To(Equal(test.expEqual)) + }) + } +} + +func TestConditionsEqual(t *testing.T) { + getDefaultConds := func() []v1.Condition { + return []v1.Condition{ + { + Type: "type1", + Status: "status1", + ObservedGeneration: 1, + LastTransitionTime: v1.Time{Time: time.Now()}, + Reason: "reason1", + Message: "message1", + }, + { + Type: "type2", + Status: "status2", + ObservedGeneration: 1, + LastTransitionTime: v1.Time{Time: time.Now()}, + Reason: "reason2", + Message: "message2", + }, + { + Type: "type3", + Status: "status3", + ObservedGeneration: 1, + LastTransitionTime: v1.Time{Time: time.Now()}, + Reason: "reason3", + Message: "message3", + }, + } + } + + getModifiedConds := func(mod func([]v1.Condition) []v1.Condition) []v1.Condition { + return mod(getDefaultConds()) + } + + tests := []struct { + name string + prevConds []v1.Condition + curConds []v1.Condition + expEqual bool + }{ + { + name: "different observed gen", + prevConds: getDefaultConds(), + curConds: getModifiedConds(func(conds []v1.Condition) []v1.Condition { + conds[2].ObservedGeneration++ + return conds + }), + expEqual: false, + }, + { + name: "different status", + prevConds: getDefaultConds(), + curConds: getModifiedConds(func(conds []v1.Condition) []v1.Condition { + conds[1].Status = "different" + return conds + }), + expEqual: false, + }, + { + name: "different type", + prevConds: getDefaultConds(), + curConds: getModifiedConds(func(conds []v1.Condition) []v1.Condition { + conds[0].Type = "different" + return conds + }), + expEqual: false, + }, + { + name: "different message", + prevConds: getDefaultConds(), + curConds: getModifiedConds(func(conds []v1.Condition) []v1.Condition { + conds[2].Message = "different" + return conds + }), + expEqual: false, + }, + { + name: "different reason", + prevConds: getDefaultConds(), + curConds: getModifiedConds(func(conds []v1.Condition) []v1.Condition { + conds[1].Reason = "different" + return conds + }), + expEqual: false, + }, + { + name: "different number of conditions", + prevConds: getDefaultConds(), + curConds: getModifiedConds(func(conds []v1.Condition) []v1.Condition { + return conds[:2] + }), + expEqual: false, + }, + { + name: "equal", + prevConds: getDefaultConds(), + curConds: getDefaultConds(), + expEqual: true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + g := NewWithT(t) + equal := conditionsEqual(test.prevConds, test.curConds) + g.Expect(equal).To(Equal(test.expEqual)) + }) + } +} + +func TestEqualPointers(t *testing.T) { + tests := []struct { + p1 *string + p2 *string + name string + expEqual bool + }{ + { + name: "first pointer nil; second has non-empty value", + p1: nil, + p2: helpers.GetPointer("test"), + expEqual: false, + }, + { + name: "second pointer nil; first has non-empty value", + p1: helpers.GetPointer("test"), + p2: nil, + expEqual: false, + }, + { + name: "different values", + p1: helpers.GetPointer("test"), + p2: helpers.GetPointer("different"), + expEqual: false, + }, + { + name: "both pointers nil", + p1: nil, + p2: nil, + expEqual: true, + }, + { + name: "first pointer nil; second empty", + p1: nil, + p2: helpers.GetPointer(""), + expEqual: true, + }, + { + name: "second pointer nil; first empty", + p1: helpers.GetPointer(""), + p2: nil, + expEqual: true, + }, + { + name: "same value", + p1: helpers.GetPointer("test"), + p2: helpers.GetPointer("test"), + expEqual: true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + g := NewWithT(t) + + val := equalPointers(test.p1, test.p2) + g.Expect(val).To(Equal(test.expEqual)) + }) + } +} diff --git a/internal/framework/status/updater.go b/internal/framework/status/updater.go index 6dd114b4ea..16797ea347 100644 --- a/internal/framework/status/updater.go +++ b/internal/framework/status/updater.go @@ -16,7 +16,6 @@ import ( ngfAPI "github.com/nginxinc/nginx-gateway-fabric/apis/v1alpha1" "github.com/nginxinc/nginx-gateway-fabric/internal/framework/controller" - "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/config" ) //go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 . Updater @@ -43,8 +42,6 @@ type UpdaterConfig struct { Client client.Client // Clock is used as a source of time for the LastTransitionTime field in Conditions in resource statuses. Clock Clock - // GatewayPodConfig contains information about this Pod. - GatewayPodConfig config.GatewayPodConfig // Logger holds a logger to be used. Logger logr.Logger // GatewayCtlrName is the name of the Gateway controller. @@ -62,20 +59,18 @@ type UpdaterConfig struct { // // It has the following limitations: // -// (1) It is not smart. It will update the status of a resource (make an API call) even if it hasn't changed. -// -// (2) It is synchronous, which means the status reporter can slow down the event loop. +// (1) It is synchronous, which means the status reporter can slow down the event loop. // Consider the following cases: // (a) Sometimes the Gateway will need to update statuses of all resources it handles, which could be ~1000. Making 1000 // status API calls sequentially will take time. // (b) k8s API can become slow or even timeout. This will increase every update status API call. // Making UpdaterImpl asynchronous will prevent it from adding variable delays to the event loop. // -// (3) It doesn't clear the statuses of a resources that are no longer handled by the Gateway. For example, if +// (2) It doesn't clear the statuses of a resources that are no longer handled by the Gateway. For example, if // an HTTPRoute resource no longer has the parentRef to the Gateway resources, the Gateway must update the status // of the resource to remove the status about the removed parentRef. // -// (4) If another controllers changes the status of the Gateway/HTTPRoute resource so that the information set by our +// (3) If another controllers changes the status of the Gateway/HTTPRoute resource so that the information set by our // Gateway is removed, our Gateway will not restore the status until the EventLoop invokes the StatusUpdater as a // result of processing some other new change to a resource(s). // FIXME(pleshakov): Make updater production ready @@ -157,16 +152,12 @@ func (upd *UpdaterImpl) updateNginxGateway(ctx context.Context, status *NginxGat upd.cfg.Logger.Info("Updating Nginx Gateway status") if status != nil { - upd.writeStatuses(ctx, status.NsName, &ngfAPI.NginxGateway{}, func(object client.Object) { - ng := object.(*ngfAPI.NginxGateway) - ng.Status = ngfAPI.NginxGatewayStatus{ - Conditions: convertConditions( - status.Conditions, - status.ObservedGeneration, - upd.cfg.Clock.Now(), - ), - } - }) + upd.writeStatuses( + ctx, + status.NsName, + &ngfAPI.NginxGateway{}, + newNginxGatewayStatusSetter(upd.cfg.Clock, *status), + ) } } @@ -187,11 +178,8 @@ func (upd *UpdaterImpl) updateGatewayAPI(ctx context.Context, statuses GatewayAP return default: } - upd.writeStatuses(ctx, nsname, &v1beta1.GatewayClass{}, func(object client.Object) { - gc := object.(*v1beta1.GatewayClass) - gc.Status = prepareGatewayClassStatus(gcs, upd.cfg.Clock.Now()) - }, - ) + + upd.writeStatuses(ctx, nsname, &v1beta1.GatewayClass{}, newGatewayClassStatusSetter(upd.cfg.Clock, gcs)) } } @@ -201,10 +189,8 @@ func (upd *UpdaterImpl) updateGatewayAPI(ctx context.Context, statuses GatewayAP return default: } - upd.writeStatuses(ctx, nsname, &v1beta1.Gateway{}, func(object client.Object) { - gw := object.(*v1beta1.Gateway) - gw.Status = prepareGatewayStatus(gs, upd.cfg.Clock.Now()) - }) + + upd.writeStatuses(ctx, nsname, &v1beta1.Gateway{}, newGatewayStatusSetter(upd.cfg.Clock, gs)) } for nsname, rs := range statuses.HTTPRouteStatuses { @@ -213,15 +199,13 @@ func (upd *UpdaterImpl) updateGatewayAPI(ctx context.Context, statuses GatewayAP return default: } - upd.writeStatuses(ctx, nsname, &v1beta1.HTTPRoute{}, func(object client.Object) { - hr := object.(*v1beta1.HTTPRoute) - // statuses.GatewayStatus is never nil when len(statuses.HTTPRouteStatuses) > 0 - hr.Status = prepareHTTPRouteStatus( - rs, - upd.cfg.GatewayCtlrName, - upd.cfg.Clock.Now(), - ) - }) + + upd.writeStatuses( + ctx, + nsname, + &v1beta1.HTTPRoute{}, + newHTTPRouteStatusSetter(upd.cfg.GatewayCtlrName, upd.cfg.Clock, rs), + ) } } @@ -229,7 +213,7 @@ func (upd *UpdaterImpl) writeStatuses( ctx context.Context, nsname types.NamespacedName, obj client.Object, - statusSetter func(client.Object), + statusSetter setter, ) { err := wait.ExponentialBackoffWithContext( ctx, @@ -284,7 +268,7 @@ func NewRetryUpdateFunc( nsname types.NamespacedName, obj client.Object, logger logr.Logger, - statusSetter func(client.Object), + statusSetter func(client.Object) bool, ) func(ctx context.Context) (bool, error) { return func(ctx context.Context) (bool, error) { // The function handles errors by reporting them in the logs. @@ -298,16 +282,28 @@ func NewRetryUpdateFunc( if apierrors.IsNotFound(err) { return true, nil } + logger.V(1).Info( "Encountered error when getting resource to update status", "error", err, "namespace", nsname.Namespace, "name", nsname.Name, - "kind", obj.GetObjectKind().GroupVersionKind().Kind) + "kind", obj.GetObjectKind().GroupVersionKind().Kind, + ) + return false, nil } - statusSetter(obj) + if !statusSetter(obj) { + logger.V(1).Info( + "Skipping status update because there's no change", + "namespace", nsname.Namespace, + "name", nsname.Name, + "kind", obj.GetObjectKind().GroupVersionKind().Kind, + ) + + return true, nil + } if err := updater.Update(ctx, obj); err != nil { logger.V(1).Info( @@ -315,7 +311,9 @@ func NewRetryUpdateFunc( "error", err, "namespace", nsname.Namespace, "name", nsname.Name, - "kind", obj.GetObjectKind().GroupVersionKind().Kind) + "kind", obj.GetObjectKind().GroupVersionKind().Kind, + ) + return false, nil } diff --git a/internal/framework/status/updater_retry_test.go b/internal/framework/status/updater_retry_test.go index db4a23b6bc..c2f084e0e7 100644 --- a/internal/framework/status/updater_retry_test.go +++ b/internal/framework/status/updater_retry_test.go @@ -20,56 +20,80 @@ import ( func TestNewRetryUpdateFunc(t *testing.T) { tests := []struct { - getReturns error - updateReturns error - name string - expConditionPassed bool + getReturns error + updateReturns error + name string + expUpdateCallCount int + statusSetterReturns bool + expConditionPassed bool }{ { - getReturns: errors.New("failed to get resource"), - updateReturns: nil, - name: "get fails", - expConditionPassed: false, + getReturns: errors.New("failed to get resource"), + updateReturns: nil, + statusSetterReturns: true, + expUpdateCallCount: 0, + name: "get fails", + expConditionPassed: false, }, { - getReturns: apierrors.NewNotFound(schema.GroupResource{}, "not found"), - updateReturns: nil, - name: "get fails and apierrors is not found", - expConditionPassed: true, + getReturns: apierrors.NewNotFound(schema.GroupResource{}, "not found"), + updateReturns: nil, + statusSetterReturns: true, + expUpdateCallCount: 0, + name: "get fails and apierrors is not found", + expConditionPassed: true, }, { - getReturns: nil, - updateReturns: errors.New("failed to update resource"), - name: "update fails", - expConditionPassed: false, + getReturns: nil, + updateReturns: errors.New("failed to update resource"), + statusSetterReturns: true, + expUpdateCallCount: 1, + name: "update fails", + expConditionPassed: false, }, { - getReturns: nil, - updateReturns: nil, - name: "nothing fails", - expConditionPassed: true, + getReturns: nil, + updateReturns: nil, + statusSetterReturns: false, + expUpdateCallCount: 0, + name: "status not set", + expConditionPassed: true, + }, + { + getReturns: nil, + updateReturns: nil, + statusSetterReturns: true, + expUpdateCallCount: 1, + name: "nothing fails", + expConditionPassed: true, }, } - fakeStatusUpdater := &statusfakes.FakeK8sUpdater{} - fakeGetter := &controllerfakes.FakeGetter{} for _, test := range tests { t.Run(test.name, func(t *testing.T) { g := NewWithT(t) + + fakeStatusUpdater := &statusfakes.FakeK8sUpdater{} + fakeGetter := &controllerfakes.FakeGetter{} + fakeStatusUpdater.UpdateReturns(test.updateReturns) fakeGetter.GetReturns(test.getReturns) + f := status.NewRetryUpdateFunc( fakeGetter, fakeStatusUpdater, types.NamespacedName{}, &v1beta1.GatewayClass{}, zap.New(), - func(client.Object) {}) + func(client.Object) bool { return test.statusSetterReturns }, + ) + conditionPassed, err := f(context.Background()) // The function should always return nil. g.Expect(err).ToNot(HaveOccurred()) g.Expect(conditionPassed).To(Equal(test.expConditionPassed)) + g.Expect(fakeStatusUpdater.UpdateCallCount()).To(Equal(test.expUpdateCallCount)) }) } } diff --git a/internal/framework/status/updater_test.go b/internal/framework/status/updater_test.go index 66512a9bdc..786ce10c26 100644 --- a/internal/framework/status/updater_test.go +++ b/internal/framework/status/updater_test.go @@ -19,7 +19,6 @@ import ( "github.com/nginxinc/nginx-gateway-fabric/internal/framework/helpers" "github.com/nginxinc/nginx-gateway-fabric/internal/framework/status" "github.com/nginxinc/nginx-gateway-fabric/internal/framework/status/statusfakes" - "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/config" staticConds "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/state/conditions" ) @@ -252,14 +251,11 @@ var _ = Describe("Updater", func() { BeforeAll(func() { updater = status.NewUpdater(status.UpdaterConfig{ - GatewayCtlrName: gatewayCtrlName, - GatewayClassName: gcName, - Client: client, - Logger: zap.New(), - Clock: fakeClock, - GatewayPodConfig: config.GatewayPodConfig{ - PodIP: "1.2.3.4", - }, + GatewayCtlrName: gatewayCtrlName, + GatewayClassName: gcName, + Client: client, + Logger: zap.New(), + Clock: fakeClock, UpdateGatewayClassStatus: true, }) @@ -424,7 +420,11 @@ var _ = Describe("Updater", func() { }, }) - err := client.Get(context.Background(), types.NamespacedName{Namespace: "test", Name: "gateway"}, latestGw) + err := client.Get( + context.Background(), + types.NamespacedName{Namespace: "test", Name: "gateway"}, + latestGw, + ) Expect(err).ToNot(HaveOccurred()) expectedGw.ResourceVersion = latestGw.ResourceVersion @@ -655,12 +655,16 @@ var _ = Describe("Updater", func() { wg := &sync.WaitGroup{} ctx := context.Background() - // spin up 10 goroutines that Update and 10 that WriteLastStatuses - // and make sure that 20 updates were made to the Gateway resource. + // Spin up 10 goroutines that Update and 10 that call Enable which writes the last statuses. + // Since we only write statuses when they've changed, we will only update the status 10 times. + // The purpose of this test is to exercise the locking mechanism embedded in the updater. + // If there is a data race, this test combined with the -race flag that we run tests with, + // should catch it. for i := 0; i < 10; i++ { wg.Add(2) + gen := 5 + i go func() { - updater.Update(ctx, createGwAPIStatuses(generations{gateways: 5})) + updater.Update(ctx, createGwAPIStatuses(generations{gateways: int64(gen)})) wg.Done() }() @@ -682,8 +686,8 @@ var _ = Describe("Updater", func() { Expect(err).ToNot(HaveOccurred()) // Before this test there were 6 updates to the Gateway resource. - // So now the resource version should equal 26. - Expect(latestGw.ResourceVersion).To(Equal("26")) + // So now the resource version should equal 16. + Expect(latestGw.ResourceVersion).To(Equal("16")) }) }) }) @@ -696,14 +700,11 @@ var _ = Describe("Updater", func() { BeforeAll(func() { updater = status.NewUpdater(status.UpdaterConfig{ - GatewayCtlrName: gatewayCtrlName, - GatewayClassName: gcName, - Client: client, - Logger: zap.New(), - Clock: fakeClock, - GatewayPodConfig: config.GatewayPodConfig{ - PodIP: "1.2.3.4", - }, + GatewayCtlrName: gatewayCtrlName, + GatewayClassName: gcName, + Client: client, + Logger: zap.New(), + Clock: fakeClock, UpdateGatewayClassStatus: false, }) @@ -747,14 +748,11 @@ var _ = Describe("Updater", func() { Describe("Edge cases", func() { It("panics on update if status type is unknown", func() { updater := status.NewUpdater(status.UpdaterConfig{ - GatewayCtlrName: gatewayCtrlName, - GatewayClassName: gcName, - Client: client, - Logger: zap.New(), - Clock: fakeClock, - GatewayPodConfig: config.GatewayPodConfig{ - PodIP: "1.2.3.4", - }, + GatewayCtlrName: gatewayCtrlName, + GatewayClassName: gcName, + Client: client, + Logger: zap.New(), + Clock: fakeClock, UpdateGatewayClassStatus: true, }) diff --git a/internal/mode/static/manager.go b/internal/mode/static/manager.go index 7a04f33e8d..3eca635143 100644 --- a/internal/mode/static/manager.go +++ b/internal/mode/static/manager.go @@ -161,7 +161,6 @@ func StartManager(cfg config.Config) error { Clock: status.NewRealClock(), UpdateGatewayClassStatus: cfg.UpdateGatewayClassStatus, LeaderElectionEnabled: cfg.LeaderElection.Enabled, - GatewayPodConfig: cfg.GatewayPodConfig, }) eventHandler := newEventHandlerImpl(eventHandlerConfig{ @@ -273,7 +272,10 @@ func registerControllers( { objectType: &apiv1.Service{}, options: func() []controller.Option { - svcNSName := types.NamespacedName{Namespace: cfg.GatewayPodConfig.Namespace, Name: cfg.GatewayPodConfig.ServiceName} + svcNSName := types.NamespacedName{ + Namespace: cfg.GatewayPodConfig.Namespace, + Name: cfg.GatewayPodConfig.ServiceName, + } return []controller.Option{ controller.WithK8sPredicate(predicate.GatewayServicePredicate{NSName: svcNSName}), } diff --git a/tests/scale/results/1.0.0/1.0.0.md b/tests/scale/results/1.0.0/1.0.0.md index 0d8b60ad62..1c2d47f189 100644 --- a/tests/scale/results/1.0.0/1.0.0.md +++ b/tests/scale/results/1.0.0/1.0.0.md @@ -17,8 +17,8 @@ NGF version: ```text -commit: "72b6c6ef8915c697626eeab88fdb6a3ce15b8da0" -date: "2023-10-04T22:22:09Z" +commit: "676cb63b1787f1379c5d0034000b4518d35b8145" // TODO: update commit to final commit for PR +date: "2023-10-16T17:00:09Z" version: "edge" ``` @@ -35,7 +35,7 @@ Kubernetes: ```text Server Version: version.Info{Major:"1", Minor:"27", -GitVersion:"v1.27.6-gke.1248000", +GitVersion:"v1.27.6-gke.1445000", GitCommit:"85a90ed8e702b392003d6757917e4cc167776e03", GitTreeState:"clean", BuildDate:"2023-09-21T22:16:57Z", GoVersion:"go1.20.8 X:boringcrypto", Compiler:"gc", @@ -46,9 +46,18 @@ Platform:"linux/amd64"} ### Scale Listeners -| Total Reloads | Total Reload Errors | Average Reload Time (ms) | -|---------------|---------------------|--------------------------| -| 107 | 0 | 155.05374791154367 | +Reloads: + +| Total | Total Errors | Ave Time (ms) | <= 500ms | +|-------|--------------|--------------------|----------| +| 128 | 0 | 134.69505494505492 | 100% | + + +Event Batch Processing: + +| Total | Ave Time (ms) | <= 500ms | <= 1000ms | +|-------|--------------------|----------|-----------| +| 398 | 218.22692307692304 | 93.22% | 100% | **NGINX Errors**: None. @@ -56,20 +65,27 @@ Platform:"linux/amd64"} **Pod Restarts**: None. -**CPU**: Steep linear increase as NGF processed all the Services. Dropped off during scaling of Listeners. -See [graph](/tests/scale/results/1.0.0/TestScale_Listeners/CPU.png). +**CPU**: ![CPU.png](/tests/scale/results/1.0.0/TestScale_Listeners/CPU.png). -**Memory**: Gradual increase in memory. Topped out at 40MiB. -See [graph](/tests/scale/results/1.0.0/TestScale_Listeners/Memory.png). +**Memory**: ![Memory.png](/tests/scale/results/1.0.0/TestScale_Listeners/Memory.png). -**Time To Ready**: Time to ready numbers consistently under 3s. 62nd Listener had longest TTR of 3.02s. -See [graph](/tests/scale/results/1.0.0/TestScale_Listeners/TTR.png). +**Time To Ready**: ![TTR.png](/tests/scale/results/1.0.0/TestScale_Listeners/TTR.png). ### Scale HTTPS Listeners -| Total Reloads | Total Reload Errors | Average Reload Time (ms) | -|---------------|---------------------|--------------------------| -| 130 | 0 | 151.49397590361446ms | +Reloads: + +| Total | Total Errors | Ave Time (ms) | <= 500ms | +|-------|--------------|--------------------|----------| +| 128 | 0 | 153.67699221223148 | 100% | + + +Event Batch Processing: + +| Total | Ave Time (ms) | <= 500ms | <= 1000ms | <= 5000ms | +|-------|--------------------|----------|-----------|-----------| +| 457 | 201.60030046204622 | 91.69% | 99.78% | 100% | + **NGINX Errors**: None. @@ -77,21 +93,29 @@ See [graph](/tests/scale/results/1.0.0/TestScale_Listeners/TTR.png). **Pod Restarts**: None. -**CPU**: Steep linear increase as NGF processed all the Services and Secrets. Dropped off during scaling of Listeners. -See [graph](/tests/scale/results/1.0.0/TestScale_HTTPSListeners/CPU.png). +**CPU**: ![CPU.png](/tests/scale/results/1.0.0/TestScale_HTTPSListeners/CPU.png). -**Memory**: Mostly linear increase. Topping out at right under 50MiB. -See [graph](/tests/scale/results/1.0.0/TestScale_HTTPSListeners/Memory.png). +**Memory**: ![Memory.png](/tests/scale/results/1.0.0/TestScale_HTTPSListeners/Memory.png). -**Time To Ready**: The time to ready numbers were pretty consistent (under 3 sec) except for one spike of 10s. I believe -this spike was client-side because the NGF logs indicated that the reload successfully happened under 3s. -See [graph](/tests/scale/results/1.0.0/TestScale_HTTPSListeners/TTR.png). +**Time To Ready**: ![TTR.png](/tests/scale/results/1.0.0/TestScale_HTTPSListeners/TTR.png). ### Scale HTTPRoutes -| Total Reloads | Total Reload Errors | Average Reload Time (ms) | -|---------------|---------------------|--------------------------| -| 1001 | 0 | 354.3878787878788ms | +Reloads: + +| Delay | Total | Total Errors | Ave Time (ms) | <= 500ms | <= 1000ms | +|-----------|-------|--------------|-------------------|----------|-----------| +| 2 seconds | 1001 | 0 | 341.7087396702501 | 83.32% | 100% | +| No delay | 1001 | 0 | 369.2821023173991 | 75.72% | 100% | + + +Event Batch Processing: + +| Delay | Total | Ave Time | <= 500ms | <= 1000ms | +|-----------|-------|--------------------|----------|-----------| +| 2 seconds | 2089 | 192.62227561490883 | 86.21% | 100% | +| No delay | 2082 | 206.68062489354992 | 83.43% | 100% | + **NGINX Errors**: None. @@ -99,37 +123,47 @@ See [graph](/tests/scale/results/1.0.0/TestScale_HTTPSListeners/TTR.png). **Pod Restarts**: None. -**CPU**: CPU mostly oscillated between .04 and .06. Several spikes over .06. -See [graph](/tests/scale/results/1.0.0/TestScale_HTTPRoutes/CPU.png). +**CPU**: + +2-sec delay: +![CPU.png](/tests/scale/results/1.0.0/TestScale_HTTPRoutes/CPU.png). -**Memory**: Memory usage gradually increased from 25 - 150MiB over course of the test with some spikes reaching up to -200MiB. See [graph](/tests/scale/results/1.0.0/TestScale_HTTPRoutes/Memory.png). +No delay: +![CPU.png](/tests/scale/results/1.0.0/TestScale_HTTPRoutes/CPU-no-delay.png). -**Time To Ready**: This time to ready graph is unique because there are three plotted lines: +**Memory**: -- Blue Line: 2-second delay after adding a new HTTPRoute. -- Red Line: No delay after adding a new HTTPRoute. -- Green Line: 10-second delay after adding a new HTTPRoute +2-sec delay: +![Memory.png](/tests/scale/results/1.0.0/TestScale_HTTPRoutes/Memory.png). -The Blue and Red lines are incomplete because the tests timed out. However, I think the implications are pretty clear. -The more time that passes between scaling events, the smaller the time to ready values are. This is because NGF -re-queues all the HTTPRoutes after updating their statuses. This is because the HTTPRoute has changed after we write its -status. This is compounded by the fact that NGF writes status for every HTTPRoute in the graph on every configuration -update. So if you add HTTPRoute 100, NGF will update the configuration with this new route and then update the status of -all 100 HTTPRoutes in the graph. +No delay: +![Memory.png](/tests/scale/results/1.0.0/TestScale_HTTPRoutes/Memory-no-delay.png). -Related issues: +**TTR**: -- https://github.com/nginxinc/nginx-gateway-fabric/issues/1013 -- https://github.com/nginxinc/nginx-gateway-fabric/issues/825 +Combined: +![TTR.png](/tests/scale/results/1.0.0/TestScale_HTTPRoutes/TTR.png) -See [graph](/tests/scale/results/1.0.0/TestScale_HTTPRoutes/TTR.png). + +Issue Filed: [Improve HTTPRoute Processing at Scale](https://github.com/nginxinc/nginx-gateway-fabric/issues/1122). ### Scale Upstream Servers -| Start Time (UNIX) | End Time (UNIX) | Duration (s) | Total Reloads | Total Reload Errors | Average Reload Time (ms) | -|-------------------|-----------------|--------------|---------------|---------------------|--------------------------| -| 1696535183 | 1696535311 | 128 | 83 | 0 | 126.55555555555557 | +| Start Time (UNIX) | End Time (UNIX) | Duration (s) | +|-------------------|-----------------|--------------| +| 1697516151 | 1697516255 | 104 | + +Reloads: + +| Total | Total Errors | Ave Time (ms) | <= 500ms | +|-------|--------------|--------------------|----------| +| 116 | 0 | 125.35251106285726 | 100% | + +Event Batch Processing: + +| Total | Ave Time (ms) | <=500ms | <= 1000ms | <= 5000ms | +|-------|--------------------|---------|-----------|-----------| +| 122 | 241.95484032851115 | 97.54% | 98.36% | 100% | **NGINX Errors**: None. @@ -137,11 +171,9 @@ See [graph](/tests/scale/results/1.0.0/TestScale_HTTPRoutes/TTR.png). **Pod Restarts**: None. -**CPU**: CPU steeply increases as NGF handles all the new Pods. Drops after they are processed. -See [graph](/tests/scale/results/1.0.0/TestScale_UpstreamServers/CPU.png). +**CPU**: ![CPU.png](/tests/scale/results/1.0.0/TestScale_UpstreamServers/CPU.png). -**Memory**: Memory stays relatively flat and under 40MiB. -See [graph](/tests/scale/results/1.0.0/TestScale_UpstreamServers/Memory.png). +**Memory**: ![Memory.png](/tests/scale/results/1.0.0/TestScale_UpstreamServers/Memory.png). ### Scale HTTP Matches diff --git a/tests/scale/results/1.0.0/TestScale_HTTPRoutes/CPU-no-delay.png b/tests/scale/results/1.0.0/TestScale_HTTPRoutes/CPU-no-delay.png new file mode 100644 index 0000000000..e3ea3cd31f Binary files /dev/null and b/tests/scale/results/1.0.0/TestScale_HTTPRoutes/CPU-no-delay.png differ diff --git a/tests/scale/results/1.0.0/TestScale_HTTPRoutes/CPU.png b/tests/scale/results/1.0.0/TestScale_HTTPRoutes/CPU.png index d881a9c70a..fa312a8f48 100644 Binary files a/tests/scale/results/1.0.0/TestScale_HTTPRoutes/CPU.png and b/tests/scale/results/1.0.0/TestScale_HTTPRoutes/CPU.png differ diff --git a/tests/scale/results/1.0.0/TestScale_HTTPRoutes/Memory-no-delay.png b/tests/scale/results/1.0.0/TestScale_HTTPRoutes/Memory-no-delay.png new file mode 100644 index 0000000000..0a4a7784c6 Binary files /dev/null and b/tests/scale/results/1.0.0/TestScale_HTTPRoutes/Memory-no-delay.png differ diff --git a/tests/scale/results/1.0.0/TestScale_HTTPRoutes/Memory.png b/tests/scale/results/1.0.0/TestScale_HTTPRoutes/Memory.png index 615e8cc70e..ff8fc0cc5c 100644 Binary files a/tests/scale/results/1.0.0/TestScale_HTTPRoutes/Memory.png and b/tests/scale/results/1.0.0/TestScale_HTTPRoutes/Memory.png differ diff --git a/tests/scale/results/1.0.0/TestScale_HTTPRoutes/TTR.png b/tests/scale/results/1.0.0/TestScale_HTTPRoutes/TTR.png index 7079e762c5..3e98afd88c 100644 Binary files a/tests/scale/results/1.0.0/TestScale_HTTPRoutes/TTR.png and b/tests/scale/results/1.0.0/TestScale_HTTPRoutes/TTR.png differ diff --git a/tests/scale/results/1.0.0/TestScale_HTTPRoutes/results-10s-delay.csv b/tests/scale/results/1.0.0/TestScale_HTTPRoutes/results-10s-delay.csv deleted file mode 100644 index 4a153e5920..0000000000 --- a/tests/scale/results/1.0.0/TestScale_HTTPRoutes/results-10s-delay.csv +++ /dev/null @@ -1,541 +0,0 @@ -# HTTPRoutes,Time to Ready (s),Error -1,0.377127053 -2,0.379007896 -3,0.384165516 -4,0.377453448 -5,0.376946431 -6,0.373370894 -7,0.37584378 -8,0.380403844 -9,0.372546743 -10,0.375794438 -11,0.380632329 -12,0.376386433 -13,0.378915807 -14,0.379204325 -15,0.37596693 -16,0.37958542 -17,0.377282514 -18,0.379717313 -19,0.378369917 -20,0.373178842 -21,0.376339261 -22,0.378602457 -23,0.377526567 -24,0.377001291 -25,0.376519688 -26,0.380761083 -27,0.376727453 -28,0.378244891 -29,0.376102086 -30,0.390081367 -31,0.379204727 -32,0.374149871 -33,0.374552792 -34,0.380052266 -35,0.375878862 -36,0.372185661 -37,0.376191911 -38,0.378403682 -39,0.376402242 -40,0.377471489 -41,0.379175334 -42,0.372701415 -43,0.378738686 -44,0.372680741 -45,0.382039848 -46,0.375800207 -47,0.37550487 -48,0.377931247 -49,0.379590986 -50,0.377128272 -51,0.373003265 -52,0.375941264 -53,0.373991868 -54,0.39105751 -55,0.377170034 -56,0.377030239 -57,0.381733031 -58,0.375338096 -59,0.373153098 -60,0.376193201 -61,0.388617968 -62,0.374054113 -63,0.385547646 -64,0.375017699 -65,0.377488533 -66,0.376849327 -67,0.379391472 -68,0.379960901 -69,0.379718635 -70,0.375880048 -71,0.381485872 -72,0.374207373 -73,0.379410339 -74,0.381847412 -75,0.387592821 -76,0.376002385 -77,0.39215576 -78,0.379137899 -79,0.374343388 -80,0.385782056 -81,0.39245103 -82,0.375190589 -83,0.38281925 -84,0.374855705 -85,0.37289945 -86,0.373501507 -87,0.37820994 -88,0.380233486 -89,0.380667334 -90,0.379121569 -91,0.375309614 -92,0.374657036 -93,0.375086793 -94,0.376036438 -95,0.376015178 -96,0.377403571 -97,0.379801309 -98,0.376607964 -99,0.377697555 -100,0.375681407 -101,0.373934892 -102,0.377706113 -103,0.376752193 -104,0.395867603 -105,0.378681264 -106,0.377698889 -107,0.377541842 -108,0.37642283 -109,0.380908328 -110,0.379868566 -111,0.374663195 -112,0.37438595 -113,0.375067552 -114,0.376440531 -115,0.378176649 -116,0.379116844 -117,0.375201091 -118,0.374259594 -119,0.3744267 -120,0.376328596 -121,0.380620307 -122,0.378390659 -123,0.379844766 -124,0.372985752 -125,0.412397255 -126,0.375595657 -127,0.376272551 -128,0.386955877 -129,0.37447506 -130,0.375448968 -131,0.380362639 -132,0.376260893 -133,0.373944286 -134,0.376313094 -135,0.379435093 -136,0.375936218 -137,0.381314325 -138,0.377391853 -139,0.373517889 -140,0.385729386 -141,0.373902254 -142,0.379373653 -143,0.375899537 -144,0.388511683 -145,0.376187254 -146,0.379788871 -147,0.375300752 -148,0.390292013 -149,0.381669126 -150,0.385621004 -151,0.378633965 -152,0.389065753 -153,0.377405501 -154,0.376104303 -155,0.374026043 -156,0.374427418 -157,0.37564084 -158,0.375753784 -159,0.378136721 -160,0.39491974 -161,0.380198391 -162,0.379450206 -163,0.376706612 -164,0.37725745 -165,0.377553209 -166,0.375263013 -167,0.374094095 -168,0.379675909 -169,0.377853894 -170,0.376477499 -171,0.377793114 -172,0.381162961 -173,0.384042619 -174,0.375727437 -175,0.379877203 -176,0.378395042 -177,0.379994525 -178,0.377707453 -179,0.380739601 -180,0.375452996 -181,0.380526888 -182,0.373074645 -183,0.378006164 -184,0.379156838 -185,0.378706723 -186,0.385387704 -187,0.376950635 -188,0.374606601 -189,0.37554332 -190,0.380540166 -191,0.383980703 -192,0.382047836 -193,0.378620724 -194,0.373282127 -195,0.3733034 -196,0.377690736 -197,0.37933952 -198,0.380632044 -199,0.375993717 -200,0.376418352 -201,0.379006358 -202,0.379446884 -203,0.377732479 -204,0.385805797 -205,0.374459096 -206,0.38552005 -207,0.375223678 -208,0.373540476 -209,0.378472994 -210,0.377177484 -211,0.389642047 -212,0.378047886 -213,0.377214844 -214,0.377815395 -215,0.380875079 -216,0.378271097 -217,0.379704667 -218,0.37518504 -219,0.372301831 -220,0.376468601 -221,0.397737433 -222,0.376489154 -223,0.384317974 -224,0.380045353 -225,0.379258414 -226,0.668478963 -227,0.955560771 -228,0.66530459 -229,0.952927996 -230,0.959922183 -231,0.9604619 -232,1.245051426 -233,0.965650032 -234,1.249752868 -235,1.244404307 -236,1.253600817 -237,1.5425500300000001 -238,1.244638903 -239,1.5327819919999999 -240,1.53215351 -241,1.242658652 -242,1.544132456 -243,1.832230929 -244,1.5336775070000002 -245,1.8136965250000001 -246,1.532000429 -247,1.822073813 -248,1.823666376 -249,1.834192744 -250,2.135217325 -251,2.116891406 -252,2.113768951 -253,2.11533498 -254,2.1031375900000002 -255,2.111154624 -256,2.394712224 -257,2.404185921 -258,2.403343508 -259,2.1192398040000002 -260,2.705668597 -261,2.425563373 -262,2.696382249 -263,2.419572901 -264,1.529554483 -265,2.693073514 -266,2.699861572 -267,2.700676579 -268,2.991834641 -269,2.98815476 -270,2.97689877 -271,2.98189013 -272,3.000758657 -273,3.2595185669999998 -274,2.979182525 -275,3.277262975 -276,3.267665804 -277,3.567651725 -278,3.267198553 -279,3.288899236 -280,3.563272702 -281,3.565613747 -282,3.583468852 -283,3.563264436 -284,3.582336183 -285,3.861246016 -286,3.837587939 -287,3.84217554 -288,3.842180473 -289,3.846408332 -290,4.161861515 -291,3.865296708 -292,4.129283475 -293,4.139912236 -294,4.142976956 -295,4.128571385 -296,4.430305149 -297,4.154271212 -298,4.44798042 -299,4.421583211 -300,4.445049136 -301,4.442084446 -302,4.719657902 -303,4.716392198 -304,4.429964955 -305,5.016602943 -306,4.712858757 -307,4.732749232 -308,5.011187843 -309,4.728635103 -310,5.009524091 -311,4.993745165 -312,4.999831959 -313,5.056810035 -314,5.299232009 -315,5.302893218 -316,5.289425198 -317,5.301798037 -318,5.316324266 -319,5.31578592 -320,5.583774038 -321,5.589489447 -322,5.585255978 -323,5.608319763 -324,5.58157805 -325,5.592130031 -326,5.88500775 -327,5.890452729 -328,5.878823069 -329,5.885211418 -330,5.888762908 -331,6.172425968 -332,6.162267084 -333,6.136797062 -334,6.188560178 -335,6.156708843 -336,6.173356479 -337,6.445751831 -338,6.466813572 -339,6.220670727 -340,6.4681506970000004 -341,6.738561065 -342,6.453701698 -343,6.739565414 -344,6.467595425 -345,6.756441375 -346,6.747759547 -347,7.021372621 -348,6.760149012 -349,7.027064959 -350,6.745440319 -351,7.03684921 -352,7.070410437 -353,7.345334938 -354,7.032605545 -355,7.314518003 -356,7.332957481 -357,7.320349864 -358,7.355330965 -359,7.324711409 -360,7.600443829 -361,7.342456916 -362,7.637212436 -363,7.61203082 -364,7.602042525 -365,7.921439749 -366,7.593254316 -367,7.8878549190000005 -368,7.878748981 -369,7.907913 -370,7.896839348 -371,8.175472822 -372,8.169038771 -373,7.900216592 -374,8.174884591 -375,8.46796053 -376,8.182866008 -377,8.173207242 -378,8.496276912 -379,8.478493881 -380,8.500768496 -381,8.458273396 -382,8.510114914 -383,8.766329095 -384,8.475009324 -385,8.768574381 -386,8.776868361 -387,8.753333382 -388,9.07963734 -389,8.777710038 -390,9.078463601 -391,8.804048374 -392,9.355398808 -393,9.063636003 -394,9.084499365 -395,9.051824166 -396,9.324349684 -397,9.347768059 -398,9.347283251 -399,9.330815998 -400,9.638099697 -401,9.345913202 -402,9.670871432 -403,9.627375821 -404,9.651899708 -405,9.651694779 -406,9.920753963 -407,9.634500355 -408,9.92951956 -409,9.979044 -410,10.009683329 -411,9.910567963 -412,10.242448294 -413,9.913652433 -414,10.229765714 -415,10.221528942 -416,10.209768128 -417,10.509570666 -418,10.178965779 -419,10.523132068 -420,10.492208191 -421,10.487077489 -422,10.561445748 -423,10.50907287 -424,10.806014193 -425,10.791503489 -426,10.776016601 -427,10.767349911 -428,10.829705152 -429,10.794738583 -430,11.100617561 -431,11.107683213 -432,11.082964287 -433,11.059634842 -434,11.134076954 -435,11.10971053 -436,11.351062358 -437,11.089311034 -438,11.673286887 -439,11.387633025 -440,11.385106315 -441,11.654051175 -442,11.379078685 -443,11.660465699 -444,11.653163049 -445,11.660672689 -446,11.942987076 -447,11.689186124 -448,11.998450883 -449,11.954422609 -450,11.93773603 -451,11.986909825 -452,11.944524062 -453,12.243822924 -454,12.225589002 -455,12.230073628 -456,12.244093371 -457,12.234125372 -458,12.290051373 -459,12.546050988 -460,12.516166401 -461,12.536425889 -462,12.58340908 -463,11.110792047 -464,12.533078231 -465,12.751411329 -466,12.638948481 -467,12.961321263 -468,12.919280084 -469,12.677135155 -470,12.955273026 -471,13.241579438 -472,12.914374355 -473,13.219126837 -474,13.246411301 -475,13.16525104 -476,13.141419195 -477,13.472758119 -478,13.147993635 -479,13.453752791 -480,13.44201971 -481,13.441499539 -482,13.685713753 -483,13.680974148 -484,13.67949153 -485,13.690136217 -486,13.680766192 -487,13.652653519 -488,13.946795614 -489,13.931486796 -490,13.660625045 -491,13.683905758 -492,13.961606635999999 -493,13.940987935 -494,13.96423573 -495,14.255488341 -496,14.268293577 -497,14.291670482 -498,14.308522077 -499,14.225553012 -500,14.52736599 -501,14.241945388 -502,14.554484094 -503,14.525298047 -504,14.541811771999999 -505,14.542870557 -506,14.831602849 -507,14.817246728 -508,14.858879123 -509,14.806777274 -510,15.113272545 -511,14.826928065 -512,15.14680419 -513,15.170079005 -514,15.127925279 -515,15.108157698 -516,15.124106934 -517,15.394785854 -518,15.456463428 -519,15.427569691 -520,15.415259891 -521,15.427105775 -522,15.758264989 -523,15.413347719 -524,15.681030381 -525,15.390239277 -526,15.725503161 -527,16.022467555 -528,15.66540755 -529,15.983720688 -530,15.693195366 -531,15.685172415 -532,16.258215425 -533,15.966483273 -534,15.995873056 -535,15.952287141 -536,16.302072662 -537,16.297618715 -538,16.269145592 -539,16.240918609 -540,16.258037758 diff --git a/tests/scale/results/1.0.0/TestScale_HTTPRoutes/results-no-delay.csv b/tests/scale/results/1.0.0/TestScale_HTTPRoutes/results-no-delay.csv index d36ab1e0a2..b9c8382ff2 100644 --- a/tests/scale/results/1.0.0/TestScale_HTTPRoutes/results-no-delay.csv +++ b/tests/scale/results/1.0.0/TestScale_HTTPRoutes/results-no-delay.csv @@ -1,476 +1,1003 @@ # HTTPRoutes,Time to Ready (s),Error -1,0.383770841 -2,0.380656052 -3,0.380485657 -4,0.097146779 -5,0.377347929 -6,0.377125682 -7,0.37841844 -8,0.385015529 -9,0.089883912 -10,0.390451446 -11,0.378943073 -12,0.37725258 -13,0.378384658 -14,0.381131982 -15,0.377787581 -16,0.380144847 -17,0.379475256 -18,0.389799799 -19,0.378165317 -20,0.380534746 -21,0.380772033 -22,0.3836257 -23,0.378608803 -24,0.375804601 -25,0.379483524 -26,0.864854344 -27,0.6795812 -28,0.977591645 -29,0.963147228 -30,0.969510444 -31,0.965014447 -32,0.962099045 -33,1.102333015 -34,0.985620344 -35,1.09272505 -36,0.381917363 -37,0.664688404 -38,0.675943838 -39,0.386166754 -40,1.242637282 -41,1.298552424 -42,1.543812998 -43,1.810062531 -44,1.5296734 -45,1.816329149 -46,1.816646567 -47,1.8099877100000001 -48,1.813467829 -49,1.819065259 -50,2.097738851 -51,1.808799378 -52,2.098034641 -53,2.097764522 -54,2.101585146 -55,2.400262015 -56,2.389255576 -57,2.106989278 -58,2.428372868 -59,2.383734736 -60,2.107034143 -61,2.671407464 -62,2.388532318 -63,2.692242566 -64,2.678131784 -65,2.67492656 -66,2.977667557 -67,2.673629509 -68,2.962085064 -69,2.676131184 -70,2.967315415 -71,3.004304221 -72,2.969081665 -73,3.269183992 -74,2.967210283 -75,3.258638326 -76,3.249844151 -77,3.244806883 -78,3.561841321 -79,3.262699845 -80,3.539880137 -81,3.557550544 -82,3.542547365 -83,3.529880477 -84,3.840924524 -85,3.560927228 -86,3.827644323 -87,3.84080638 -88,3.833997502 -89,3.844903735 -90,4.117603379 -91,3.825068601 -92,4.124596355 -93,4.113714081 -94,4.152235268 -95,0.380709799 -96,3.548207755 -97,4.426409231 -98,4.424787019 -99,4.419769537 -100,4.422898661 -101,4.407654049 -102,4.706503788 -103,4.411553937 -104,4.7018559490000005 -105,4.715109887 -106,4.694933353 -107,4.698910085 -108,4.968518843 -109,4.691070862 -110,4.989728545 -111,4.981128082 -112,5.268325019 -113,4.97037176 -114,4.988800755 -115,5.271450693 -116,5.295387155 -117,5.272393873 -118,5.557244961 -119,5.280333265 -120,0.945514043 -121,5.262105221 -122,5.595568334 -123,5.574083237 -124,5.589206743 -125,5.552453588 -126,5.851369792 -127,1.233311532 -128,5.855350395 -129,5.964516196 -130,5.8471394530000005 -131,5.848075205 -132,5.860991674 -133,6.133646201 -134,6.145907893 -135,6.262283384 -136,6.155964576 -137,6.131266442 -138,6.43505469 -139,6.419861492 -140,6.415178561 -141,6.427191442 -142,6.434567097 -143,6.420440836 -144,6.73294683 -145,6.730110685 -146,6.713098847 -147,6.72025458 -148,6.716271473 -149,6.99301375 -150,6.731237644 -151,6.9977301 -152,7.021136399 -153,6.986413808 -154,7.003035744 -155,7.310633121 -156,7.317694499 -157,7.269159095 -158,7.283305354 -159,7.2859621820000005 -160,7.271657811 -161,7.582609967 -162,7.606299408 -163,7.568498447 -164,7.575934876 -165,7.5890662330000005 -166,7.568851525 -167,7.897321315 -168,7.924148761 -169,7.881869427 -170,7.889892944 -171,7.852905591 -172,8.161059877 -173,7.865857095 -174,8.174832698 -175,8.155205304 -176,8.15236726 -177,8.186432332 -178,8.442840984 -179,8.460232955 -180,8.451031473 -181,8.460532762 -182,8.449290509 -183,8.454978046 -184,8.767023309 -185,8.455161403 -186,8.780258723 -187,8.770188478 -188,9.013137528 -189,8.77018534 -190,6.147018329 -191,9.018709577 -192,9.019548294 -193,8.984440027 -194,9.02189999 -195,9.319032523 -196,9.330832487 -197,9.018864846 -198,9.587344531 -199,9.30696642 -200,9.315142028 -201,9.586112288 -202,9.298035512 -203,9.617492623 -204,9.585508158 -205,9.912826088 -206,9.610537584 -207,9.847416084 -208,9.602136188 -209,9.893778128 -210,10.170237989 -211,9.595497944 -212,10.176073337 -213,9.894899045 -214,10.162996596 -215,9.952862936 -216,10.48106678 -217,10.162380194 -218,10.162266336 -219,10.488419733 -220,10.496234762 -221,10.470966589 -222,10.50118922 -223,10.499243832 -224,10.479238461 -225,10.749923528 -226,10.776629856 -227,10.743692394 -228,10.755004548 -229,10.771770892 -230,11.021130656 -231,10.747373535 -232,11.0750719 -233,11.041423527 -234,11.360049616 -235,11.058005117 -236,11.340463323 -237,11.050966395 -238,11.344113357 -239,11.337348929000001 -240,11.603600576 -241,11.346556197 -242,7.034123487 -243,11.636125892 -244,11.633910611 -245,11.603578323 -246,11.636285816000001 -247,11.897112891 -248,7.307798539 -249,11.918549641 -250,11.906533268 -251,11.8995394 -252,11.900474696 -253,12.201526993 -254,11.944296505 -255,12.190401503 -256,12.226729616 -257,12.230866968 -258,12.232820174 -259,12.47781475 -260,12.240100055 -261,12.520273912 -262,12.214461539 -263,12.522577857 -264,12.795913592 -265,12.494518731 -266,12.79278819 -267,12.759949243 -268,12.887502352 -269,12.770259288 -270,32.395021702 -271,0.374506815 -272,11.901244048 -273,13.075312528 -274,13.059521502 -275,13.077884811 -276,13.340200974 -277,13.146416903 -278,13.345507272 -279,13.37279045 -280,13.371691526 -281,13.669128218 -282,13.353862711 -283,13.678822998 -284,13.672502392 -285,13.651719506 -286,13.658850703 -287,13.661302626 -288,13.978328505 -289,13.940908396 -290,13.936206684 -291,13.947572959 -292,13.945947007000001 -293,14.224175465 -294,13.956113138 -295,14.206128572 -296,14.230369493 -297,14.206886939 -298,14.229910918 -299,14.239633901 -300,14.225149174 -301,14.506786921 -302,14.208907617 -303,14.234825371 -304,14.495108177 -305,14.524795147 -306,14.760110319 -307,14.822820149 -308,14.837508014 -309,14.80072829 -310,14.786506376 -311,15.092856733 -312,15.083235162 -313,14.806087759 -314,15.034910757 -315,15.056887306 -316,15.357751318 -317,15.065564849 -318,15.370088909 -319,15.353822604 -320,15.357593312 -321,15.40037503 -322,15.66120014 -323,15.362804896 -324,15.705511462 -325,15.645579946 -326,15.618289891 -327,13.942397622 -328,15.922382777 -329,15.947337829 -330,15.945278678 -331,15.943202858 -332,16.0073185 -333,15.927050813 -334,16.226592783 -335,16.187233433 -336,16.246480633 -337,16.237334103 -338,16.227720886 -339,15.946216830000001 -340,16.534781017 -341,16.507881368 -342,16.254285373 -343,16.576747317 -344,16.54642211 -345,16.545732963 -346,16.843086099 -347,16.232294914 -348,16.8136862 -349,16.820897867 -350,16.821277668 -351,16.816728191 -352,16.839415343 -353,17.153682019 -354,17.113154336 -355,17.144029836 -356,17.170722512 -357,17.092822527 -358,17.411367673 -359,17.46173771 -360,17.389891699 -361,17.43299391 -362,17.437457132 -363,17.414578314 -364,17.704094626 -365,17.721105545 -366,17.74807521 -367,17.726058221 -368,17.708324845 -369,17.737824773 -370,17.997769394 -371,17.754324679 -372,18.010773053 -373,18.012075537 -374,18.307598961 -375,18.0699199 -376,18.294276854 -377,18.316656373 -378,18.066631139 -379,18.405606592 -380,18.40470596 -381,18.629784072 -382,18.326288914 -383,18.641559759 -384,18.633411613 -385,18.328779618 -386,18.597437479 -387,18.704598885 -388,18.883604591 -389,18.89240392 -390,18.918688037 -391,18.917226774 -392,18.892069689 -393,18.95492239 -394,19.218288714 -395,19.169093596 -396,14.873706174 -397,19.180519994 -398,19.605025347 -399,19.178308424 -400,19.494217796 -401,19.446356302 -402,19.495733812 -403,19.803003043 -404,19.464744612 -405,19.800134155 -406,19.496257638 -407,20.016203759 -408,19.794462513 -409,15.441952501 -410,20.115836025 -411,19.796751609 -412,20.029376436 -413,20.171004803 -414,20.08184473 -415,20.325934549 -416,20.333374709 -417,20.093339258 -418,20.356575706 -419,20.411727088 -420,16.011443907 -421,20.69810284 -422,20.63230193 -423,20.598879281 -424,20.387827233 -425,20.761187109 -426,20.66080738 -427,20.629535227 -428,20.909389827 -429,20.923543042 -430,20.637488247 -431,21.245912296 -432,20.992789011 -433,20.979198568 -434,21.225965227 -435,21.205642512 -436,21.212557428 -437,21.242860302 -438,21.523383588 -439,21.241824455 -440,21.499328997 -441,21.549901038 -442,21.520652532 -443,21.780803876 -444,21.562871924 -445,21.829179892 -446,21.800966716 -447,21.809903338 -448,21.7910108 -449,21.83978298 -450,22.082463304 -451,22.126015909 -452,21.831557769 -453,22.248887912 -454,22.126643502 -455,48.365839369 -456,0.376811942 -457,21.215613992 -458,22.445884133 -459,22.380127349 -460,22.265444665 -461,22.725412845 -462,22.36771563 -463,22.659603907 -464,22.65619497 -465,22.723849525 -466,22.700845057 -467,22.719689394 -468,22.962046973 -469,22.789032688 -470,22.975549597 -471,22.986912774 -472,22.949533459 -473,23.289465192 -474,22.997424888 -475,23.240723 +1,0.180642267 +2,0.175798644 +3,0.180055404 +4,0.184884811 +5,0.175715341 +6,0.17714107 +7,0.200388764 +8,0.178864498 +9,0.18247283 +10,0.179748302 +11,0.181745357 +12,0.179667373 +13,0.178753984 +14,0.205107523 +15,0.181276954 +16,0.177085027 +17,0.177286821 +18,0.17910343 +19,0.176510174 +20,0.188810287 +21,0.178625562 +22,0.198373377 +23,0.182303729 +24,0.178430475 +25,0.183556461 +26,0.176153339 +27,0.186438448 +28,0.18921829 +29,0.178907192 +30,0.178512115 +31,0.250011799 +32,0.175861365 +33,0.179647188 +34,0.181664321 +35,0.185458204 +36,0.182106794 +37,0.176220326 +38,0.175555665 +39,0.187432081 +40,0.175208952 +41,0.179343554 +42,0.17819142 +43,0.17596777 +44,0.1920162 +45,0.176224971 +46,0.176932133 +47,0.192994202 +48,0.181322177 +49,0.180007936 +50,0.189950013 +51,0.17815618 +52,0.177027295 +53,0.186257341 +54,0.186444411 +55,0.180504671 +56,0.177053924 +57,0.265313575 +58,0.196324888 +59,0.201612386 +60,0.177576885 +61,0.175185963 +62,0.184342659 +63,0.176307499 +64,0.175019753 +65,0.178061392 +66,0.187440786 +67,0.183783478 +68,0.182012878 +69,0.179946771 +70,0.18122838 +71,0.181500126 +72,0.193568684 +73,0.179791629 +74,0.179707453 +75,0.195033056 +76,0.182119143 +77,0.187205098 +78,0.184456346 +79,0.180517618 +80,0.175825425 +81,0.178523647 +82,0.176265371 +83,0.177759445 +84,0.178235776 +85,0.182564314 +86,0.184302372 +87,0.1780885 +88,0.178709767 +89,0.176793396 +90,0.177582826 +91,0.181564596 +92,0.181163975 +93,0.18589236 +94,0.174760273 +95,0.190998694 +96,0.179270221 +97,0.179383768 +98,0.184475168 +99,0.177327875 +100,0.178498202 +101,0.18474112 +102,0.177803954 +103,0.177984578 +104,0.176976487 +105,0.180781047 +106,0.181055287 +107,0.179096305 +108,0.180411728 +109,0.176961689 +110,0.191826226 +111,0.180668721 +112,0.177527122 +113,0.185478496 +114,0.180944484 +115,0.182057095 +116,0.181670193 +117,0.17993014 +118,0.181088428 +119,0.183831015 +120,0.176821487 +121,0.17828494 +122,0.177087767 +123,0.187420545 +124,0.181666006 +125,0.17800609 +126,0.182537127 +127,0.181666688 +128,0.18143996 +129,0.186480859 +130,0.177624574 +131,0.177151838 +132,0.179346698 +133,0.180372834 +134,0.18129542 +135,0.17744806 +136,0.181358076 +137,0.183031381 +138,0.180422395 +139,0.182140417 +140,0.178747309 +141,0.177798725 +142,0.17654436 +143,0.185468443 +144,0.180328768 +145,0.177769124 +146,0.181233426 +147,0.182300771 +148,0.177704553 +149,0.175583994 +150,0.178229371 +151,0.184020646 +152,0.183524848 +153,0.181776798 +154,0.180427367 +155,0.189892528 +156,0.191243991 +157,0.471520662 +158,0.183025661 +159,0.18119594 +160,0.469773824 +161,0.180079178 +162,0.183294293 +163,0.183113756 +164,0.187139043 +165,0.179602598 +166,0.191114661 +167,0.194314748 +168,0.473927922 +169,0.187399592 +170,0.185907078 +171,0.468726078 +172,0.48203109 +173,0.192091825 +174,0.182607985 +175,0.183916619 +176,0.193411088 +177,0.467843453 +178,0.190814616 +179,0.472128942 +180,0.486860447 +181,0.476735841 +182,0.187887895 +183,0.47304764 +184,0.189048905 +185,0.467278451 +186,0.474093495 +187,0.465316383 +188,0.487247642 +189,0.470184964 +190,0.47092468 +191,0.4710119 +192,0.467747641 +193,0.46766052 +194,0.475368333 +195,0.471919414 +196,0.469862861 +197,0.475892679 +198,0.465185148 +199,0.469313146 +200,0.470863152 +201,0.473661391 +202,0.473400889 +203,0.471707143 +204,0.464505599 +205,0.469108848 +206,0.464108711 +207,0.47257847 +208,0.466186191 +209,0.47356131 +210,0.469547351 +211,0.468420449 +212,0.48570324 +213,0.465475639 +214,0.464697674 +215,0.500620801 +216,0.479900773 +217,0.472984951 +218,0.472136524 +219,0.465724485 +220,0.504683589 +221,0.466725261 +222,0.473059285 +223,0.472381897 +224,0.466013904 +225,0.476043144 +226,0.472341971 +227,0.470289711 +228,0.475620564 +229,0.470120682 +230,0.475601064 +231,0.47971675 +232,0.47170492 +233,0.475804163 +234,0.466776766 +235,0.473565152 +236,0.474945244 +237,0.469006523 +238,0.473659402 +239,0.469227362 +240,0.48634288 +241,0.487730181 +242,0.47317721 +243,0.471380589 +244,0.465073996 +245,1.093621746 +246,0.49433934 +247,0.223446442 +248,0.476730658 +249,0.474868212 +250,0.488409171 +251,0.474710689 +252,0.471222537 +253,0.475589409 +254,0.470291878 +255,0.471056189 +256,0.474987795 +257,0.476543664 +258,0.476898388 +259,0.489624882 +260,0.470819031 +261,0.468674817 +262,0.473045681 +263,0.468036475 +264,0.477522243 +265,0.473275173 +266,0.486494411 +267,0.470741966 +268,0.470403796 +269,0.469265535 +270,0.479935153 +271,0.479231811 +272,0.470435309 +273,0.484671358 +274,0.499360244 +275,0.471344081 +276,0.506206218 +277,0.469763362 +278,0.47317602 +279,0.488259381 +280,0.471411885 +281,0.489260453 +282,0.472505122 +283,0.471408004 +284,0.469567211 +285,0.466242636 +286,0.47249529 +287,0.469789488 +288,0.471158036 +289,0.472183719 +290,0.489034245 +291,0.47194114 +292,0.470023093 +293,0.48789928 +294,0.470652296 +295,0.489559747 +296,0.473001493 +297,0.477775703 +298,0.467288052 +299,0.473196166 +300,0.468740746 +301,0.472309855 +302,0.471139702 +303,0.483567481 +304,0.474562178 +305,0.476292619 +306,0.468479927 +307,0.467915695 +308,0.470471512 +309,0.468353939 +310,0.47367419 +311,0.47008994 +312,0.470528586 +313,0.471160005 +314,0.465196956 +315,0.467318447 +316,0.46800346 +317,0.467687262 +318,0.464390958 +319,0.474726546 +320,0.471858252 +321,0.483027657 +322,0.47376917 +323,0.481518 +324,0.473308365 +325,0.470998626 +326,0.482107453 +327,0.475150038 +328,0.470504854 +329,0.472923009 +330,0.467139495 +331,0.472427813 +332,0.725221691 +333,0.475962422 +334,0.494187172 +335,0.476340771 +336,0.48031322 +337,1.18496815 +338,0.470213405 +339,0.469165592 +340,0.474158149 +341,0.467901054 +342,0.486010552 +343,0.468173655 +344,0.483982891 +345,0.468689021 +346,0.491555939 +347,0.465185699 +348,0.473682959 +349,0.470984571 +350,0.474051279 +351,0.46867483 +352,0.475889078 +353,0.467867765 +354,0.465563291 +355,0.474320443 +356,0.468577546 +357,0.476190094 +358,0.46784297 +359,0.477282023 +360,0.485482928 +361,0.467601101 +362,0.473796648 +363,0.473073935 +364,0.466480056 +365,0.482640448 +366,0.511004601 +367,0.471660224 +368,0.47491812 +369,0.415065998 +370,0.482107095 +371,0.467572299 +372,0.474629451 +373,0.47905021 +374,0.473194441 +375,0.487298273 +376,0.480513726 +377,0.468495892 +378,0.482955393 +379,0.47096244 +380,0.471102319 +381,0.474623702 +382,0.492194827 +383,0.486765805 +384,0.474962833 +385,0.467472998 +386,0.46733427 +387,0.480207973 +388,0.467865529 +389,0.487389236 +390,0.471361259 +391,0.47127741 +392,0.467938829 +393,0.481689875 +394,0.46641978 +395,0.471498001 +396,0.473526191 +397,0.500195344 +398,0.465060096 +399,0.473147651 +400,0.468734932 +401,0.481888042 +402,0.472790348 +403,0.472783216 +404,0.485787191 +405,0.468890913 +406,0.478033275 +407,0.469677149 +408,0.46777024 +409,0.472865224 +410,0.466788403 +411,0.462886473 +412,0.467301466 +413,0.469640119 +414,0.4696917 +415,0.468514981 +416,0.469740907 +417,0.473443651 +418,0.466009314 +419,0.464868017 +420,0.495817243 +421,0.468515574 +422,0.481347946 +423,0.487013078 +424,0.47475606 +425,0.484697352 +426,0.477367843 +427,0.469882207 +428,0.469178359 +429,0.469568958 +430,0.479914513 +431,0.469667151 +432,0.468416744 +433,0.473005231 +434,0.470651081 +435,0.472059395 +436,0.466524736 +437,0.468914425 +438,0.483110151 +439,0.471856082 +440,0.465929901 +441,0.469490715 +442,0.483247804 +443,0.467403444 +444,0.470225588 +445,0.465355218 +446,0.467803441 +447,0.465019048 +448,0.463013827 +449,0.474524577 +450,0.470248689 +451,0.479254245 +452,0.478909137 +453,0.472645252 +454,0.478532483 +455,0.466801051 +456,0.468957286 +457,0.46797787 +458,0.474848568 +459,0.470585513 +460,0.468963827 +461,0.469271144 +462,0.478342506 +463,0.480397061 +464,0.47016222 +465,0.476335835 +466,0.466285552 +467,0.463480997 +468,0.466686451 +469,0.471562679 +470,0.470457753 +471,0.468256988 +472,0.465765915 +473,0.469404205 +474,0.505697051 +475,0.467550711 +476,0.469099376 +477,0.47351364 +478,0.465028464 +479,0.469969879 +480,0.465406336 +481,0.468595737 +482,0.475735487 +483,0.46740757 +484,0.472865429 +485,0.476346115 +486,0.465095342 +487,0.464865977 +488,0.469311317 +489,0.466868599 +490,0.46502121 +491,0.471484352 +492,0.468972233 +493,0.469068508 +494,0.473820969 +495,0.46567541 +496,0.466766369 +497,0.469872195 +498,0.47568092 +499,0.469689371 +500,0.472277261 +501,0.469985242 +502,0.469075123 +503,0.47089101 +504,0.474411111 +505,0.47017549 +506,0.470323991 +507,0.468336882 +508,0.46659827 +509,0.467516553 +510,0.47552044 +511,0.470519462 +512,0.469609665 +513,0.470927976 +514,0.488139269 +515,0.473090732 +516,0.470205041 +517,0.472265942 +518,0.476312993 +519,0.468468714 +520,0.477051772 +521,0.467964666 +522,0.464184698 +523,0.469296463 +524,0.462434843 +525,0.484558043 +526,0.47066103 +527,0.471017316 +528,0.479765536 +529,0.474388762 +530,0.466759317 +531,0.467628049 +532,0.475693426 +533,0.466908981 +534,0.475523295 +535,0.468523094 +536,0.479623516 +537,0.470551685 +538,0.467967591 +539,1.09466888 +540,0.465794704 +541,0.472259636 +542,0.487442582 +543,0.483695021 +544,0.463451577 +545,0.47365141 +546,0.473827545 +547,0.470562346 +548,0.46950801 +549,0.467894571 +550,0.463850308 +551,0.47195295 +552,0.479130132 +553,0.470163104 +554,0.47242397 +555,0.468654843 +556,0.468717143 +557,0.472122761 +558,0.467944563 +559,0.467309168 +560,0.475286702 +561,0.49329471 +562,0.488617179 +563,0.474853623 +564,0.471474831 +565,0.466867819 +566,0.474019322 +567,0.47621309 +568,0.467364769 +569,0.49959175 +570,0.482347708 +571,0.465083846 +572,0.471826345 +573,0.471500068 +574,0.471048397 +575,0.47203872 +576,0.472888549 +577,0.467706537 +578,0.466456513 +579,0.466172951 +580,0.488322213 +581,0.469759331 +582,0.488251692 +583,0.478353874 +584,0.471088633 +585,0.478042002 +586,0.482380239 +587,0.49734024 +588,0.484038798 +589,0.494815538 +590,0.470201991 +591,0.47723405 +592,0.470824249 +593,0.471390837 +594,0.472827924 +595,0.491550312 +596,0.464303901 +597,0.4705594 +598,0.473421531 +599,0.471482449 +600,0.479115396 +601,0.468807215 +602,0.473963289 +603,0.471487185 +604,0.469079909 +605,0.473111466 +606,0.467888832 +607,0.469532689 +608,0.468436184 +609,0.471607578 +610,0.491078892 +611,0.471711771 +612,0.471151483 +613,0.46742629 +614,0.477120448 +615,0.470695282 +616,0.484910699 +617,0.471761236 +618,0.470989398 +619,0.472131452 +620,0.469535508 +621,0.468513577 +622,0.46843084 +623,0.472195738 +624,0.469767529 +625,0.47544004 +626,0.471585228 +627,0.473084259 +628,0.471333171 +629,0.467654188 +630,0.467234979 +631,0.474175791 +632,0.471033706 +633,0.471652099 +634,0.471999668 +635,0.826305202 +636,0.466077328 +637,0.465445165 +638,0.46943783 +639,0.474026477 +640,0.471928507 +641,0.470045037 +642,0.467620395 +643,0.466461065 +644,0.722798784 +645,0.468348516 +646,0.466032403 +647,0.477279275 +648,0.467950033 +649,0.466237201 +650,0.48578357 +651,0.493842935 +652,0.468545326 +653,0.472105591 +654,0.480931632 +655,0.516009667 +656,0.486868684 +657,0.468661996 +658,0.478882292 +659,0.490604131 +660,0.478730481 +661,0.472567587 +662,0.469935705 +663,0.471007436 +664,0.464050068 +665,0.472503754 +666,0.470619385 +667,0.473306142 +668,0.467538775 +669,0.467838603 +670,0.473301785 +671,0.482115864 +672,0.477955491 +673,0.471887571 +674,0.4837311 +675,0.486613784 +676,0.476592751 +677,0.467397732 +678,0.473271579 +679,0.474888478 +680,0.473776598 +681,0.474903812 +682,0.473867784 +683,0.483772426 +684,0.482950425 +685,0.474227898 +686,0.479101845 +687,0.5008312 +688,0.472080157 +689,0.467817596 +690,0.46455834 +691,0.474099518 +692,0.47736335 +693,0.470811323 +694,0.477226609 +695,0.479127 +696,0.472463696 +697,0.474910157 +698,0.48450114 +699,0.467315917 +700,0.474141877 +701,0.497151381 +702,0.669543853 +703,0.469932075 +704,0.472480327 +705,0.473160605 +706,0.480987304 +707,0.472360432 +708,0.468623703 +709,0.477789541 +710,0.470897384 +711,0.474844144 +712,0.47167805 +713,0.512138372 +714,0.486822405 +715,0.475255673 +716,0.487685773 +717,0.493712685 +718,0.48559083 +719,0.483217663 +720,0.475147222 +721,0.476405489 +722,0.472264863 +723,0.480024908 +724,0.479634761 +725,0.473896603 +726,0.477112438 +727,0.490079886 +728,0.667398888 +729,0.670035791 +730,0.475021325 +731,0.477946378 +732,0.669724922 +733,0.483898386 +734,0.484569122 +735,0.666722978 +736,0.669459054 +737,0.672609896 +738,0.488843567 +739,0.667778767 +740,0.669335663 +741,0.665917583 +742,0.494021704 +743,0.674725698 +744,0.674626894 +745,0.486426529 +746,0.669831546 +747,0.478087179 +748,0.482909267 +749,0.677852784 +750,0.489301718 +751,0.672531933 +752,0.670399876 +753,0.671771716 +754,0.669845566 +755,0.67844154 +756,0.49952632 +757,0.671889033 +758,0.47948432 +759,0.4896225 +760,0.681537027 +761,0.671366913 +762,0.673480153 +763,0.666671722 +764,0.664481047 +765,0.493161005 +766,1.473090268 +767,0.667360264 +768,0.672761876 +769,0.672445101 +770,0.512260092 +771,0.672229452 +772,0.678517149 +773,0.670989383 +774,0.674497195 +775,0.667487522 +776,0.66941202 +777,0.677190335 +778,0.48536923 +779,0.665201767 +780,0.669112278 +781,0.671845662 +782,0.675760112 +783,0.669697473 +784,0.674529657 +785,1.4697405190000001 +786,0.669707285 +787,0.685950475 +788,0.685490732 +789,0.673492253 +790,0.666552712 +791,0.687502524 +792,0.669669006 +793,0.675690265 +794,0.668665015 +795,0.675104854 +796,0.673437539 +797,0.677831243 +798,0.683615647 +799,0.671068945 +800,0.672461285 +801,0.674199475 +802,0.668837472 +803,0.671981142 +804,0.669124643 +805,0.667840471 +806,0.669703101 +807,0.672926261 +808,0.671632222 +809,0.67911818 +810,0.668906497 +811,0.67686612 +812,0.67820144 +813,0.670573891 +814,0.673185885 +815,0.668739486 +816,0.67193227 +817,0.718659004 +818,0.668403858 +819,0.67155456 +820,0.706356762 +821,0.693762085 +822,0.674610562 +823,0.672983083 +824,0.670362659 +825,0.671288367 +826,0.664708424 +827,1.50149149 +828,0.689010228 +829,0.738624637 +830,0.687252861 +831,0.669205918 +832,0.66990102 +833,0.688319915 +834,0.665046884 +835,0.682393878 +836,0.67240212 +837,0.671092744 +838,0.674002984 +839,0.668979827 +840,0.667546164 +841,0.676731456 +842,0.673053238 +843,0.672158969 +844,0.670286061 +845,0.684697753 +846,0.67219836 +847,0.671245113 +848,0.679713819 +849,0.669924297 +850,0.671602141 +851,0.66796187 +852,0.669868318 +853,0.667766047 +854,0.670000433 +855,0.678690017 +856,0.669637439 +857,0.666190971 +858,0.66953525 +859,0.669787168 +860,0.671563906 +861,0.66850443 +862,0.681874244 +863,0.666568788 +864,0.673598619 +865,0.673024557 +866,0.673755813 +867,0.696636394 +868,0.692649986 +869,0.681335554 +870,0.674886391 +871,0.67493335 +872,0.669294132 +873,0.666757681 +874,0.666592764 +875,0.674407847 +876,0.687156777 +877,0.669914873 +878,0.668836938 +879,0.680802596 +880,1.179016047 +881,0.689759299 +882,0.675263249 +883,0.672577558 +884,0.670159482 +885,0.669332438 +886,1.672386427 +887,0.669596331 +888,1.201976072 +889,0.663991496 +890,0.681345459 +891,0.686543324 +892,0.684694755 +893,0.668889883 +894,0.66812306 +895,0.670277719 +896,0.670828448 +897,1.187661974 +898,0.669920073 +899,0.685789822 +900,0.892782599 +901,0.67228013 +902,0.664346741 +903,0.672692055 +904,0.669233983 +905,0.671876365 +906,0.668162267 +907,0.678915617 +908,0.667992598 +909,0.665863374 +910,0.663952164 +911,0.671701979 +912,0.676562041 +913,0.67208203 +914,0.666224144 +915,0.669636569 +916,0.673810686 +917,0.587948698 +918,0.666037157 +919,0.669333618 +920,0.672814663 +921,0.667380749 +922,0.667476818 +923,0.672506413 +924,0.668500404 +925,0.66731807 +926,0.6706213 +927,0.671518674 +928,0.678447688 +929,0.678058211 +930,0.6807431 +931,0.66804263 +932,0.674729972 +933,0.673959005 +934,0.677648725 +935,0.676215451 +936,0.671735541 +937,0.670462356 +938,0.688077422 +939,0.668618926 +940,0.664319633 +941,0.703610749 +942,0.709338807 +943,0.668935906 +944,0.667980615 +945,0.667649245 +946,0.675773044 +947,0.699541125 +948,0.706644797 +949,0.471842318 +950,0.675074576 +951,0.673531947 +952,0.672806752 +953,0.667946242 +954,0.665863115 +955,0.671425378 +956,0.672976737 +957,0.667779764 +958,0.681346236 +959,0.675853229 +960,0.669985939 +961,0.66970822 +962,0.672371635 +963,0.670805927 +964,0.667503962 +965,0.669879387 +966,0.676660975 +967,0.567186056 +968,0.689664464 +969,0.671445906 +970,0.668388431 +971,0.668661303 +972,0.669119226 +973,0.667934155 +974,0.669943781 +975,0.67605033 +976,0.670516447 +977,0.83114747 +978,0.697752084 +979,0.673964152 +980,0.66932688 +981,0.669645187 +982,0.738075718 +983,0.689077216 +984,0.674020701 +985,0.669637133 +986,0.669468049 +987,0.670446155 +988,0.671458132 +989,0.677136077 +990,0.674740128 +991,0.667940642 +992,0.666841975 +993,0.673896719 +994,0.677699597 +995,0.667682217 +996,0.673588164 +997,0.671318614 +998,0.671653463 +999,0.47661524 +1000,0.67041907 +Test Start,Test End,Test End + 10s,Duration +1697510170,1697511171,1697511181,16m41.430620362s diff --git a/tests/scale/results/1.0.0/TestScale_HTTPRoutes/results.csv b/tests/scale/results/1.0.0/TestScale_HTTPRoutes/results.csv index 36b989c8c4..acb0fabdc8 100644 --- a/tests/scale/results/1.0.0/TestScale_HTTPRoutes/results.csv +++ b/tests/scale/results/1.0.0/TestScale_HTTPRoutes/results.csv @@ -1,1001 +1,1003 @@ # HTTPRoutes,Time to Ready (s),Error -1,0.385042 -2,0.382147622 -3,0.379821284 -4,0.379737071 -5,0.379980332 -6,0.380883635 -7,0.38194278 -8,0.384607201 -9,0.399597381 -10,0.383712896 -11,0.385982892 -12,0.381628865 -13,0.383431779 -14,0.380270067 -15,0.378141185 -16,0.387142744 -17,0.380637439 -18,0.383168174 -19,0.384756966 -20,0.384278116 -21,0.384212425 -22,0.38308232 -23,0.382446783 -24,0.388838492 -25,0.387291714 -26,0.380083607 -27,0.446689351 -28,0.385081587 -29,0.385361996 -30,0.387850995 -31,0.377779806 -32,0.39151114 -33,0.392356289 -34,0.427523592 -35,0.381354729 -36,0.384113814 -37,0.3801672 -38,0.379487191 -39,0.383137692 -40,0.381893477 -41,0.380364125 -42,0.381806096 -43,0.388057082 -44,0.384284274 -45,0.385182757 -46,0.382352598 -47,0.389805938 -48,0.400674567 -49,0.392630929 -50,0.384568685 -51,0.38524618 -52,0.379389459 -53,0.386951052 -54,0.384325197 -55,0.382886417 -56,0.389441321 -57,0.383999268 -58,0.384693315 -59,0.380554606 -60,0.38608117 -61,0.38488563 -62,0.382124047 -63,0.389271177 -64,0.387389756 -65,0.383198306 -66,0.682481106 -67,0.67444258 -68,0.970568223 -69,1.014999035 -70,0.973646955 -71,0.969847787 -72,0.991308799 -73,1.254453876 -74,0.985181179 -75,1.262578394 -76,1.260318123 -77,1.260580925 -78,1.263775268 -79,1.289057704 -80,1.271338337 -81,1.553105417 -82,1.559913882 -83,1.678703766 -84,1.591168978 -85,1.86223424 -86,1.844302761 -87,1.547678299 -88,1.847528396 -89,1.894979287 -90,2.144782547 -91,1.8621425139999999 -92,2.148959875 -93,2.129363923 -94,1.837864742 -95,1.846953274 -96,2.267352278 -97,2.155376881 -98,2.420211349 -99,2.414074736 -100,2.424816646 -101,2.721810069 -102,2.473841073 -103,2.748770853 -104,2.4316491239999998 -105,2.7178590160000002 -106,2.7861162520000002 -107,2.738862137 -108,3.041759396 -109,3.018813415 -110,2.708517359 -111,3.013564487 -112,3.2934272399999998 -113,3.009758894 -114,3.02650995 -115,3.2986881009999998 -116,3.141020256 -117,3.325437798 -118,3.583768892 -119,3.2999014 -120,3.595940144 -121,3.034143626 -122,3.605313412 -123,3.54850254 -124,3.617334108 -125,3.637293178 -126,3.9288259610000003 -127,3.881287462 -128,4.61811947 -129,3.288423539 -130,3.620755448 -131,3.934249756 -132,4.054183417 -133,4.213836762 -134,4.175238814 -135,4.175630634 -136,4.163063512 -137,4.202729586 -138,4.491626064 -139,4.263553526 -140,4.506628162 -141,4.489577994 -142,4.510760647 -143,4.452220136 -144,4.782550678 -145,4.77699404 -146,4.478480558 -147,5.038061317 -148,4.77519762 -149,4.769659284 -150,5.067124139 -151,5.057393805 -152,5.005089052 -153,5.119731288 -154,5.093113355 -155,5.183158537 -156,5.359393964 -157,5.057175814 -158,5.366343244 -159,5.405545673 -160,5.653089137 -161,5.355915223 -162,5.65874337 -163,5.637884689 -164,5.64887917 -165,5.615412919 -166,5.6381729069999995 -167,5.979529192 -168,5.9348313820000005 -169,5.648260129 -170,5.950106851 -171,6.066447468 -172,6.096931754 -173,5.764695952 -174,6.380062327 -175,5.66656968 -176,6.272783281 -177,6.221946278 -178,6.240302251 -179,6.414905571 -180,6.563974786 -181,6.522895801 -182,6.493242072 -183,6.443757982 -184,6.974011986 -185,6.52774633 -186,6.81419512 -187,6.790342793 -188,7.758369739 -189,5.937684343 -190,7.076457047 -191,7.123723743 -192,7.557060908 -193,6.5163503590000005 -194,7.108920551 -195,7.115321489 -196,7.40913478 -197,7.239179751 -198,7.395974213 -199,7.39538685 -200,7.517393048 -201,7.501897832 -202,7.814676309 -203,7.123856981 -204,7.700579924 -205,7.840622743 -206,7.722739872 -207,7.55302047 -208,7.687103622 -209,7.480004886 -210,8.156368151 -211,7.804533559 -212,8.288270125 -213,8.005565296 -214,8.004393888 -215,8.462546078999999 -216,8.024345326 -217,8.313862247 -218,8.344053023 -219,8.579421373 -220,8.289709383 -221,8.611967649 -222,8.585668337 -223,8.429041015 -224,8.580136978 -225,8.951947128 -226,8.578459164 -227,8.915455951 -228,8.857611337 -229,8.705042557 -230,9.170933217 -231,8.623892856 -232,8.921918869 -233,9.190434372 -234,9.159182995 -235,9.15488773 -236,9.471499052 -237,9.21651202 -238,9.233143149 -239,9.447514648 -240,9.586912402 -241,9.449483537 -242,9.490955118 -243,9.446495527 -244,9.726599567 -245,9.881813864 -246,9.584750159 -247,9.751090104 -248,10.009108372 -249,10.014152559 -250,9.758181883 -251,9.893224133 -252,10.106693395 -253,10.313508271 -254,10.072765931 -255,10.138212372 -256,9.296253551 -257,10.311199826 -258,10.128726716 -259,10.415380248 -260,10.443652387 -261,10.6420528 -262,10.614467204 -263,10.386039166 -264,10.726860816 -265,10.376580013 -266,10.740635426 -267,10.942521519 -268,10.901106595 -269,10.877931949 -270,11.012175771 -271,10.961585794 -272,10.931964082 -273,11.232116387 -274,11.210639484 -275,11.275499703 -276,11.210823213 -277,11.219041249 -278,11.533387578 -279,11.215886923 -280,11.546196755 -281,11.565836952 -282,11.571423236 -283,11.512723487 -284,11.528767818 -285,11.849987387 -286,11.864171536 -287,11.635332997999999 -288,11.889636363 -289,11.76892983 -290,12.207261147 -291,11.792182402 -292,12.335200656 -293,12.097684492 -294,12.103063918 -295,12.085841953 -296,12.469395154 -297,12.131030775 -298,12.405417177 -299,12.391954757 -300,12.379394268 -301,12.565762398 -302,12.408597337 -303,12.684903978 -304,12.708412852 -305,12.371383459 -306,12.693234396 -307,12.719631498 -308,13.040710355 -309,12.927983789 -310,12.977783187 -311,12.807163133 -312,13.379932757 -313,12.709859358 -314,13.291819766 -315,13.060006443 -316,13.303808197 -317,13.284253253 -318,13.29751204 -319,13.271607192 -320,13.417060694 -321,13.70029229 -322,13.377623602 -323,13.622685939 -324,13.528297491 -325,13.754196761 -326,13.875762092 -327,13.825902329 -328,13.931361 -329,13.758134165 -330,14.109854762 -331,14.010441761 -332,13.643428951 -333,14.144946622 -334,14.185458655 -335,14.392189885 -336,14.230359781 -337,14.32012791 -338,14.19807588 -339,14.256977214 -340,14.548063136 -341,14.465300027 -342,14.723959772 -343,14.339139374 -344,14.87078413 -345,14.443061959 -346,14.875051509 -347,14.87759207 -348,14.77773053 -349,14.769552148 -350,15.10367367 -351,15.117947396 -352,14.855381983000001 -353,15.017789933 -354,15.145974441 -355,15.32010491 -356,15.068143838 -357,15.275396618 -358,15.23281466 -359,15.189269566 -360,15.577918159 -361,15.372148684 -362,15.671111492 -363,15.303942189 -364,15.668559627 -365,15.797643447 -366,15.663171149 -367,15.933899011 -368,15.665407129 -369,15.852120998 -370,15.889898323 -371,15.963718218 -372,15.977373067 -373,16.163176669 -374,15.927281837 -375,14.736798144 -376,16.156760056 -377,16.468384748 -378,16.348876685 -379,16.398853451 -380,15.976793608 -381,16.514288568 -382,16.639794391 -383,16.453938836 -384,16.316995395 -385,16.743604474 -386,16.506053018 -387,16.783119188 -388,16.607363112 -389,17.008478779 -390,16.823745788 -391,17.060766644 -392,17.077873714 -393,17.046714215 -394,17.110553004 -395,17.047490045 -396,17.063928259 -397,17.373884652 -398,17.157657931 -399,17.489016202 -400,17.351843145 -401,17.47217862 -402,17.375319708 -403,17.459777531 -404,17.715900978 -405,17.646793539 -406,17.757515262 -407,17.606350828 -408,17.97744462 -409,17.740024189 -410,18.029429741 -411,18.008791589 -412,17.638611049 -413,18.059927561 -414,18.278911876 -415,17.95843105 -416,18.312597149 -417,18.225649801 -418,18.394198185 -419,18.520789779 -420,18.228401405 -421,18.595082022 -422,18.291398404 -423,18.588732109 -424,18.74791212 -425,18.599117429 -426,18.664008051 -427,18.564616431 -428,18.918485469 -429,18.92435889 -430,18.850794908 -431,19.141210012 -432,18.892189834 -433,19.117511665 -434,18.917876091 -435,19.161540682 -436,19.123684349 -437,19.162010725 -438,19.427617013 -439,19.521264909 -440,19.21184219 -441,19.554330723 -442,19.657042827 -443,19.452228143 -444,19.553777667 -445,19.680911127 -446,19.851861609 -447,19.672833393 -448,19.724328509 -449,19.616568579 -450,19.958856777 -451,19.765061333 -452,20.005162847 -453,19.757921307 -454,20.237635434 -455,20.31129502 -456,20.149187526 -457,20.413330424 -458,20.157038319 -459,20.360130697 -460,20.434425328 -461,20.420543853 -462,16.054128791 -463,20.270353233 -464,20.720230307 -465,20.755472527 -466,20.442446491 -467,21.142439627 -468,20.55118376 -469,20.729758839 -470,20.770440613 -471,20.867173747 -472,21.012146976 -473,21.143066767 -474,19.871434676 -475,20.98680626 -476,21.070800301 -477,21.389798069 -478,21.1654169 -479,21.494387617 -480,21.036759754 -481,21.112269574 -482,21.531892915 -483,21.337126066 -484,21.250978546 -485,20.509124166 -486,21.745883133 -487,21.103062782 -488,21.937275097 -489,21.920068816 -490,22.346367017 -491,21.292977791 -492,21.92061976 -493,22.092450352 -494,22.584700292 -495,21.458868392 -496,22.052234616 -497,22.454638002 -498,22.127435985 -499,22.522374324 -500,21.883213828 -501,21.982803364 -502,22.33556325 -503,22.415271362 -504,22.661676083 -505,22.806925196 -506,22.707423954 -507,22.340561781 -508,22.672490776 -509,22.631177416 -510,22.895915001 -511,22.713789195 -512,22.750381261 -513,22.957853464 -514,18.375919838 -515,22.860031114 -516,22.99797962 -517,23.299321407 -518,23.372230714 -519,23.269856979 -520,23.502336018 -521,23.20181453 -522,23.49991449 -523,23.64418039 -524,23.704222722 -525,23.769078018 -526,23.767130476 -527,23.738705835 -528,23.965922729 -529,23.869472989 -530,24.05009783 -531,23.824101502 -532,24.05573653 -533,23.989579074 -534,22.357893156 -535,23.18549222 -536,24.480278265 -537,24.026677194 -538,24.22988682 -539,24.386426265 -540,24.473809548 -541,24.674271787 -542,24.543278449 -543,24.700960874 -544,24.969447494 -545,24.333870883 -546,24.668663243 -547,24.666292367 -548,24.939866772 -549,24.946053397 -550,24.753292338 -551,24.820927632 -552,25.192553635 -553,25.075685951 -554,25.206709839 -555,25.248519888 -556,25.188306372 -557,25.192406035 -558,25.330387328 -559,25.471257659 -560,25.378147555 -561,25.726585109 -562,25.424377618 -563,25.790521544 -564,25.619911227 -565,25.536843057 -566,25.903676717 -567,25.87893036 -568,25.648297482 -569,25.87120104 -570,25.933177205 -571,26.192813251 -572,25.90081028 -573,26.214134206 -574,26.141011842 -575,26.246625585 -576,26.09809675 -577,26.340708263 -578,26.512834545 -579,26.240917631 -580,26.438277176 -581,26.764700643 -582,26.54277495 -583,26.498550859 -584,26.558657017 -585,26.815180342 -586,27.024340032 -587,26.431015433 -588,26.937928339 -589,26.791352087 -590,27.714522501 -591,26.201116306 -592,26.436610151 -593,27.102208878 -594,26.19530116 -595,27.118518584 -596,27.388642723 -597,27.480835235 -598,27.253742123 -599,27.324889889 -600,27.635807636 -601,27.254484846 -602,27.752763426 -603,27.48113274 -604,27.416303825 -605,27.408340749 -606,27.697793493 -607,27.796882583 -608,27.964799934 -609,27.823316641 -610,27.898344034 -611,28.209644515 -612,27.908292479 -613,28.218555957 -614,27.880263563 -615,27.867922688 -616,27.810873363 -617,28.684313227 -618,28.004002305 -619,28.350767599 -620,28.548666574 -621,28.256627827 -622,28.609898549 -623,28.448800978 -624,28.412905517 -625,28.763264369 -626,28.800030098 -627,28.918824589 -628,28.682836165 -629,28.970455229 -630,28.994155213 -631,28.920960888 -632,29.113591282 -633,29.261662013 -634,28.950615635 -635,29.243960904 -636,29.294680653 -637,29.33256142 -638,29.2900358 -639,29.697713659 -640,29.126868285 -641,29.734468814 -642,29.372990843 -643,29.594829808 -644,29.885468063 -645,29.587723895 -646,29.766930204 -647,29.828231551000002 -648,29.64953129 -649,29.966260729 -650,30.177644134 -651,29.972401615 -652,30.091857544 -653,30.093859102 -654,30.093775194 -655,30.206717448 -656,28.681422529 -657,30.219944058 -658,30.281830661 -659,30.666410314 -660,30.319944612 -661,30.532342413 -662,30.65329485 -663,30.646258714 -664,30.372179529 -665,30.937523534 -666,30.607411245 -667,30.845437066 -668,30.874295987 -669,30.786269839 -670,31.10182997 -671,31.055038211 -672,30.950370529 -673,30.997416668 -674,31.202138494 -675,31.252602837 -676,32.094032061 -677,30.515181124 -678,31.443896745 -679,31.451071023 -680,31.244367746 -681,31.522649979 -682,31.682840281 -683,31.368488661 -684,31.763013361 -685,31.608248307 -686,32.006498258 -687,31.703430811 -688,31.690898548 -689,32.036667949 -690,32.075787202 -691,31.783247357 -692,32.066928141 -693,32.236504458 -694,32.15537187 -695,32.208397621 -696,32.066546288 -697,32.536006169 -698,31.82230569 -699,32.434571521 -700,32.485642805 -701,32.698211289 -702,32.02567053 -703,32.613965368 -704,32.485869517 -705,32.828500744 -706,32.577307339 -707,32.333139515 -708,32.888567315 -709,32.614495432 -710,33.059036775 -711,33.216463344 -712,32.996813024 -713,32.954969642 -714,33.368942064 -715,33.146787957 -716,33.312489754 -717,33.029403738 -718,33.375930208 -719,33.487482005 -720,33.536680637 -721,33.307924586 -722,33.143614974 -723,33.52987231 -724,33.220851826 -725,33.656253523 -726,33.818601528 -727,33.589326889 -728,33.829996074 -729,33.895954039 -730,33.943975879 -731,34.038779124 -732,33.877155671 -733,34.253951991 -734,33.841181341 -735,34.021483843 -736,34.275737255 -737,34.160015773 -738,34.035368635 -739,34.280323753 -740,34.43637143 -741,34.407867884 -742,34.551317412 -743,34.666116549 -744,34.618400609 -745,34.675952834 -746,34.866508502 -747,34.632815747 -748,34.775578238 -749,35.50506069 -750,34.443795652 -751,34.89670591 -752,34.924520112 -753,35.180212727 -754,35.215338369 -755,34.987348077 -756,35.608344882 -757,33.874212088 -758,34.044744082 -759,35.574077401 -760,35.150056506 -761,35.72634878 -762,35.45079798 -763,35.865771632 -764,35.427604226 -765,35.667238458 -766,35.898466903 -767,35.804097476 -768,35.834170134 -769,35.896964583 -770,36.082061928 -771,35.836167064 -772,36.047323096 -773,36.202295966 -774,35.99166311 -775,36.412145704 -776,36.283841013 -777,35.983344695 -778,36.108361798 -779,36.508695673 -780,35.926579826 -781,36.500876559 -782,36.622011676 -783,36.550970389 -784,36.683771644 -785,36.872314682 -786,36.540312097 -787,36.783760094 -788,36.91872591 -789,37.096798082 -790,36.685051202 -791,37.20530758 -792,37.000189263 -793,37.099625874 -794,37.012226699 -795,37.448803223 -796,37.300768721 -797,37.223420776 -798,37.29162253 -799,37.435941317 -800,37.208434043 -801,37.2499197 -802,37.66582782 -803,37.654677445 -804,37.794832342 -805,37.454374325 -806,37.862383292 -807,37.93023652 -808,37.754437049 -809,38.038564907 -810,37.852294342 -811,37.99269785 -812,38.071864254 -813,38.17196806 -814,37.898905795 -815,38.407297019 -816,38.326513593 -817,38.24558466 -818,38.279104566 -819,38.409669597 -820,38.376204833 -821,38.128068756 -822,38.619899762 -823,37.983904743 -824,38.628779013 -825,38.911680247 -826,38.679771917 -827,38.714261932 -828,39.115790419 -829,38.641115 -830,38.937191293 -831,39.307233431 -832,38.824771817 -833,39.291982239 -834,38.68330027 -835,39.322497359 -836,39.090506468 -837,39.515033154 -838,39.253855315 -839,39.459788169 -840,38.920449303 -841,39.775861892 -842,39.686079355 -843,39.238609607 -844,39.723426937 -845,39.504629846 -846,39.289219859 -847,39.762318544 -848,38.778510905 -849,39.725515759 -850,39.987143133000004 -851,40.091022857 -852,40.349603916 -853,39.779709219 -854,40.195892127 -855,39.752316977 -856,40.311265409 -857,40.300417486 -858,40.315248098 -859,40.434328954 -860,40.19435715 -861,40.363335057 -862,41.669585049 -863,39.781274404 -864,40.354705579 -865,40.84619771 -866,40.728768619 -867,40.813227822 -868,40.964212391 -869,40.800967426 -870,41.000432604 -871,41.086572241 -872,40.821199926 -873,41.088048355 -874,41.370710885 -875,41.05639203 -876,41.416772172 -877,40.804547158 -878,41.370783479 -879,41.627574969 -880,40.853632539 -881,41.35864723 -882,41.645450331 -883,41.540242523 -884,41.794797105 -885,41.616038219 -886,41.920267929 -887,41.544109063 -888,42.411710046 -889,41.264691296 -890,41.743791518 -891,42.015610323 -892,41.848968881 -893,41.805073703 -894,42.027773351 -895,42.210190339 -896,42.076725874 -897,42.150341443 -898,42.315650886 -899,42.307507141 -900,42.217072285 -901,42.617782652 -902,42.241970775 -903,42.637235072 -904,42.493406701 -905,42.575583091 -906,42.736866699 -907,43.002907711 -908,42.634991819 -909,43.082808944 -910,42.908847193 -911,42.823499198 -912,43.132879978 -913,43.108164235 -914,43.058552019 -915,43.393716098 -916,42.691245144 -917,58.73102609 -918,28.066846227 -919,43.057087543 -920,43.147919358 -921,43.46887098 -922,43.663000603 -923,43.123878417 -924,43.766435259 -925,43.634602739 -926,43.516437928 -927,43.824348339 -928,43.447019554 -929,42.414165678 -930,43.808199767 -931,44.086641079 -932,44.070733436 -933,44.211348569 -934,43.920230992 -935,44.215260913 -936,44.179635296 -937,44.389108829 -938,44.214206702 -939,44.529518909 -940,44.387881114 -941,44.432899966 -942,44.307549914 -943,44.337567962 -944,44.641681413 -945,43.979862547 -946,44.914763536 -947,44.677172145 -948,44.909389865 -949,44.910000713 -950,44.719413381 -951,45.01212029 -952,44.898786641 -953,45.142755969 -954,44.859578864 -955,45.169217004 -956,44.870462995 -957,45.098422865 -958,45.468057985 -959,45.339417667 -960,45.277504668 -961,45.616167171 -962,45.406287131 -963,45.438691896 -964,45.801433717 -965,45.221075323 -966,45.770892791 -967,45.507180707 -968,45.670349562 -969,45.655492147 -970,45.881600854 -971,45.934390478 -972,45.924218752 -973,45.530736568 -974,46.092620072 -975,46.180485415 -976,46.246379199 -977,46.380686715 -978,46.202818149 -979,46.343931401 -980,46.355746266 -981,46.601280513 -982,46.462298547 -983,46.453345524 -984,46.581686759 -985,46.847757213 -986,46.617841844 -987,46.871250103 -988,46.653667552 -989,46.859011274 -990,46.951427338 -991,46.879632876 -992,47.099152468 -993,46.691930454 -994,47.226925086 -995,47.06015949 -996,47.288046477 -997,47.105833116 -998,47.400701356 -999,47.319450545 -1000,47.584592417 +1,0.183770058 +2,0.18429907 +3,0.178736902 +4,0.180433554 +5,0.177322556 +6,0.090063808 +7,0.175909753 +8,0.178956305 +9,0.189403783 +10,0.176246129 +11,0.181074554 +12,0.180951497 +13,0.180374672 +14,0.177966033 +15,0.179857183 +16,0.178908658 +17,0.182294792 +18,0.198625032 +19,0.179090509 +20,0.179950934 +21,0.179489473 +22,0.197288228 +23,0.178357786 +24,0.184475082 +25,0.176978907 +26,0.211447994 +27,0.182978593 +28,0.184059858 +29,0.182087235 +30,0.192683588 +31,0.182706682 +32,0.18491064 +33,0.20970519 +34,0.194538821 +35,0.180045121 +36,0.188880829 +37,0.181044941 +38,0.181163883 +39,0.178762605 +40,0.183043302 +41,0.184659019 +42,0.187713389 +43,0.198593217 +44,0.183731307 +45,0.187789955 +46,0.182765428 +47,0.181484792 +48,0.177835574 +49,0.192442342 +50,0.1768032 +51,0.203578284 +52,0.178314604 +53,0.44811865 +54,0.178177599 +55,0.181361311 +56,0.177863745 +57,0.179436398 +58,0.184569784 +59,0.176842935 +60,0.181604204 +61,0.225744823 +62,0.181101715 +63,0.185278256 +64,0.188792799 +65,0.179167259 +66,0.185886424 +67,0.174780247 +68,0.182264505 +69,0.176226937 +70,0.177624865 +71,0.18224057 +72,0.179335823 +73,0.212672217 +74,0.180874313 +75,0.182360116 +76,0.194619678 +77,0.179346502 +78,0.182365816 +79,0.271485869 +80,0.180630316 +81,0.181836164 +82,0.179892934 +83,0.179345656 +84,0.187249136 +85,0.179015014 +86,0.206643492 +87,0.18084779 +88,0.194449211 +89,0.176229375 +90,0.179891499 +91,1.194842614 +92,0.177898301 +93,0.186332968 +94,0.177075845 +95,0.175888548 +96,0.192179174 +97,0.179172159 +98,0.191667703 +99,0.184845546 +100,0.204597828 +101,0.18752747 +102,0.175367017 +103,0.181699519 +104,0.178046071 +105,0.176990063 +106,0.191471644 +107,0.18058151 +108,0.174076106 +109,0.180111662 +110,0.17614972 +111,0.180956794 +112,0.426130266 +113,0.186004452 +114,0.177351931 +115,0.181760195 +116,0.288788121 +117,0.178373818 +118,0.178494975 +119,0.18191559 +120,0.184394469 +121,0.185252774 +122,0.179272875 +123,0.182633097 +124,0.179136564 +125,0.183941894 +126,0.180805744 +127,0.183719037 +128,0.179118388 +129,0.183278708 +130,0.180359353 +131,0.181369745 +132,0.193476351 +133,0.179700993 +134,0.180364557 +135,0.184179548 +136,0.198410358 +137,0.179332014 +138,0.180058077 +139,0.18221053 +140,0.178708189 +141,0.177179652 +142,0.178842154 +143,0.125156646 +144,0.181416464 +145,0.183768244 +146,0.195882516 +147,0.181738179 +148,0.19903695 +149,0.210426442 +150,0.180454563 +151,0.176898448 +152,0.182937556 +153,0.182780117 +154,0.181912447 +155,0.177414153 +156,0.182697437 +157,0.189333692 +158,0.187863269 +159,0.471118221 +160,0.18593903 +161,0.180757169 +162,0.466944193 +163,0.184438009 +164,0.180484542 +165,0.183409141 +166,0.185515818 +167,0.182578911 +168,0.189699851 +169,0.189119797 +170,0.242267557 +171,0.186330216 +172,0.184248177 +173,0.180865213 +174,0.186596313 +175,0.193360083 +176,0.179512294 +177,0.18596241 +178,0.178454238 +179,0.465893282 +180,0.464241881 +181,0.470321876 +182,0.190036996 +183,0.17639207 +184,0.189172159 +185,0.465930908 +186,0.464091697 +187,0.468945234 +188,0.468404382 +189,0.473502736 +190,0.473774253 +191,0.209884242 +192,0.468894257 +193,0.190588454 +194,0.474979417 +195,0.467635581 +196,0.467699659 +197,0.476005352 +198,0.475211081 +199,0.468860655 +200,0.47149591 +201,0.469168913 +202,0.21059177 +203,0.476620441 +204,0.471947432 +205,0.477835389 +206,0.47436291 +207,0.471169731 +208,0.198457161 +209,0.48620524 +210,0.474260229 +211,0.218698902 +212,0.830638331 +213,0.47138435 +214,0.472473686 +215,0.472546448 +216,0.199750385 +217,0.47021389 +218,0.194569281 +219,0.486712621 +220,0.469907998 +221,0.475773961 +222,0.484381486 +223,0.46821363 +224,0.49928308 +225,0.474234041 +226,0.471839693 +227,0.468574136 +228,0.197661701 +229,0.475689495 +230,0.48219455 +231,0.501346743 +232,0.47045589 +233,0.473609708 +234,0.471296657 +235,0.487126591 +236,0.472763233 +237,0.474166615 +238,0.72611853 +239,0.489723628 +240,0.494815103 +241,0.47187761 +242,0.472262772 +243,0.477338438 +244,0.473223501 +245,0.471734002 +246,0.467174292 +247,0.476279277 +248,0.470249187 +249,0.482293445 +250,0.490747818 +251,0.478913995 +252,0.480136956 +253,0.472484572 +254,0.472148765 +255,0.345653486 +256,0.494509997 +257,0.339933164 +258,0.48406298 +259,0.469659685 +260,0.476046154 +261,0.470891522 +262,0.476668356 +263,0.468649012 +264,0.493196502 +265,0.472055343 +266,0.476762809 +267,0.47226666 +268,0.483063546 +269,0.472297613 +270,0.472210251 +271,0.488122225 +272,0.468923918 +273,0.471427414 +274,0.48977265 +275,0.471201224 +276,0.092325009 +277,0.486119333 +278,0.475672466 +279,0.470639506 +280,0.485967927 +281,0.471184557 +282,0.469717941 +283,0.468930911 +284,0.473363591 +285,0.476509392 +286,0.468165222 +287,0.509074027 +288,0.468996532 +289,0.474162251 +290,0.534925193 +291,0.480948203 +292,0.479702191 +293,0.473037377 +294,0.470110172 +295,0.506183066 +296,0.475708514 +297,0.476291119 +298,0.47058561 +299,0.474979697 +300,0.475671705 +301,0.488072421 +302,0.483036483 +303,0.468863293 +304,0.471166383 +305,0.477101686 +306,0.502081873 +307,0.476863819 +308,0.479603082 +309,0.469318755 +310,0.473874661 +311,0.475366832 +312,0.465899539 +313,0.480672977 +314,0.471500091 +315,0.475066492 +316,0.471466077 +317,0.472645513 +318,0.475183189 +319,0.476962197 +320,0.472215522 +321,0.581190978 +322,0.488939467 +323,0.481193026 +324,0.510487132 +325,0.519747591 +326,0.495105277 +327,0.473513214 +328,0.498411932 +329,0.487108317 +330,0.484046634 +331,0.485293085 +332,0.480545664 +333,0.517951589 +334,0.474444899 +335,0.478234279 +336,0.46963921 +337,0.476941825 +338,0.480065863 +339,0.475829728 +340,0.484793375 +341,0.47155622 +342,0.475354677 +343,0.491472528 +344,0.483965544 +345,0.478022786 +346,0.473892682 +347,0.487256621 +348,0.479628978 +349,0.47180862 +350,0.49524842 +351,0.470490363 +352,0.488830359 +353,0.480086673 +354,0.473797611 +355,0.476955373 +356,0.471465906 +357,0.479236384 +358,0.510472313 +359,0.470677687 +360,0.476600895 +361,0.469861228 +362,0.476407144 +363,0.49241007 +364,0.473646536 +365,0.475155545 +366,0.476798771 +367,0.475408101 +368,0.472585572 +369,0.482416042 +370,0.46883109 +371,0.494367748 +372,0.475244848 +373,0.470956891 +374,0.4703609 +375,0.472817748 +376,0.486990158 +377,0.503410382 +378,0.477179474 +379,0.493005675 +380,0.475540101 +381,0.477651468 +382,0.478888344 +383,0.506889379 +384,0.476047143 +385,0.517152798 +386,0.512169118 +387,0.475271708 +388,0.468129298 +389,0.485807918 +390,0.482354418 +391,0.481768348 +392,0.463292979 +393,0.408841525 +394,0.473647688 +395,0.487770625 +396,0.485636062 +397,0.498086924 +398,0.470682306 +399,0.476036111 +400,0.484356388 +401,0.484359993 +402,0.479592229 +403,0.476059899 +404,0.481162471 +405,0.489544115 +406,0.473825665 +407,0.482159864 +408,0.485391437 +409,0.48323204 +410,0.491821032 +411,0.492568874 +412,0.468978429 +413,0.473059708 +414,0.485682896 +415,0.462817266 +416,0.494126943 +417,0.462074599 +418,0.472601644 +419,0.472249525 +420,0.469431008 +421,0.474623598 +422,0.476649359 +423,0.475621533 +424,0.4719869 +425,0.471288268 +426,0.477399435 +427,0.474086057 +428,0.473185507 +429,0.475803129 +430,0.470998258 +431,0.489681685 +432,0.467929372 +433,0.489098529 +434,0.46529457 +435,0.465367344 +436,0.466871192 +437,0.469314915 +438,0.472082599 +439,0.472622712 +440,0.470537616 +441,0.467365208 +442,0.466685508 +443,0.467543041 +444,0.472964289 +445,0.48447569 +446,0.483875492 +447,0.475921228 +448,0.481545183 +449,0.471672454 +450,0.470198438 +451,0.471236042 +452,0.48181349 +453,0.467395593 +454,0.484226663 +455,0.487993627 +456,0.464474091 +457,0.495411515 +458,0.496861478 +459,0.4810659 +460,0.472201599 +461,0.492326587 +462,0.466690417 +463,0.468187689 +464,0.474286435 +465,0.466754391 +466,0.466173836 +467,0.472495734 +468,0.470415078 +469,0.476273929 +470,0.463294271 +471,0.47672312 +472,0.469957408 +473,0.475469406 +474,0.469103828 +475,0.473224807 +476,0.466264091 +477,0.471383594 +478,0.485790597 +479,0.487395785 +480,0.47593698 +481,0.491897792 +482,0.470885741 +483,0.47134244 +484,0.485798405 +485,1.474793129 +486,0.471686871 +487,0.473868293 +488,0.473177443 +489,0.467201138 +490,0.47530534 +491,0.479273643 +492,0.475383299 +493,0.486459043 +494,0.468763839 +495,0.467466012 +496,0.472056218 +497,0.487678035 +498,0.469569441 +499,0.512808169 +500,0.520553015 +501,0.469988417 +502,0.474803326 +503,0.491277045 +504,0.46658729 +505,0.467936994 +506,0.520247822 +507,0.475796071 +508,0.482666357 +509,0.465587631 +510,0.472186374 +511,0.477225747 +512,0.468411633 +513,0.473022351 +514,0.487660535 +515,0.469290806 +516,0.470542473 +517,0.4763971 +518,0.490468513 +519,0.472929121 +520,0.484533727 +521,0.470416908 +522,0.475434593 +523,0.472443635 +524,0.484810354 +525,0.479831425 +526,0.482499489 +527,0.470549286 +528,0.476805245 +529,0.488671038 +530,0.468885688 +531,0.463059782 +532,0.462853161 +533,0.470654042 +534,0.468437879 +535,0.48339108 +536,0.477773133 +537,0.473516806 +538,0.476597862 +539,0.479137758 +540,0.468044398 +541,0.473187344 +542,0.472617662 +543,0.489670111 +544,0.472825663 +545,0.486331871 +546,0.468051537 +547,0.483915865 +548,0.476052924 +549,0.484619546 +550,0.196848633 +551,0.474209494 +552,0.469995287 +553,0.469202014 +554,0.484869483 +555,0.473355175 +556,0.474134137 +557,0.495373887 +558,0.499085141 +559,0.469350123 +560,0.487863072 +561,1.474208904 +562,0.474554179 +563,0.481832766 +564,0.469831081 +565,0.470934879 +566,0.479276886 +567,0.476833076 +568,0.506298609 +569,0.474485396 +570,0.483499039 +571,0.470914953 +572,0.469677951 +573,0.485250325 +574,0.47432712 +575,0.472372186 +576,0.482501028 +577,0.488468951 +578,0.469502737 +579,0.476468062 +580,0.473926002 +581,0.491594623 +582,0.473710609 +583,0.478420116 +584,0.474256627 +585,0.472877644 +586,0.472583074 +587,0.480719649 +588,0.507773844 +589,0.469649158 +590,0.469805887 +591,0.46976167 +592,0.475687126 +593,0.514680346 +594,0.471862002 +595,0.471916681 +596,0.493914641 +597,1.488507213 +598,0.507459725 +599,0.480859864 +600,0.497933427 +601,0.481353817 +602,0.469949714 +603,0.488677739 +604,0.478222497 +605,0.480090016 +606,0.464621161 +607,0.471612789 +608,0.489535373 +609,0.471229233 +610,0.470118501 +611,0.466944749 +612,0.479090829 +613,0.497047914 +614,0.464191853 +615,0.471162275 +616,0.461689925 +617,0.472520418 +618,0.482494758 +619,0.474008897 +620,0.486196317 +621,0.467398442 +622,0.474630889 +623,0.468924157 +624,0.186834822 +625,0.472363354 +626,0.472295304 +627,0.470900007 +628,0.471051803 +629,0.477506772 +630,0.482262764 +631,0.479972009 +632,0.472454142 +633,0.473871403 +634,0.473868801 +635,0.472920057 +636,0.486751277 +637,0.482527281 +638,0.471762946 +639,0.475475412 +640,0.492793913 +641,0.477189958 +642,0.467541313 +643,0.472150281 +644,0.471834105 +645,0.565850608 +646,0.480453238 +647,0.469209886 +648,0.474117678 +649,0.49658828 +650,0.46595657 +651,0.471165084 +652,0.474113366 +653,0.478148819 +654,1.093446763 +655,0.479909133 +656,0.471755871 +657,0.496537487 +658,0.474987171 +659,0.47160141 +660,0.474254075 +661,0.476356702 +662,0.482284482 +663,0.482926788 +664,0.470492598 +665,0.476846315 +666,0.46990197 +667,0.483766469 +668,0.476381347 +669,0.46804935 +670,0.468006641 +671,0.4850487 +672,0.471921813 +673,0.475951769 +674,0.508172326 +675,0.472312118 +676,0.487001478 +677,0.472216087 +678,0.482386042 +679,0.481470371 +680,0.46972223 +681,0.485054997 +682,0.472763802 +683,0.471674776 +684,0.488552584 +685,0.473568933 +686,0.471931374 +687,0.472779715 +688,0.470148934 +689,0.487639081 +690,0.468393289 +691,0.472984952 +692,0.474233785 +693,0.468930892 +694,0.487346914 +695,0.467639529 +696,0.487115164 +697,0.470380127 +698,0.484371811 +699,0.470957789 +700,0.474464619 +701,0.471709622 +702,0.466106574 +703,0.470518598 +704,0.471323158 +705,0.478118153 +706,0.468370575 +707,0.465150191 +708,0.472525914 +709,0.476327033 +710,0.467145328 +711,0.470092795 +712,0.490756347 +713,0.470226373 +714,0.485797795 +715,0.475992281 +716,0.491564721 +717,0.477205399 +718,0.73746484 +719,0.466220309 +720,0.473137091 +721,0.472343844 +722,0.471364582 +723,0.478149986 +724,0.497342051 +725,0.470753418 +726,0.470716139 +727,0.470284608 +728,0.504443764 +729,0.484902667 +730,0.471326043 +731,0.490016333 +732,0.472725831 +733,0.471951291 +734,0.471347572 +735,0.476967035 +736,0.483500058 +737,0.512473511 +738,0.467036499 +739,0.48361215 +740,0.471088344 +741,0.475159413 +742,0.472732304 +743,0.509121708 +744,0.472268625 +745,0.506463749 +746,0.499917884 +747,0.479893649 +748,0.479159112 +749,0.470664817 +750,0.473797824 +751,0.476359451 +752,0.476557348 +753,0.495627527 +754,0.464583555 +755,0.480875142 +756,0.471250715 +757,0.471736576 +758,0.494072185 +759,0.475249541 +760,0.469375822 +761,0.473128984 +762,0.48791302 +763,0.48581588 +764,0.472564726 +765,0.484115445 +766,0.475368762 +767,0.475303202 +768,0.485852596 +769,0.473385934 +770,0.490044786 +771,0.467983054 +772,0.465957186 +773,0.47564658 +774,0.471628509 +775,0.472867691 +776,0.462520631 +777,0.473119722 +778,0.482032883 +779,1.188559848 +780,0.489263125 +781,0.474836255 +782,0.470863436 +783,0.495544884 +784,0.471128459 +785,0.475897087 +786,0.470209736 +787,0.475256127 +788,0.674356313 +789,0.478907332 +790,0.474295668 +791,0.47970478 +792,0.467901631 +793,0.502259622 +794,0.473457698 +795,0.473994004 +796,0.481206793 +797,0.476811329 +798,0.473401659 +799,0.495433422 +800,0.480902656 +801,0.482541849 +802,0.475180759 +803,0.675421176 +804,0.495622251 +805,0.479811207 +806,0.473915942 +807,0.476771722 +808,0.493747714 +809,0.497043309 +810,0.673142306 +811,0.497053242 +812,0.483508398 +813,0.681487892 +814,0.672837243 +815,0.674041089 +816,0.672761539 +817,0.677884671 +818,0.674907412 +819,0.474213955 +820,0.493656901 +821,0.672465919 +822,0.668615583 +823,0.673762932 +824,0.480351014 +825,0.480760332 +826,0.675104883 +827,0.480798165 +828,0.495165996 +829,0.670400716 +830,0.491896577 +831,0.684217149 +832,0.675343916 +833,1.100448081 +834,0.479805816 +835,0.689898057 +836,0.489638776 +837,0.481199215 +838,0.477693631 +839,0.668876228 +840,0.698672203 +841,0.672901909 +842,0.482116962 +843,0.48921656 +844,0.678361948 +845,0.675679229 +846,0.4949783 +847,0.673540548 +848,0.678371244 +849,0.671931562 +850,0.6711929 +851,0.49340463 +852,0.684861749 +853,0.675129385 +854,0.483924848 +855,0.673345442 +856,0.689221209 +857,0.6925151 +858,0.677184288 +859,0.672182086 +860,0.506835074 +861,0.674339251 +862,0.678290868 +863,0.672732123 +864,0.673128759 +865,0.67084837 +866,0.672786884 +867,0.680949048 +868,0.669079279 +869,0.508846498 +870,0.512277397 +871,0.667494379 +872,0.67674981 +873,0.668781813 +874,0.669352151 +875,0.684285396 +876,0.670474987 +877,0.704573504 +878,0.672240192 +879,0.678228187 +880,0.680226278 +881,0.674305233 +882,0.676853209 +883,0.664633834 +884,0.671824609 +885,1.685718051 +886,0.671390377 +887,0.678097852 +888,0.673249522 +889,0.676656399 +890,0.66995941 +891,0.676406324 +892,0.682236666 +893,0.687064489 +894,0.672644348 +895,0.673288723 +896,0.691228138 +897,0.67286248 +898,0.681584491 +899,0.681150645 +900,0.697236659 +901,0.67692937 +902,0.672503988 +903,0.671383562 +904,0.677842121 +905,0.676404631 +906,0.691059764 +907,0.675199689 +908,0.67255197 +909,0.675500888 +910,0.673856847 +911,0.672460864 +912,0.672431091 +913,0.670957384 +914,0.682004091 +915,0.672490155 +916,0.680062049 +917,0.676188122 +918,0.677329317 +919,0.551398757 +920,0.675656749 +921,0.682646249 +922,0.677603707 +923,0.691432477 +924,0.674729586 +925,1.479589265 +926,0.665774661 +927,0.669698137 +928,0.676253656 +929,0.673367343 +930,0.667783261 +931,0.670910771 +932,0.670879652 +933,0.679421827 +934,0.685933975 +935,0.683200613 +936,0.675710847 +937,0.668899389 +938,0.674235109 +939,0.673632912 +940,0.679565629 +941,0.697697849 +942,0.706985685 +943,0.714076848 +944,0.674130404 +945,1.016263271 +946,0.670502223 +947,0.697924414 +948,0.674235526 +949,0.673913587 +950,0.690245135 +951,0.669467218 +952,0.675796111 +953,0.676252122 +954,0.678198658 +955,0.694674744 +956,0.673758545 +957,0.687541361 +958,0.675239745 +959,0.67522024 +960,0.677388789 +961,0.670913077 +962,0.666790404 +963,0.674811912 +964,0.66705688 +965,0.679296209 +966,0.674312337 +967,0.672205621 +968,0.675865606 +969,0.673969669 +970,0.684236005 +971,0.668477507 +972,0.681371389 +973,0.675618987 +974,0.677045634 +975,0.676050496 +976,0.676999331 +977,0.675383188 +978,0.674749853 +979,0.672004967 +980,0.670180711 +981,0.675741514 +982,0.676463165 +983,0.67609914 +984,0.671741006 +985,0.669241966 +986,0.68851294 +987,0.676301796 +988,0.674206791 +989,0.687658983 +990,0.670698605 +991,0.676956668 +992,0.676174668 +993,0.673980813 +994,0.673059575 +995,0.680813504 +996,0.676427282 +997,0.668824636 +998,0.672318931 +999,0.674823501 +1000,0.676305237 +Test Start,Test End,Test End + 10s,Duration +1697500817,1697503831,1697503841,50m13.906476296s diff --git a/tests/scale/results/1.0.0/TestScale_HTTPSListeners/CPU.png b/tests/scale/results/1.0.0/TestScale_HTTPSListeners/CPU.png index 55de292fa7..15ce148d78 100644 Binary files a/tests/scale/results/1.0.0/TestScale_HTTPSListeners/CPU.png and b/tests/scale/results/1.0.0/TestScale_HTTPSListeners/CPU.png differ diff --git a/tests/scale/results/1.0.0/TestScale_HTTPSListeners/Memory.png b/tests/scale/results/1.0.0/TestScale_HTTPSListeners/Memory.png index 3ddf1f1601..954b623230 100644 Binary files a/tests/scale/results/1.0.0/TestScale_HTTPSListeners/Memory.png and b/tests/scale/results/1.0.0/TestScale_HTTPSListeners/Memory.png differ diff --git a/tests/scale/results/1.0.0/TestScale_HTTPSListeners/TTR.png b/tests/scale/results/1.0.0/TestScale_HTTPSListeners/TTR.png index 14e9d920c6..0b77fa24b2 100644 Binary files a/tests/scale/results/1.0.0/TestScale_HTTPSListeners/TTR.png and b/tests/scale/results/1.0.0/TestScale_HTTPSListeners/TTR.png differ diff --git a/tests/scale/results/1.0.0/TestScale_HTTPSListeners/results.csv b/tests/scale/results/1.0.0/TestScale_HTTPSListeners/results.csv index 1c30ee62f9..3ae575fb22 100644 --- a/tests/scale/results/1.0.0/TestScale_HTTPSListeners/results.csv +++ b/tests/scale/results/1.0.0/TestScale_HTTPSListeners/results.csv @@ -1,65 +1,67 @@ # HTTPS Listeners,Time to Ready (s),Error -1,0.169995759 -2,0.316317206 -3,0.335810125 -4,0.35087607 -5,0.374128886 -6,0.786490003 -7,0.395602107 -8,0.335063415 -9,0.351511284 -10,0.306195938 -11,1.079579494 -12,1.085989951 -13,0.710736746 -14,0.706770998 -15,0.791338601 -16,0.910576504 -17,0.72462806 -18,0.891562537 -19,0.699489578 -20,0.989158916 -21,0.992727987 -22,1.080868064 -23,0.793088182 -24,0.691032678 -25,0.926785913 -26,1.308280347 -27,0.878097664 -28,1.29426 -29,1.306243568 -30,2.749318703 -31,1.091964274 -32,1.412320275 -33,1.581276201 -34,1.9931428759999998 -35,1.205374759 -36,1.723513203 -37,1.720674308 -38,1.55956355 -39,2.236604075 -40,1.67136963 -41,2.272512461 -42,1.944905419 -43,2.689681995 -44,1.462145122 -45,1.891378043 -46,2.483272037 -47,1.632890784 -48,1.225230643 -49,1.417660855 -50,0.330152469 -51,1.357993629 -52,1.9156297869999999 -53,1.4871654460000001 -54,0.612916622 -55,2.981474032 -56,10.003182785 -57,2.873364642 -58,2.174200222 -59,2.501841692 -60,2.342435315 -61,1.335872465 -62,2.8931696909999998 -63,3.182950872 -64,6.230650774 +1,0.275460118 +2,0.288850428 +3,0.298762958 +4,0.275357124 +5,0.276795779 +6,0.277205719 +7,0.284058152 +8,0.269822734 +9,0.27229328 +10,0.273108774 +11,0.147417568 +12,0.283057992 +13,0.281158353 +14,0.273877068 +15,0.286165723 +16,0.271805322 +17,0.615278829 +18,0.611982861 +19,0.282430851 +20,0.617603219 +21,0.617296454 +22,0.613296831 +23,0.610197359 +24,0.61913954 +25,0.610887585 +26,1.639631742 +27,0.627326815 +28,0.64769405 +29,0.613625447 +30,0.646178701 +31,0.613924874 +32,0.616708166 +33,0.277891253 +34,0.633328001 +35,0.613160153 +36,0.620836128 +37,0.558986376 +38,0.675165246 +39,0.612616707 +40,0.623200392 +41,0.655276552 +42,0.617860902 +43,0.614938672 +44,0.824353795 +45,1.017062126 +46,0.831737853 +47,0.615700002 +48,0.832489494 +49,0.622095121 +50,0.62331981 +51,0.76123751 +52,1.072843365 +53,0.827471577 +54,0.612518245 +55,0.65196546 +56,0.813851186 +57,0.269014371 +58,0.648998161 +59,0.823982125 +60,1.03057249 +61,0.809049528 +62,0.812008331 +63,0.812412792 +64,0.624002087 +Test Start,Test End,Test End + 10s,Duration +1697513662,1697513812,1697513822,2m30.085233889s diff --git a/tests/scale/results/1.0.0/TestScale_Listeners/CPU.png b/tests/scale/results/1.0.0/TestScale_Listeners/CPU.png index b6028ec1db..f419c7b8f1 100644 Binary files a/tests/scale/results/1.0.0/TestScale_Listeners/CPU.png and b/tests/scale/results/1.0.0/TestScale_Listeners/CPU.png differ diff --git a/tests/scale/results/1.0.0/TestScale_Listeners/Memory.png b/tests/scale/results/1.0.0/TestScale_Listeners/Memory.png index 3e7800da30..68dd6abac5 100644 Binary files a/tests/scale/results/1.0.0/TestScale_Listeners/Memory.png and b/tests/scale/results/1.0.0/TestScale_Listeners/Memory.png differ diff --git a/tests/scale/results/1.0.0/TestScale_Listeners/TTR.png b/tests/scale/results/1.0.0/TestScale_Listeners/TTR.png index 67aeb8c23e..ef6e27b37f 100644 Binary files a/tests/scale/results/1.0.0/TestScale_Listeners/TTR.png and b/tests/scale/results/1.0.0/TestScale_Listeners/TTR.png differ diff --git a/tests/scale/results/1.0.0/TestScale_Listeners/results.csv b/tests/scale/results/1.0.0/TestScale_Listeners/results.csv index 89ef9e114b..c802ed2bbc 100644 --- a/tests/scale/results/1.0.0/TestScale_Listeners/results.csv +++ b/tests/scale/results/1.0.0/TestScale_Listeners/results.csv @@ -1,65 +1,67 @@ # Listeners,Time to Ready (s),Error -1,0.389434458 -2,0.385901884 -3,0.386300635 -4,0.39178591 -5,0.382599518 -6,0.381957681 -7,0.381679551 -8,0.673451873 -9,0.38174132 -10,0.394320317 -11,0.386441444 -12,1.257764007 -13,0.678272427 -14,0.674835931 -15,0.961024281 -16,0.674262046 -17,0.678830512 -18,0.385906626 -19,0.679167754 -20,0.964288262 -21,1.254985027 -22,0.674975659 -23,0.678150708 -24,0.674943489 -25,1.2649119770000001 -26,1.2595624 -27,1.8420411049999998 -28,0.964987293 -29,0.963224162 -30,1.5548014540000001 -31,0.392973186 -32,1.591366332 -33,1.5760351959999999 -34,0.67539319 -35,1.836873 -36,0.390074267 -37,0.669736812 -38,2.159392649 -39,0.394423826 -40,2.265563849 -41,0.386839254 -42,2.145638601 -43,2.146480487 -44,0.395269441 -45,2.426101109 -46,0.966531362 -47,0.67866656 -48,1.577306721 -49,0.384150846 -50,0.385452621 -51,2.152355796 -52,0.380525188 -53,0.395430389 -54,0.387830157 -55,2.742531289 -56,0.966502151 -57,0.672086839 -58,0.674754009 -59,2.74438795 -60,0.378782959 -61,2.436529727 -62,3.019968911 -63,0.380224085 -64,0.383710448 +1,0.18237684 +2,0.182742815 +3,0.195675447 +4,0.178862545 +5,0.180935635 +6,0.177669341 +7,0.18382896 +8,0.214773304 +9,0.18390344 +10,0.180049743 +11,0.191997389 +12,0.183914045 +13,0.189440271 +14,0.185146274 +15,0.182140865 +16,0.193861979 +17,0.469956374 +18,0.179860564 +19,0.185043675 +20,0.468109219 +21,0.180882773 +22,0.178957273 +23,0.508805759 +24,0.473293157 +25,0.474382089 +26,0.471838438 +27,0.472529369 +28,0.470836384 +29,0.473700369 +30,0.480004049 +31,0.471685754 +32,0.476254796 +33,0.482731123 +34,0.473120931 +35,0.480416813 +36,0.1876526 +37,0.473850293 +38,0.470654185 +39,0.484869479 +40,0.496493515 +41,0.4745361 +42,0.476442215 +43,0.666888522 +44,0.482037341 +45,0.475596972 +46,0.679201577 +47,0.47752307 +48,0.68219485 +49,0.673420162 +50,0.478945287 +51,0.467964754 +52,0.485408614 +53,0.47829759 +54,0.473425684 +55,0.690957228 +56,0.485929862 +57,0.498929569 +58,0.498465386 +59,0.893005799 +60,0.47911405 +61,0.877671069 +62,0.501556486 +63,0.473695304 +64,0.872768629 +Test Start,Test End,Test End + 10s,Duration +1697512460,1697512590,1697512600,2m10.057601184s diff --git a/tests/scale/results/1.0.0/TestScale_UpstreamServers/CPU.png b/tests/scale/results/1.0.0/TestScale_UpstreamServers/CPU.png index 92eebb022f..d0d11125b6 100644 Binary files a/tests/scale/results/1.0.0/TestScale_UpstreamServers/CPU.png and b/tests/scale/results/1.0.0/TestScale_UpstreamServers/CPU.png differ diff --git a/tests/scale/results/1.0.0/TestScale_UpstreamServers/Memory.png b/tests/scale/results/1.0.0/TestScale_UpstreamServers/Memory.png index 6d035f3ed4..197325b441 100644 Binary files a/tests/scale/results/1.0.0/TestScale_UpstreamServers/Memory.png and b/tests/scale/results/1.0.0/TestScale_UpstreamServers/Memory.png differ diff --git a/tests/scale/scale.md b/tests/scale/scale.md index 54a052ccf5..c638809fb9 100644 --- a/tests/scale/scale.md +++ b/tests/scale/scale.md @@ -51,7 +51,7 @@ are listed in the [Scale Upstream Servers](#scale-upstream-servers) test steps. - Install edge NGF and save the Pod Name and LoadBalancer IP for tests: ```console - helm install scale-test oci://ghcr.io/nginxinc/charts/nginx-gateway-fabric --create-namespace --wait -n nginx-gateway --version=0.0.0-edge + helm install scale-test oci://ghcr.io/nginxinc/charts/nginx-gateway-fabric --create-namespace --wait -n nginx-gateway --version=0.0.0-edge --set nginxGateway.config.logging.level=debug ``` ```console @@ -192,17 +192,23 @@ Total Resources Created: - 1000 HTTPRoutes - 1 Service, Deployment, Pod -This test takes around 7 hours to run, so I recommend running it on a VM, or overnight with the aid of -[caffeinate](https://www.theapplegeek.co.uk/blog/caffeinate) for MAC users. - Follow the steps below to run the test: - Run the test: ```console - go test -v -tags scale -timeout 600m -run TestScale_HTTPRoutes -i 1000 -delay 2s + go test -v -tags scale -timeout 30m -run TestScale_HTTPRoutes -i 1000 + ``` + + To test with a delay in between each new HTTPRoute, you can add the `-delay` flag to the above command. For example, + to add a 2-second delay: + + ```console + go test -v -tags scale -timeout 60m -run TestScale_HTTPRoutes -i 1000 -delay 2s ``` + The test takes longer to run with a delay so make sure to adjust the timeout value. + - [Analyze](#analyze) the results. - Clean up: @@ -350,6 +356,8 @@ Follow these steps to run the test: kubectl describe httproute route ``` +- Edit your /etc/hosts file and add an entry for "NGF_IP cafe.example.com". + - Test the first match: ```console @@ -406,6 +414,31 @@ Follow these steps to run the test: rate(nginx_gateway_fabric_nginx_reloads_milliseconds_count[<Duration>] @ <Test End + 10s>) ``` + Reload Time Distribution: + + ```console + nginx_gateway_fabric_nginx_reloads_milliseconds_bucket - nginx_gateway_fabric_nginx_reloads_milliseconds_bucket @ <Test Start> + ``` + + Total number of event batches processed: + + ```console + nginx_gateway_fabric_event_batch_processing_milliseconds_count - nginx_gateway_fabric_event_batch_processing_milliseconds_count @ <Test Start> + ``` + + Average event batch processing time (ms): + + ```console + rate(nginx_gateway_fabric_event_batch_processing_milliseconds_sum[<Duration>] @ <Test End + 10s>) / + rate(nginx_gateway_fabric_event_batch_processing_milliseconds_count[<Duration>] @ <Test End + 10s>) + ``` + + Event Batch Processing Time Distribution: + + ```console + nginx_gateway_fabric_event_batch_processing_milliseconds_bucket - nginx_gateway_fabric_event_batch_processing_milliseconds_bucket @ <Test Start> + ``` + Record these numbers in a table in the results file. - Take screenshots of memory and CPU usage in GKE Dashboard diff --git a/tests/scale/scale_test.go b/tests/scale/scale_test.go index cf18eaacd6..c3174de8f3 100644 --- a/tests/scale/scale_test.go +++ b/tests/scale/scale_test.go @@ -25,7 +25,7 @@ import ( var ( numIterations = flag.Int("i", 1, "number of times to scale the resource") delay = flag.Duration("delay", 0, "delay between each scaling iteration") - version = flag.String("version", "1.0", "version of NGF under test") + version = flag.String("version", "1.0.0", "version of NGF under test") ) func TestScale_Listeners(t *testing.T) { @@ -202,7 +202,8 @@ func kubectlWaitAllPodsReady() error { func kubectlExec(arg ...string) error { cmd := exec.Command("kubectl", arg...) - return cmd.Err + + return cmd.Run() } func waitForResponseForHost(url, host string) (time.Duration, error) {