fix modified bucket trimming errors

Makefile

@@ -190,7 +190,7 @@ service-restart-all:
 
 test: ginkgo
 	@echo "ginkgo --require-suite --poll-progress-after=10s --poll-progress-interval=20s -r $(TEST)"
-	@cd $(ROOT_DIRECTORY) && . ./env.test.sh && ginkgo --fail-fast --require-suite --poll-progress-after=10s --poll-progress-interval=20s -r $(TEST)
+	@cd $(ROOT_DIRECTORY) && . ./env.test.sh && ginkgo --require-suite --poll-progress-after=10s --poll-progress-interval=20s -r $(TEST)
 
 test-until-failure: ginkgo
 	@echo "ginkgo --require-suite --poll-progress-after=10s --poll-progress-interval=20s -r -untilItFails $(TEST)"

data/summary/summary.go

@@ -174,13 +174,13 @@ func (c *GlucoseSummarizer[T, A]) UpdateSummary(ctx context.Context, userId stri
 		userSummary.Dates.Reset()
 	}
 
-	// we currently don't just pull modified records, even if some code supports it, make a copy of status without these
+	// we currently don't only pull modified records, even if some code supports it, make a copy of status without these
+
+	userSummary.Stats.ClearInvalidatedBuckets(status)
 	dataRange := *status
 	dataRange.LastUpdated = time.Time{}
 	dataRange.NextLastUpdated = time.Now()
 
-	userSummary.Stats.ClearInvalidatedBuckets(status)
-
 	var cursor *mongo.Cursor
 	cursor, err = c.deviceData.GetDataRange(ctx, userId, types.GetDeviceDataTypeString[T, A](), &dataRange)
 	if err != nil {

data/summary/types/bgm.go

@@ -131,12 +131,18 @@ func (s *BGMStats) GetBucketDate(i int) time.Time {
 }
 
 func (s *BGMStats) ClearInvalidatedBuckets(status *UserLastUpdated) {
-	offset := int(status.EarliestModified.Sub(s.Buckets[0].Date).Hours())
+	if status.EarliestModified.After(s.Buckets[len(s.Buckets)-1].LastRecordTime) {
+		return
+	}
+
+	offset := len(s.Buckets) - (int(s.Buckets[len(s.Buckets)-1].Date.Sub(status.EarliestModified.UTC().Truncate(time.Hour)).Hours()) + 1)
 
 	for i := offset; i < len(s.Buckets); i++ {
 		s.Buckets[i] = nil
 	}
-	s.Buckets = s.Buckets[offset:]
+	s.Buckets = s.Buckets[:offset]
+
+	status.FirstData = s.Buckets[len(s.Buckets)-1].LastRecordTime
 }
 
 func (s *BGMStats) Update(ctx context.Context, cursor *mongo.Cursor) error {

data/summary/types/cgm.go

@@ -186,21 +186,25 @@ func (s *CGMStats) GetBucketDate(i int) time.Time {
 }
 
 func (s *CGMStats) ClearInvalidatedBuckets(status *UserLastUpdated) {
-	offset := int(status.EarliestModified.Sub(s.Buckets[0].Date).Hours())
+	if status.EarliestModified.After(s.Buckets[len(s.Buckets)-1].LastRecordTime) {
+		return
+	}
+
+	offset := len(s.Buckets) - (int(s.Buckets[len(s.Buckets)-1].Date.Sub(status.EarliestModified.UTC().Truncate(time.Hour)).Hours()) + 1)
 
 	for i := offset; i < len(s.Buckets); i++ {
 		s.Buckets[i] = nil
 	}
-	s.Buckets = s.Buckets[offset:]
+	s.Buckets = s.Buckets[:offset]
+
+	status.FirstData = s.Buckets[len(s.Buckets)-1].LastRecordTime
 }
 
 func (s *CGMStats) Update(ctx context.Context, cursor *mongo.Cursor) error {
 	var userData []*glucoseDatum.Glucose = nil
 	var err error
 
-	count := 0
 	for cursor.Next(ctx) {
-		count++
 		if userData == nil {
 			userData = make([]*glucoseDatum.Glucose, 0, cursor.RemainingBatchLength())
 		}

data/summary/types/summary_bgm_test.go

@@ -917,5 +917,97 @@ var _ = Describe("BGM Summary", func() {
 				Expect(*userBGMSummary.Stats.OffsetPeriods["30d"].TotalRecords).To(Equal(144))
 			})
 		})
+
+		Context("ClearInvalidatedBuckets", func() {
+			It("trims the correct buckets", func() {
+				var dataSetBGMDataCursor *mongo.Cursor
+				userBGMSummary = types.Create[*types.BGMStats](userId)
+				dataSetBGMData = NewDataSetBGMDataAvg(deviceId, datumTime, 10, inTargetBloodGlucose)
+				dataSetBGMDataCursor, err = mongo.NewCursorFromDocuments(ConvertToIntArray(dataSetBGMData), nil, nil)
+
+				err = userBGMSummary.Stats.Update(ctx, dataSetBGMDataCursor)
+				Expect(err).ToNot(HaveOccurred())
+				Expect(len(userBGMSummary.Stats.Buckets)).To(Equal(10))
+				Expect(userBGMSummary.Stats.TotalHours).To(Equal(10))
+
+				status := &types.UserLastUpdated{EarliestModified: datumTime.Add(-5 * time.Hour)}
+
+				userBGMSummary.Stats.ClearInvalidatedBuckets(status)
+
+				// we have the right length
+				Expect(len(userBGMSummary.Stats.Buckets)).To(Equal(5))
+
+				// we didn't overshoot and nil something we shouldn't have
+				Expect(userBGMSummary.Stats.Buckets[len(userBGMSummary.Stats.Buckets)-1]).ToNot(BeNil())
+			})
+
+			It("doesnt trim if only modified in the future", func() {
+				var dataSetBGMDataCursor *mongo.Cursor
+				userBGMSummary = types.Create[*types.BGMStats](userId)
+				dataSetBGMData = NewDataSetBGMDataAvg(deviceId, datumTime, 10, inTargetBloodGlucose)
+				dataSetBGMDataCursor, err = mongo.NewCursorFromDocuments(ConvertToIntArray(dataSetBGMData), nil, nil)
+
+				err = userBGMSummary.Stats.Update(ctx, dataSetBGMDataCursor)
+				Expect(err).ToNot(HaveOccurred())
+				Expect(len(userBGMSummary.Stats.Buckets)).To(Equal(10))
+				Expect(userBGMSummary.Stats.TotalHours).To(Equal(10))
+
+				status := &types.UserLastUpdated{EarliestModified: datumTime.Add(time.Hour)}
+
+				userBGMSummary.Stats.ClearInvalidatedBuckets(status)
+
+				// we have the right length
+				Expect(len(userBGMSummary.Stats.Buckets)).To(Equal(10))
+
+				// we didn't overshoot and nil something we shouldn't have
+				Expect(userBGMSummary.Stats.Buckets[len(userBGMSummary.Stats.Buckets)-1]).ToNot(BeNil())
+			})
+
+			It("doesnt trim if only modified on the same hour, but after the bucket time", func() {
+				var dataSetBGMDataCursor *mongo.Cursor
+				userBGMSummary = types.Create[*types.BGMStats](userId)
+				midDatumTime := datumTime.Add(30 * time.Minute)
+				dataSetBGMData = NewDataSetBGMDataAvg(deviceId, midDatumTime, 9, inTargetBloodGlucose)
+				dataSetBGMDataCursor, err = mongo.NewCursorFromDocuments(ConvertToIntArray(dataSetBGMData), nil, nil)
+
+				err = userBGMSummary.Stats.Update(ctx, dataSetBGMDataCursor)
+				Expect(err).ToNot(HaveOccurred())
+				Expect(len(userBGMSummary.Stats.Buckets)).To(Equal(10))
+				Expect(userBGMSummary.Stats.TotalHours).To(Equal(10))
+
+				status := &types.UserLastUpdated{EarliestModified: midDatumTime.Add(10 * time.Minute)}
+
+				userBGMSummary.Stats.ClearInvalidatedBuckets(status)
+
+				// we have the right length
+				Expect(len(userBGMSummary.Stats.Buckets)).To(Equal(10))
+
+				// we didn't overshoot and nil something we shouldn't have
+				Expect(userBGMSummary.Stats.Buckets[len(userBGMSummary.Stats.Buckets)-1]).ToNot(BeNil())
+			})
+
+			It("trims if modified on the same hour, and before the bucket time", func() {
+				var dataSetBGMDataCursor *mongo.Cursor
+				userBGMSummary = types.Create[*types.BGMStats](userId)
+				midDatumTime := datumTime.Add(30 * time.Minute)
+				dataSetBGMData = NewDataSetBGMDataAvg(deviceId, midDatumTime, 9, inTargetBloodGlucose)
+				dataSetBGMDataCursor, err = mongo.NewCursorFromDocuments(ConvertToIntArray(dataSetBGMData), nil, nil)
+
+				err = userBGMSummary.Stats.Update(ctx, dataSetBGMDataCursor)
+				Expect(err).ToNot(HaveOccurred())
+				Expect(len(userBGMSummary.Stats.Buckets)).To(Equal(10))
+				Expect(userBGMSummary.Stats.TotalHours).To(Equal(10))
+
+				status := &types.UserLastUpdated{EarliestModified: midDatumTime.Add(-10 * time.Minute)}
+
+				userBGMSummary.Stats.ClearInvalidatedBuckets(status)
+
+				// we have the right length
+				Expect(len(userBGMSummary.Stats.Buckets)).To(Equal(9))
+
+				// we didn't overshoot and nil something we shouldn't have
+				Expect(userBGMSummary.Stats.Buckets[len(userBGMSummary.Stats.Buckets)-1]).ToNot(BeNil())
+			})
+		})
 	})
 })

data/summary/types/summary_cgm_test.go

@@ -1435,5 +1435,97 @@ var _ = Describe("CGM Summary", func() {
 				Expect(*userCGMSummary.Stats.OffsetPeriods["30d"].TotalRecords).To(Equal(288))
 			})
 		})
+
+		Context("ClearInvalidatedBuckets", func() {
+			It("trims the correct buckets", func() {
+				var dataSetCGMDataCursor *mongo.Cursor
+				userCGMSummary = types.Create[*types.CGMStats](userId)
+				dataSetCGMData = NewDataSetCGMDataAvg(deviceId, datumTime, 10, inTargetBloodGlucose)
+				dataSetCGMDataCursor, err = mongo.NewCursorFromDocuments(ConvertToIntArray(dataSetCGMData), nil, nil)
+
+				err = userCGMSummary.Stats.Update(ctx, dataSetCGMDataCursor)
+				Expect(err).ToNot(HaveOccurred())
+				Expect(len(userCGMSummary.Stats.Buckets)).To(Equal(10))
+				Expect(userCGMSummary.Stats.TotalHours).To(Equal(10))
+
+				status := &types.UserLastUpdated{EarliestModified: datumTime.Add(-5 * time.Hour)}
+
+				userCGMSummary.Stats.ClearInvalidatedBuckets(status)
+
+				// we have the right length
+				Expect(len(userCGMSummary.Stats.Buckets)).To(Equal(5))
+
+				// we didn't overshoot and nil something we shouldn't have
+				Expect(userCGMSummary.Stats.Buckets[len(userCGMSummary.Stats.Buckets)-1]).ToNot(BeNil())
+			})
+
+			It("doesnt trim if only modified in the future", func() {
+				var dataSetCGMDataCursor *mongo.Cursor
+				userCGMSummary = types.Create[*types.CGMStats](userId)
+				dataSetCGMData = NewDataSetCGMDataAvg(deviceId, datumTime, 10, inTargetBloodGlucose)
+				dataSetCGMDataCursor, err = mongo.NewCursorFromDocuments(ConvertToIntArray(dataSetCGMData), nil, nil)
+
+				err = userCGMSummary.Stats.Update(ctx, dataSetCGMDataCursor)
+				Expect(err).ToNot(HaveOccurred())
+				Expect(len(userCGMSummary.Stats.Buckets)).To(Equal(10))
+				Expect(userCGMSummary.Stats.TotalHours).To(Equal(10))
+
+				status := &types.UserLastUpdated{EarliestModified: datumTime.Add(time.Hour)}
+
+				userCGMSummary.Stats.ClearInvalidatedBuckets(status)
+
+				// we have the right length
+				Expect(len(userCGMSummary.Stats.Buckets)).To(Equal(10))
+
+				// we didn't overshoot and nil something we shouldn't have
+				Expect(userCGMSummary.Stats.Buckets[len(userCGMSummary.Stats.Buckets)-1]).ToNot(BeNil())
+			})
+
+			It("doesnt trim if only modified on the same hour, but after the bucket time", func() {
+				var dataSetCGMDataCursor *mongo.Cursor
+				userCGMSummary = types.Create[*types.CGMStats](userId)
+				midDatumTime := datumTime.Add(30 * time.Minute)
+				dataSetCGMData = NewDataSetCGMDataAvg(deviceId, midDatumTime, 9, inTargetBloodGlucose)
+				dataSetCGMDataCursor, err = mongo.NewCursorFromDocuments(ConvertToIntArray(dataSetCGMData), nil, nil)
+
+				err = userCGMSummary.Stats.Update(ctx, dataSetCGMDataCursor)
+				Expect(err).ToNot(HaveOccurred())
+				Expect(len(userCGMSummary.Stats.Buckets)).To(Equal(10))
+				Expect(userCGMSummary.Stats.TotalHours).To(Equal(10))
+
+				status := &types.UserLastUpdated{EarliestModified: midDatumTime.Add(10 * time.Minute)}
+
+				userCGMSummary.Stats.ClearInvalidatedBuckets(status)
+
+				// we have the right length
+				Expect(len(userCGMSummary.Stats.Buckets)).To(Equal(10))
+
+				// we didn't overshoot and nil something we shouldn't have
+				Expect(userCGMSummary.Stats.Buckets[len(userCGMSummary.Stats.Buckets)-1]).ToNot(BeNil())
+			})
+
+			It("trims if modified on the same hour, and before the bucket time", func() {
+				var dataSetCGMDataCursor *mongo.Cursor
+				userCGMSummary = types.Create[*types.CGMStats](userId)
+				midDatumTime := datumTime.Add(30 * time.Minute)
+				dataSetCGMData = NewDataSetCGMDataAvg(deviceId, midDatumTime, 9, inTargetBloodGlucose)
+				dataSetCGMDataCursor, err = mongo.NewCursorFromDocuments(ConvertToIntArray(dataSetCGMData), nil, nil)
+
+				err = userCGMSummary.Stats.Update(ctx, dataSetCGMDataCursor)
+				Expect(err).ToNot(HaveOccurred())
+				Expect(len(userCGMSummary.Stats.Buckets)).To(Equal(10))
+				Expect(userCGMSummary.Stats.TotalHours).To(Equal(10))
+
+				status := &types.UserLastUpdated{EarliestModified: midDatumTime.Add(-10 * time.Minute)}
+
+				userCGMSummary.Stats.ClearInvalidatedBuckets(status)
+
+				// we have the right length
+				Expect(len(userCGMSummary.Stats.Buckets)).To(Equal(9))
+
+				// we didn't overshoot and nil something we shouldn't have
+				Expect(userCGMSummary.Stats.Buckets[len(userCGMSummary.Stats.Buckets)-1]).ToNot(BeNil())
+			})
+		})
 	})
 })

go.mod

@@ -119,7 +119,7 @@ require (
 	github.com/montanaflynn/stats v0.7.1 // indirect
 	github.com/oapi-codegen/runtime v1.1.1 // indirect
 	github.com/pelletier/go-toml/v2 v2.1.1 // indirect
-	github.com/pierrec/lz4/v4 v4.1.19 // indirect
+	github.com/pierrec/lz4/v4 v4.1.21 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/prometheus/client_model v0.5.0 // indirect
 	github.com/prometheus/common v0.45.0 // indirect
@@ -157,7 +157,7 @@ require (
 	golang.org/x/time v0.5.0 // indirect
 	google.golang.org/appengine v1.6.8 // indirect
 	google.golang.org/genproto v0.0.0-20240108191215-35c7eff3a6b1 // indirect
-	google.golang.org/genproto/googleapis/api v0.0.0-20240102182953-50ed04b92917 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20240108191215-35c7eff3a6b1 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20240108191215-35c7eff3a6b1 // indirect
 	google.golang.org/protobuf v1.32.0 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect

go.sum

@@ -252,8 +252,8 @@ github.com/onsi/gomega v1.30.0 h1:hvMK7xYz4D3HapigLTeGdId/NcfQx1VHMJc60ew99+8=
 github.com/onsi/gomega v1.30.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ=
 github.com/pelletier/go-toml/v2 v2.1.1 h1:LWAJwfNvjQZCFIDKWYQaM62NcYeYViCmWIwmOStowAI=
 github.com/pelletier/go-toml/v2 v2.1.1/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc=
-github.com/pierrec/lz4/v4 v4.1.19 h1:tYLzDnjDXh9qIxSTKHwXwOYmm9d887Y7Y1ZkyXYHAN4=
-github.com/pierrec/lz4/v4 v4.1.19/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
+github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ=
+github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@@ -464,8 +464,8 @@ google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAs
 google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds=
 google.golang.org/genproto v0.0.0-20240108191215-35c7eff3a6b1 h1:/IWabOtPziuXTEtI1KYCpM6Ss7vaAkeMxk+uXV/xvZs=
 google.golang.org/genproto v0.0.0-20240108191215-35c7eff3a6b1/go.mod h1:+Rvu7ElI+aLzyDQhpHMFMMltsD6m7nqpuWDd2CwJw3k=
-google.golang.org/genproto/googleapis/api v0.0.0-20240102182953-50ed04b92917 h1:rcS6EyEaoCO52hQDupoSfrxI3R6C2Tq741is7X8OvnM=
-google.golang.org/genproto/googleapis/api v0.0.0-20240102182953-50ed04b92917/go.mod h1:CmlNWB9lSezaYELKS5Ym1r44VrrbPUa7JTvw+6MbpJ0=
+google.golang.org/genproto/googleapis/api v0.0.0-20240108191215-35c7eff3a6b1 h1:OPXtXn7fNMaXwO3JvOmF1QyTc00jsSFFz1vXXBOdCDo=
+google.golang.org/genproto/googleapis/api v0.0.0-20240108191215-35c7eff3a6b1/go.mod h1:B5xPO//w8qmBDjGReYLpR6UJPnkldGkCSMoH/2vxJeg=
 google.golang.org/genproto/googleapis/rpc v0.0.0-20240108191215-35c7eff3a6b1 h1:gphdwh0npgs8elJ4T6J+DQJHPVF7RsuJHCfwztUb4J4=
 google.golang.org/genproto/googleapis/rpc v0.0.0-20240108191215-35c7eff3a6b1/go.mod h1:daQN87bsDqDoe316QbbvX60nMoJQa4r6Ds0ZuoAe5yA=
 google.golang.org/grpc v1.60.1 h1:26+wFr+cNqSGFcOXcabYC0lUVJVRa2Sb2ortSK7VrEU=