try dDataCursor.SetBatchSize(1000)

tidepool-org · Dec 21, 2023 · 02392b5 · 02392b5
1 parent 00b3e7c
commit 02392b5
Showing 1 changed file with 21 additions and 20 deletions.
diff --git a/migrations/20231128_jellyfish_migration/20231128_jellyfish_migration.go b/migrations/20231128_jellyfish_migration/20231128_jellyfish_migration.go
@@ -10,6 +10,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/tidepool-org/platform/migrations/20231128_jellyfish_migration/utils"
 	"github.com/urfave/cli"
 	"go.mongodb.org/mongo-driver/bson"
 	"go.mongodb.org/mongo-driver/mongo"
@@ -388,7 +389,7 @@ func (m *Migration) fetchAndUpdateBatch() bool {
 				Sort:  bson.M{"_id": 1},
 			},
 		)
-		//dDataCursor.SetBatchSize(1000)
+		dDataCursor.SetBatchSize(1000)
 
 		if err != nil {
 			log.Printf("failed to select data: %s", err)
@@ -401,25 +402,25 @@ func (m *Migration) fetchAndUpdateBatch() bool {
 		updateStart := time.Now()
 		for dDataCursor.Next(m.ctx) {
 
-			// start := time.Now()
-			// var dDataResult bson.M
-			// if err = dDataCursor.Decode(&dDataResult); err != nil {
-			// 	log.Printf("failed decoding data: %s", err)
-			// 	return false
-			// }
-			// log.Printf("cursor decode %s", time.Since(start))
-			// datumID, datumUpdates, err := utils.GetDatumUpdates(dDataResult)
-			// if err != nil {
-			// 	m.onError(err, datumID, "failed getting updates")
-			// 	continue
-			// }
-			// log.Printf("datum updates %s", time.Since(start))
-			// updateOp := mongo.NewUpdateOneModel()
-			// updateOp.SetFilter(bson.M{"_id": datumID, "modifiedTime": dDataResult["modifiedTime"]})
-			// updateOp.SetUpdate(datumUpdates)
-			// m.updates = append(m.updates, updateOp)
-			// m.lastUpdatedId = datumID
-			// log.Printf("added to updates %s", time.Since(start))
+			start := time.Now()
+			var dDataResult bson.M
+			if err = dDataCursor.Decode(&dDataResult); err != nil {
+				log.Printf("failed decoding data: %s", err)
+				return false
+			}
+			log.Printf("cursor decode %s", time.Since(start))
+			datumID, datumUpdates, err := utils.GetDatumUpdates(dDataResult)
+			if err != nil {
+				m.onError(err, datumID, "failed getting updates")
+				continue
+			}
+			log.Printf("datum updates %s", time.Since(start))
+			updateOp := mongo.NewUpdateOneModel()
+			updateOp.SetFilter(bson.M{"_id": datumID, "modifiedTime": dDataResult["modifiedTime"]})
+			updateOp.SetUpdate(datumUpdates)
+			m.updates = append(m.updates, updateOp)
+			m.lastUpdatedId = datumID
+			log.Printf("added to updates %s", time.Since(start))
 		}
 
 		log.Printf("batch iteration took %s", time.Since(updateStart))