Skip to content

Commit

Permalink
errors either log to file or stop
Browse files Browse the repository at this point in the history
  • Loading branch information
@jh-bate
jh-bate committed Dec 20, 2023
1 parent d41ccc5 commit 899539b
Show file tree
Hide file tree
Showing 2 changed files with 42 additions and 28 deletions.
23 changes: 20 additions & 3 deletions migrations/20231128_jellyfish_migration/20231128_jellyfish_migration.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -117,7 +117,7 @@ func (m *Migration) Initialize() error {
cli.BoolFlag{
Name: "stop-error",
Usage: "stop migration on error",
Destination: &m.dryRun,
Destination: &m.stopOnErr,
},
cli.Int64Flag{
Name: "batch-size",
Expand Down Expand Up @@ -176,6 +176,23 @@ func (m *Migration) getDataCollection() *mongo.Collection {
func (m *Migration) getOplogCollection() *mongo.Collection {
return m.client.Database("local").Collection(oplogName)
}
func (m *Migration) onError(err error, id string, msg string) {
var errFormat = "[id=%s] %s %s"
if err != nil {
if m.stopOnErr {
log.Printf(errFormat, id, msg, err.Error())
os.Exit(1)
}
f, err := os.OpenFile("error.log",
os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
if err != nil {
log.Println(err)
os.Exit(1)
}
defer f.Close()
f.WriteString(fmt.Sprintf(errFormat, id, msg, err.Error()))
}
}

func (m *Migration) prepare() error {
if err := m.checkFreeSpace(); err != nil {
Expand Down Expand Up @@ -387,8 +404,8 @@ func (m *Migration) fetchAndUpdateBatch() bool {

datumID, datumUpdates, err := utils.GetDatumUpdates(dDataResult)
if err != nil {
log.Printf("failed getting datum updates: %s", err)
return false
m.onError(err, datumID, "failed getting updates")
continue
}

updateOp := mongo.NewUpdateOneModel()
Expand Down
47 changes: 22 additions & 25 deletions migrations/20231128_jellyfish_migration/utils/utils.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ package utils
import (
"encoding/json"
"fmt"
"log"
"slices"
"strings"

Expand Down Expand Up @@ -81,67 +80,65 @@ func GetDatumUpdates(bsonData bson.M) (string, bson.M, error) {
var rename bson.M
var identityFields []string

// while doing test runs
var errorDebug = func(id string, err error) (string, bson.M, error) {
log.Printf("[%s] error [%s] creating hash for datum %v", id, err, bsonData)
var errorHandler = func(id string, err error) (string, bson.M, error) {
return id, nil, err
}

datumID, ok := bsonData["_id"].(string)
if !ok {
return errorDebug("", errors.New("cannot get the datum id"))
return errorHandler("", errors.New("cannot get the datum id"))
}

datumType, ok := bsonData["type"].(string)
if !ok {
return errorDebug(datumID, errors.New("cannot get the datum type"))
return errorHandler(datumID, errors.New("cannot get the datum type"))
}

dataBytes, err := bson.Marshal(bsonData)
if err != nil {
return errorDebug(datumID, err)
return errorHandler(datumID, err)
}

switch datumType {
case basal.Type:
var datum *basal.Basal
err = bson.Unmarshal(dataBytes, &datum)
if err != nil {
return errorDebug(datumID, err)
return errorHandler(datumID, err)
}
identityFields, err = datum.IdentityFields()
if err != nil {
return errorDebug(datumID, err)
return errorHandler(datumID, err)
}
case bolus.Type:
var datum *bolus.Bolus
err = bson.Unmarshal(dataBytes, &datum)
if err != nil {
return errorDebug(datumID, err)
return errorHandler(datumID, err)
}
identityFields, err = datum.IdentityFields()
if err != nil {
return errorDebug(datumID, err)
return errorHandler(datumID, err)
}
case device.Type:
var datum *bolus.Bolus
err = bson.Unmarshal(dataBytes, &datum)
if err != nil {
return errorDebug(datumID, err)
return errorHandler(datumID, err)
}
identityFields, err = datum.IdentityFields()
if err != nil {
return errorDebug(datumID, err)
return errorHandler(datumID, err)
}
case pump.Type:
var datum *types.Base
err = bson.Unmarshal(dataBytes, &datum)
if err != nil {
return errorDebug(datumID, err)
return errorHandler(datumID, err)
}
identityFields, err = datum.IdentityFields()
if err != nil {
return errorDebug(datumID, err)
return errorHandler(datumID, err)
}

if pumpSettingsHasBolus(bsonData) {
Expand All @@ -150,15 +147,15 @@ func GetDatumUpdates(bsonData bson.M) (string, bson.M, error) {

sleepSchedules, err := updateIfExistsPumpSettingsSleepSchedules(bsonData)
if err != nil {
return errorDebug(datumID, err)
return errorHandler(datumID, err)
} else if sleepSchedules != nil {
set["sleepSchedules"] = sleepSchedules
}
case selfmonitored.Type:
var datum *selfmonitored.SelfMonitored
err = bson.Unmarshal(dataBytes, &datum)
if err != nil {
return errorDebug(datumID, err)
return errorHandler(datumID, err)
}
if *datum.Units != glucose.MgdL && *datum.Units != glucose.Mgdl {
// NOTE: we need to ensure the same precision for the
Expand All @@ -168,13 +165,13 @@ func GetDatumUpdates(bsonData bson.M) (string, bson.M, error) {
}
identityFields, err = datum.IdentityFields()
if err != nil {
return errorDebug(datumID, err)
return errorHandler(datumID, err)
}
case ketone.Type:
var datum *ketone.Ketone
err = bson.Unmarshal(dataBytes, &datum)
if err != nil {
return errorDebug(datumID, err)
return errorHandler(datumID, err)
}
if *datum.Units != glucose.MgdL && *datum.Units != glucose.Mgdl {
// NOTE: we need to ensure the same precision for the
Expand All @@ -184,13 +181,13 @@ func GetDatumUpdates(bsonData bson.M) (string, bson.M, error) {
}
identityFields, err = datum.IdentityFields()
if err != nil {
return errorDebug(datumID, err)
return errorHandler(datumID, err)
}
case continuous.Type:
var datum *continuous.Continuous
err = bson.Unmarshal(dataBytes, &datum)
if err != nil {
return errorDebug(datumID, err)
return errorHandler(datumID, err)
}
if *datum.Units != glucose.MgdL && *datum.Units != glucose.Mgdl {
// NOTE: we need to ensure the same precision for the
Expand All @@ -200,23 +197,23 @@ func GetDatumUpdates(bsonData bson.M) (string, bson.M, error) {
}
identityFields, err = datum.IdentityFields()
if err != nil {
return errorDebug(datumID, err)
return errorHandler(datumID, err)
}
default:
var datum *types.Base
err = bson.Unmarshal(dataBytes, &datum)
if err != nil {
return errorDebug(datumID, err)
return errorHandler(datumID, err)
}
identityFields, err = datum.IdentityFields()
if err != nil {
return errorDebug(datumID, err)
return errorHandler(datumID, err)
}
}

hash, err := deduplicator.GenerateIdentityHash(identityFields)
if err != nil {
return errorDebug(datumID, err)
return errorHandler(datumID, err)
}
set["_deduplicator"] = bson.M{"hash": hash}

Expand Down

0 comments on commit 899539b

Please sign in to comment.