-
Notifications
You must be signed in to change notification settings - Fork 835
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* small update to docs * initial experiment store commit * single db * updated experiment db * update git ignore * review updates * fix db path
- Loading branch information
1 parent
9dde04c
commit a92bda9
Showing
16 changed files
with
293 additions
and
42 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,7 +1,8 @@ | ||
| .idea/ | ||
| *~ | ||
| /bin/* | ||
| /mnt/pipelinedb | ||
| /mnt/db/pipelinedb | ||
| /mnt/db/experimentdb | ||
| /mnt/mlserver | ||
| /mnt/triton | ||
| /apis-TEMP | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Empty file.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,78 @@ | ||
| package experiment | ||
|
|
||
| import ( | ||
| "github.com/dgraph-io/badger/v3" | ||
| "github.com/seldonio/seldon-core/scheduler/apis/mlops/scheduler" | ||
| "github.com/sirupsen/logrus" | ||
| "google.golang.org/protobuf/proto" | ||
| ) | ||
|
|
||
| type ExperimentDBManager struct { | ||
| db *badger.DB | ||
| } | ||
|
|
||
| func newExperimentDbManager(path string, logger logrus.FieldLogger) (*ExperimentDBManager, error) { | ||
| options := badger.DefaultOptions(path) | ||
| options.Logger = logger.WithField("source", "experimentDb") | ||
| db, err := badger.Open(options) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| return &ExperimentDBManager{ | ||
| db: db, | ||
| }, nil | ||
| } | ||
|
|
||
| func (edb *ExperimentDBManager) Stop() error { | ||
| return edb.db.Close() | ||
| } | ||
|
|
||
| func (edb *ExperimentDBManager) save(experiment *Experiment) error { | ||
| experimentProto := CreateExperimentProto(experiment) | ||
| experimentBytes, err := proto.Marshal(experimentProto) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| return edb.db.Update(func(txn *badger.Txn) error { | ||
| err = txn.Set([]byte(experiment.Name), experimentBytes) | ||
| return err | ||
| }) | ||
| } | ||
|
|
||
| func (edb *ExperimentDBManager) delete(experiment *Experiment) error { | ||
| return edb.db.Update(func(txn *badger.Txn) error { | ||
| err := txn.Delete([]byte(experiment.Name)) | ||
| return err | ||
| }) | ||
| } | ||
|
|
||
| func (edb *ExperimentDBManager) restore(startExperimentCb func(*Experiment) error) error { | ||
| return edb.db.View(func(txn *badger.Txn) error { | ||
| opts := badger.DefaultIteratorOptions | ||
| it := txn.NewIterator(opts) | ||
| defer it.Close() | ||
| for it.Rewind(); it.Valid(); it.Next() { | ||
| item := it.Item() | ||
| err := item.Value(func(v []byte) error { | ||
| snapshot := scheduler.Experiment{} | ||
| err := proto.Unmarshal(v, &snapshot) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| experiment := CreateExperimentFromRequest(&snapshot) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| err = startExperimentCb(experiment) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| return nil | ||
| }) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| } | ||
| return nil | ||
| }) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,73 @@ | ||
| package experiment | ||
|
|
||
| import ( | ||
| "fmt" | ||
| "testing" | ||
|
|
||
| "github.com/google/go-cmp/cmp" | ||
| log "github.com/sirupsen/logrus" | ||
|
|
||
| . "github.com/onsi/gomega" | ||
| ) | ||
|
|
||
| func TestSaveAndRestore(t *testing.T) { | ||
| g := NewGomegaWithT(t) | ||
| type test struct { | ||
| name string | ||
| experiments []*Experiment | ||
| } | ||
|
|
||
| tests := []test{ | ||
| { | ||
| name: "basic experiment", | ||
| experiments: []*Experiment{ | ||
| { | ||
| Name: "test1", | ||
| Candidates: []*Candidate{ | ||
| { | ||
| ModelName: "model1", | ||
| Weight: 50, | ||
| }, | ||
| { | ||
| ModelName: "model2", | ||
| Weight: 50, | ||
| }, | ||
| }, | ||
| Mirror: &Mirror{ | ||
| ModelName: "model3", | ||
| Percent: 90, | ||
| }, | ||
| Config: &Config{ | ||
| StickySessions: true, | ||
| }, | ||
| KubernetesMeta: &KubernetesMeta{ | ||
| Namespace: "default", | ||
| Generation: 2, | ||
| }, | ||
| }, | ||
| }, | ||
| }, | ||
| } | ||
|
|
||
| for _, test := range tests { | ||
| t.Run(test.name, func(t *testing.T) { | ||
| path := fmt.Sprintf("%s/db", t.TempDir()) | ||
| logger := log.New() | ||
| db, err := newExperimentDbManager(getExperimentDbFolder(path), logger) | ||
| g.Expect(err).To(BeNil()) | ||
| for _, p := range test.experiments { | ||
| err := db.save(p) | ||
| g.Expect(err).To(BeNil()) | ||
| } | ||
| err = db.Stop() | ||
| g.Expect(err).To(BeNil()) | ||
|
|
||
| es := NewExperimentServer(log.New(), nil, nil) | ||
| err = es.InitialiseOrRestoreDB(path) | ||
| g.Expect(err).To(BeNil()) | ||
| for _, p := range test.experiments { | ||
| g.Expect(cmp.Equal(p, es.experiments[p.Name])).To(BeTrue()) | ||
| } | ||
| }) | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.