Skip to content

Commit

Permalink
Implement MR service layer (kubeflow#68)
Browse files Browse the repository at this point in the history
* Add service layer

Co-authored-by: Matteo Mortari <[email protected]>

* Refactor service layer api

Co-authored-by: Matteo Mortari <[email protected]>

* Adapt to refactored openapi models

Co-authored-by: Matteo Mortari <[email protected]>

* Add ModelVersion API core impl and test update

* Refactor core layer testing

* Improve core testing

* Cleanup comments

* check put parent context error

* cleanup readme

* Correct terminology in tests

* Using more explicit test-nocache on Makefile

* Refactor core layer to make use of *List from openapi

Co-authored-by: Andrea Lamparelli <[email protected]>

---------

Co-authored-by: Matteo Mortari <[email protected]>
  • Loading branch information
lampajr and tarilabs authored Oct 20, 2023
1 parent 659ec4e commit 86204d4
Show file tree
Hide file tree
Showing 12 changed files with 1,445 additions and 4 deletions.
4 changes: 4 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,10 @@ lint: gen
test: gen
go test ./internal/...

.PHONY: test-nocache
test-nocache: gen
go test ./internal/... -count=1

.PHONY: run/migrate
run/migrate: gen
go run main.go migrate --logtostderr=true -m config/metadata-library
Expand Down
48 changes: 47 additions & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,10 @@ require (
github.com/spf13/cobra v1.7.0
github.com/spf13/pflag v1.0.5
github.com/spf13/viper v1.16.0
github.com/stretchr/testify v1.8.4
github.com/testcontainers/testcontainers-go v0.25.0
github.com/vektah/gqlparser/v2 v2.5.8
golang.org/x/sync v0.2.0
golang.org/x/sync v0.3.0
google.golang.org/grpc v1.57.0
google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.3.0
google.golang.org/protobuf v1.31.0
Expand All @@ -23,29 +25,73 @@ require (
)

require (
github.com/cucumber/gherkin/go/v26 v26.2.0 // indirect
github.com/cucumber/messages/go/v21 v21.0.1 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/gofrs/uuid v4.3.1+incompatible // indirect
github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
github.com/hashicorp/go-memdb v1.3.4 // indirect
github.com/hashicorp/golang-lru v0.5.4 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
)

require (
dario.cat/mergo v1.0.0 // indirect
github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
github.com/Microsoft/go-winio v0.6.1 // indirect
github.com/Microsoft/hcsshim v0.11.0 // indirect
github.com/agnivade/levenshtein v1.1.1 // indirect
github.com/cenkalti/backoff/v4 v4.2.1 // indirect
github.com/containerd/containerd v1.7.6 // indirect
github.com/cpuguy83/dockercfg v0.3.1 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
github.com/cucumber/godog v0.13.0
github.com/docker/distribution v2.8.2+incompatible // indirect
github.com/docker/docker v24.0.6+incompatible // indirect
github.com/docker/go-connections v0.4.0 // indirect
github.com/docker/go-units v0.5.0 // indirect
github.com/fsnotify/fsnotify v1.6.0 // indirect
github.com/go-ole/go-ole v1.2.6 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/protobuf v1.5.3 // indirect
github.com/google/uuid v1.3.1 // indirect
github.com/gorilla/websocket v1.5.0 // indirect
github.com/hashicorp/golang-lru/v2 v2.0.3 // indirect
github.com/hashicorp/hcl v1.0.0 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/jinzhu/inflection v1.0.0 // indirect
github.com/jinzhu/now v1.1.5 // indirect
github.com/klauspost/compress v1.16.0 // indirect
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
github.com/magiconair/properties v1.8.7 // indirect
github.com/mattn/go-sqlite3 v1.14.17 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/moby/patternmatcher v0.5.0 // indirect
github.com/moby/sys/sequential v0.5.0 // indirect
github.com/moby/term v0.5.0 // indirect
github.com/morikuni/aec v1.0.0 // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/opencontainers/image-spec v1.1.0-rc4 // indirect
github.com/opencontainers/runc v1.1.5 // indirect
github.com/pelletier/go-toml/v2 v2.0.8 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/searKing/golang/go v1.2.52 // indirect
github.com/searKing/golang/tools v1.2.29 // indirect
github.com/shirou/gopsutil/v3 v3.23.8 // indirect
github.com/shoenig/go-m1cpu v0.1.6 // indirect
github.com/sirupsen/logrus v1.9.0 // indirect
github.com/spf13/afero v1.9.5 // indirect
github.com/spf13/cast v1.5.1 // indirect
github.com/spf13/jwalterweatherman v1.1.0 // indirect
github.com/subosito/gotenv v1.4.2 // indirect
github.com/tklauser/go-sysconf v0.3.12 // indirect
github.com/tklauser/numcpus v0.6.1 // indirect
github.com/urfave/cli/v2 v2.25.5 // indirect
github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
github.com/yusufpapurcu/wmi v1.2.3 // indirect
golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 // indirect
golang.org/x/mod v0.10.0 // indirect
golang.org/x/net v0.17.0 // indirect
golang.org/x/sys v0.13.0 // indirect
Expand Down
143 changes: 140 additions & 3 deletions go.sum

Large diffs are not rendered by default.

45 changes: 45 additions & 0 deletions internal/core/api.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
package core

import "github.com/opendatahub-io/model-registry/internal/model/openapi"

// Note: for convention, here we are keeping here adherence to the mlmd side
type BaseResourceId int64

type ListOptions struct {
PageSize *int32
OrderBy *string
SortOrder *string
NextPageToken *string
}

type ModelRegistryApi interface {
// REGISTERED MODEL

// UpsertRegisteredModel create or update a registered model, the behavior follows the same
// approach used by MLMD gRPC api. If Id is provided update the entity otherwise create a new one.
UpsertRegisteredModel(registeredModel *openapi.RegisteredModel) (*openapi.RegisteredModel, error)

GetRegisteredModelById(id *BaseResourceId) (*openapi.RegisteredModel, error)
GetRegisteredModelByParams(name *string, externalId *string) (*openapi.RegisteredModel, error)
GetRegisteredModels(listOptions ListOptions) (*openapi.RegisteredModelList, error)

// MODEL VERSION

// Create a new Model Version
// or update a Model Version associated to a specific RegisteredModel identified by registeredModelId parameter
UpsertModelVersion(modelVersion *openapi.ModelVersion, registeredModelId *BaseResourceId) (*openapi.ModelVersion, error)

GetModelVersionById(id *BaseResourceId) (*openapi.ModelVersion, error)
GetModelVersionByParams(name *string, externalId *string) (*openapi.ModelVersion, error)
GetModelVersions(listOptions ListOptions, registeredModelId *BaseResourceId) (*openapi.ModelVersionList, error)

// MODEL ARTIFACT

// Create or update a Model Artifact associated to a specific ModelVersion
// identified by ModelArtifact.ModelVersionId
UpsertModelArtifact(modelArtifact *openapi.ModelArtifact, modelVersionId *BaseResourceId) (*openapi.ModelArtifact, error)

GetModelArtifactById(id *BaseResourceId) (*openapi.ModelArtifact, error)
GetModelArtifactByParams(name *string, externalId *string) (*openapi.ModelArtifact, error)
GetModelArtifacts(listOptions ListOptions, modelVersionId *BaseResourceId) (*openapi.ModelArtifactList, error)
}
43 changes: 43 additions & 0 deletions internal/core/api_utils.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
package core

import (
"github.com/opendatahub-io/model-registry/internal/ml_metadata/proto"
)

func zeroIfNil[T any](input *T) T {
if input != nil {
return *input
}
return *new(T)
}

func BuildListOperationOptions(listOptions ListOptions) (*proto.ListOperationOptions, error) {

result := proto.ListOperationOptions{}
if listOptions.PageSize != nil {
result.MaxResultSize = listOptions.PageSize
}
if listOptions.NextPageToken != nil {
result.NextPageToken = listOptions.NextPageToken
}
if listOptions.OrderBy != nil {
so := listOptions.SortOrder

// default is DESC
isAsc := false
if so != nil && *so == "ASC" {
isAsc = true
}

var orderByField proto.ListOperationOptions_OrderByField_Field
if val, ok := proto.ListOperationOptions_OrderByField_Field_value[*listOptions.OrderBy]; ok {
orderByField = proto.ListOperationOptions_OrderByField_Field(val)
}

result.OrderByField = &proto.ListOperationOptions_OrderByField{
Field: &orderByField,
IsAsc: &isAsc,
}
}
return &result, nil
}
Loading

0 comments on commit 86204d4

Please sign in to comment.