Refactor directory structure for provider-services

go.mod

@@ -78,7 +78,6 @@ require (
 	github.com/prometheus/client_model v0.2.0 // indirect
 	github.com/prometheus/common v0.32.1 // indirect
 	github.com/prometheus/procfs v0.7.3 // indirect
-	github.com/reugn/go-streams v0.10.0 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/sagikazarmark/locafero v0.4.0 // indirect
 	github.com/sagikazarmark/slog-shim v0.1.0 // indirect

go.sum

@@ -468,8 +468,6 @@ github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1
 github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU=
 github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
-github.com/reugn/go-streams v0.10.0 h1:Y0wHNihEbHsFOFV2/xTOKvud4ZpJPaRTET01fwx2/rQ=
-github.com/reugn/go-streams v0.10.0/go.mod h1:QI5XXifJkVJl2jQ6Cra8I9DvWdJTgqcFYR7amvXZ9Lg=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
 github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=

provider-service/Makefile

@@ -1,9 +1,11 @@
 build:
-	$(MAKE) -C internal/kfp build
-	$(MAKE) -C internal/vai build
+	$(MAKE) -C kfp build
+	$(MAKE) -C vai build
 
 test:
-	go test ./... -tags=unit
+#	$(MAKE) -C base test
+	$(MAKE) -C kfp test
+	$(MAKE) -C vai test
 
 #functional-test: docker-build
 #	docker compose up -d
@@ -13,11 +15,11 @@ test:
 ##@ Containers
 
 docker-build:
-	$(MAKE) -C internal/kfp docker-build
-	$(MAKE) -C internal/vai docker-build
+	$(MAKE) -C kfp docker-build
+	$(MAKE) -C vai docker-build
 
 docker-push:
-	$(MAKE) -C internal/kfp docker-push
-	$(MAKE) -C internal/vai docker-push
+	$(MAKE) -C kfp docker-push
+	$(MAKE) -C vai docker-push
 
 

provider-service/base/Makefile

@@ -0,0 +1,5 @@
+##@ Development
+unit-test:
+	go test ./... -tags=unit
+
+test: unit-test

provider-service/base/go.mod

@@ -0,0 +1,90 @@
+module github.com/sky-uk/kfp-operator/provider-service/base
+
+go 1.20
+
+require (
+	github.com/go-logr/logr v1.2.3
+	github.com/go-resty/resty/v2 v2.16.0
+	github.com/jarcoal/httpmock v1.3.1
+	github.com/onsi/ginkgo/v2 v2.8.0
+	github.com/onsi/gomega v1.25.0
+	github.com/sky-uk/kfp-operator v0.6.0
+	github.com/sky-uk/kfp-operator/argo/common v0.0.0-20241107144142-477120a374b9
+	github.com/spf13/viper v1.16.0
+	go.uber.org/zap v1.24.0
+	k8s.io/apimachinery v0.25.4
+	k8s.io/client-go v0.25.4
+)
+
+require (
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/docker/distribution v2.7.1+incompatible // indirect
+	github.com/emicklei/go-restful/v3 v3.8.0 // indirect
+	github.com/evanphx/json-patch v5.6.0+incompatible // indirect
+	github.com/fsnotify/fsnotify v1.6.0 // indirect
+	github.com/go-logr/zapr v1.2.3 // indirect
+	github.com/go-openapi/jsonpointer v0.19.5 // indirect
+	github.com/go-openapi/jsonreference v0.20.0 // indirect
+	github.com/go-openapi/swag v0.22.3 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
+	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/google/gnostic v0.5.7-v3refs // indirect
+	github.com/google/go-cmp v0.5.9 // indirect
+	github.com/google/gofuzz v1.2.0 // indirect
+	github.com/google/uuid v1.3.0 // indirect
+	github.com/hashicorp/hcl v1.0.0 // indirect
+	github.com/imdario/mergo v0.3.15 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/magiconair/properties v1.8.7 // indirect
+	github.com/mailru/easyjson v0.7.7 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
+	github.com/mitchellh/mapstructure v1.5.0 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/opencontainers/go-digest v1.0.0 // indirect
+	github.com/pelletier/go-toml/v2 v2.0.8 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/prometheus/client_golang v1.15.1 // indirect
+	github.com/prometheus/client_model v0.3.0 // indirect
+	github.com/prometheus/common v0.42.0 // indirect
+	github.com/prometheus/procfs v0.9.0 // indirect
+	github.com/spf13/afero v1.9.5 // indirect
+	github.com/spf13/cast v1.5.1 // indirect
+	github.com/spf13/jwalterweatherman v1.1.0 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/stretchr/testify v1.8.4 // indirect
+	github.com/subosito/gotenv v1.4.2 // indirect
+	github.com/thanhpk/randstr v1.0.4 // indirect
+	go.uber.org/atomic v1.9.0 // indirect
+	go.uber.org/multierr v1.8.0 // indirect
+	golang.org/x/net v0.27.0 // indirect
+	golang.org/x/oauth2 v0.11.0 // indirect
+	golang.org/x/sys v0.22.0 // indirect
+	golang.org/x/term v0.22.0 // indirect
+	golang.org/x/text v0.16.0 // indirect
+	golang.org/x/time v0.6.0 // indirect
+	gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect
+	google.golang.org/appengine v1.6.7 // indirect
+	google.golang.org/protobuf v1.31.0 // indirect
+	gopkg.in/inf.v0 v0.9.1 // indirect
+	gopkg.in/ini.v1 v1.67.0 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+	k8s.io/api v0.25.4 // indirect
+	k8s.io/apiextensions-apiserver v0.23.5 // indirect
+	k8s.io/component-base v0.23.5 // indirect
+	k8s.io/klog/v2 v2.70.1 // indirect
+	k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1 // indirect
+	k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed // indirect
+	sigs.k8s.io/controller-runtime v0.11.2 // indirect
+	sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect
+	sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
+	sigs.k8s.io/yaml v1.3.0 // indirect
+)
+
+replace k8s.io/kubernetes => k8s.io/kubernetes v1.11.1

provider-service/base/go.work

@@ -0,0 +1,7 @@
+go 1.20
+
+use (
+	.
+	../..
+	../../argo/common
+)

provider-service/internal/base/k8s_api.go → provider-service/base/pkg/k8s_api.go

@@ -1,4 +1,4 @@
-package base
+package pkg
 
 import (
 	"context"

provider-service/internal/publisher/http_webhook.go → provider-service/base/pkg/publisher/http_webhook.go

@@ -7,7 +7,7 @@ import (
 	"fmt"
 	"github.com/go-resty/resty/v2"
 	"github.com/sky-uk/kfp-operator/argo/common"
-	"github.com/sky-uk/kfp-operator/provider-service/internal/base"
+	"github.com/sky-uk/kfp-operator/provider-service/base/pkg"
 )
 
 type HttpWebhookSink struct {
@@ -35,7 +35,7 @@ func (hws HttpWebhookSink) SendEvents() {
 	for data := range hws.in {
 		var err error
 		switch object := data.(type) {
-		case base.StreamMessage:
+		case pkg.StreamMessage:
 			err = hws.send(object.RunCompletionEventData)
 			if err != nil {
 				logger.Error(err, "Failed to send event", "event", fmt.Sprintf("%+v", object))

provider-service/internal/publisher/http_webhook_unit_test.go → provider-service/base/pkg/publisher/http_webhook_unit_test.go

@@ -3,14 +3,13 @@ package publisher
 import (
 	"context"
 	"fmt"
-
 	"github.com/go-logr/logr"
-	"github.com/go-resty/resty/v2"
+	resty "github.com/go-resty/resty/v2"
 	"github.com/jarcoal/httpmock"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	"github.com/sky-uk/kfp-operator/argo/common"
-	"github.com/sky-uk/kfp-operator/provider-service/internal/base"
+	. "github.com/sky-uk/kfp-operator/provider-service/base/pkg"
 	"go.uber.org/zap/zapcore"
 )
 
@@ -31,12 +30,12 @@ var _ = Context("SendEvents", func() {
 			onSuccessCalled := make(chan any, 1)
 
 			go httpWebhook.SendEvents()
-			handlers := base.OnCompleteHandlers{
+			handlers := OnCompleteHandlers{
 				OnSuccessHandler: func() {
 					onSuccessCalled <- "ring ring!"
 				},
 			}
-			streamMessage := base.StreamMessage{OnCompleteHandlers: handlers}
+			streamMessage := StreamMessage{OnCompleteHandlers: handlers}
 			in <- streamMessage
 
 			Eventually(func() int { return httpmock.GetCallCountInfo()[fmt.Sprintf("POST %s", webhookUrl)] }).Should(Equal(1))
@@ -55,12 +54,12 @@ var _ = Context("SendEvents", func() {
 			onFailureCalled := make(chan any)
 
 			go httpWebhook.SendEvents()
-			handlers := base.OnCompleteHandlers{
+			handlers := OnCompleteHandlers{
 				OnFailureHandler: func() {
 					onFailureCalled <- "ring ring!"
 				},
 			}
-			streamMessage := base.StreamMessage{OnCompleteHandlers: handlers}
+			streamMessage := StreamMessage{OnCompleteHandlers: handlers}
 			in <- streamMessage
 
 			Eventually(func() int { return httpmock.GetCallCountInfo()[fmt.Sprintf("POST %s", webhookUrl)] }).Should(Equal(1))

provider-service/internal/base/stream_message.go → provider-service/base/pkg/stream_message.go

@@ -1,4 +1,4 @@
-package base
+package pkg
 
 import "github.com/sky-uk/kfp-operator/argo/common"
 

provider-service/internal/kfp/Makefile → provider-service/kfp/Makefile

@@ -1,5 +1,7 @@
-include ../../../common.mk
-include ../../get-proto.mk
+include ../../common.mk
+include ../../docker-targets.mk
+include ./get-proto.mk
+
 IMG := kfp-operator-kfp-provider-eventing
 
 all: build
@@ -12,18 +14,18 @@ mockgen: ## Download mockgen locally if necessary.
 
 generate: protoc-gen-go mockgen ## Generate service definitions from protobuf
 	$(call get-proto,github.com/google/ml-metadata,v1.5.0)
-	protoc --go_out=. --go-grpc_out=ml_metadata \
+	protoc --go_out=. --go-grpc_out=internal/ml_metadata \
 		-I $(PROTOPATH)/github.com/google/[email protected]/ \
-		--go_opt=Mml_metadata/proto/metadata_store_service.proto=/ml_metadata \
-		--go_opt=Mml_metadata/proto/metadata_store.proto=/ml_metadata \
-		--go_opt=Mml_metadata/proto/metadata_source.proto=/ml_metadata \
+		--go_opt=Mml_metadata/proto/metadata_store_service.proto=/internal/ml_metadata \
+		--go_opt=Mml_metadata/proto/metadata_store.proto=/internal/ml_metadata \
+		--go_opt=Mml_metadata/proto/metadata_source.proto=/internal/ml_metadata \
 		--go-grpc_opt=module=ml_metadata/proto \
 		ml_metadata/proto/metadata_store_service.proto \
 		ml_metadata/proto/metadata_store.proto
 
-	$(MOCKGEN) -destination ml_metadata/metadata_store_service_grpc_mock.go -package=ml_metadata -source=ml_metadata/metadata_store_service_grpc.pb.go
-	$(MOCKGEN) -destination run_service_grpc_mock.go -package=kfp github.com/kubeflow/pipelines/backend/api/go_client RunServiceClient
-	$(MOCKGEN) -destination job_service_grpc_mock.go -package=kfp github.com/kubeflow/pipelines/backend/api/go_client JobServiceClient
+	$(MOCKGEN) -destination internal/ml_metadata/metadata_store_service_grpc_mock.go -package=ml_metadata -source=internal/ml_metadata/metadata_store_service_grpc.pb.go
+	$(MOCKGEN) -destination internal/run_service_grpc_mock.go -package=internal github.com/kubeflow/pipelines/backend/api/go_client RunServiceClient
+	$(MOCKGEN) -destination internal/job_service_grpc_mock.go -package=internal github.com/kubeflow/pipelines/backend/api/go_client JobServiceClient
 
 ##@ Development
 
@@ -38,8 +40,5 @@ test: unit-test decoupled-test
 ##@ Build
 
 build: generate
-	CGO_ENABLED=0 GOOS=$(GOOS) GOARCH=$(GOARCH) go build -o bin/kfp-provider-service ../../cmd/kfp/main.go
-
-##@ Containers
+	CGO_ENABLED=0 GOOS=$(GOOS) GOARCH=$(GOARCH) go build -o bin/kfp-provider-service cmd/main.go
 
-include ../../../docker-targets.mk

provider-service/cmd/kfp/main.go → provider-service/kfp/cmd/main.go

@@ -1,13 +1,13 @@
-package main
+package cmd
 
 import (
 	"context"
 	"fmt"
-	"github.com/sky-uk/kfp-operator/provider-service/internal/kfp"
+	kfp "github.com/sky-uk/kfp-operator/provider-service/kfp/internal"
 
 	"github.com/go-logr/logr"
 	"github.com/sky-uk/kfp-operator/argo/common"
-	configLoader "github.com/sky-uk/kfp-operator/provider-service/cmd/config"
+	configLoader "github.com/sky-uk/kfp-operator/provider-service/base/pkg/config"
 	"go.uber.org/zap/zapcore"
 )