Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[TEST] brokerapi chi and keb std lib ServeMux #1480

Draft
wants to merge 9 commits into
base: main
Choose a base branch
from
Draft
2 changes: 1 addition & 1 deletion cmd/broker/binding_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ import (
"testing"

"github.com/google/uuid"
"github.com/pivotal-cf/brokerapi/v8/domain"
"github.com/pivotal-cf/brokerapi/v11/domain"
)

type ErrorResponse struct {
Expand Down
32 changes: 15 additions & 17 deletions cmd/broker/bindings_envtest_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,33 +6,32 @@ import (
"encoding/json"
"fmt"
"io"
"log/slog"
"net/http"
"net/http/httptest"
"os"
"testing"
"time"

"github.com/kyma-project/kyma-environment-broker/internal/event"

"github.com/golang-jwt/jwt/v4"
"github.com/google/uuid"
brokerBindings "github.com/kyma-project/kyma-environment-broker/internal/broker/bindings"
"gopkg.in/yaml.v2"
rbacv1 "k8s.io/api/rbac/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"

"code.cloudfoundry.org/lager"
"github.com/gorilla/mux"
"github.com/kyma-project/kyma-environment-broker/internal"
"github.com/kyma-project/kyma-environment-broker/internal/broker"
brokerBindings "github.com/kyma-project/kyma-environment-broker/internal/broker/bindings"
"github.com/kyma-project/kyma-environment-broker/internal/event"
"github.com/kyma-project/kyma-environment-broker/internal/fixture"
"github.com/kyma-project/kyma-environment-broker/internal/httputil"
"github.com/kyma-project/kyma-environment-broker/internal/kubeconfig"
"github.com/kyma-project/kyma-environment-broker/internal/storage"
"github.com/pivotal-cf/brokerapi/v8/domain"
"github.com/pivotal-cf/brokerapi/v8/handlers"
"github.com/pivotal-cf/brokerapi/v11/domain"
"github.com/pivotal-cf/brokerapi/v11/handlers"
"github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"gopkg.in/yaml.v2"
corev1 "k8s.io/api/core/v1"
rbacv1 "k8s.io/api/rbac/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/util/wait"
Expand Down Expand Up @@ -81,8 +80,7 @@ func TestCreateBindingEndpoint(t *testing.T) {
TimestampFormat: time.RFC3339Nano,
})

brokerLogger := lager.NewLogger("test")
brokerLogger.RegisterSink(lager.NewWriterSink(logs.Writer(), lager.DEBUG))
brokerLogger := slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelDebug})).With("source", "test")

//// schema
sch := runtime.NewScheme()
Expand Down Expand Up @@ -220,10 +218,10 @@ func TestCreateBindingEndpoint(t *testing.T) {
}, brokerLogger)

//// attach bindings api
router := mux.NewRouter()
router.HandleFunc("/v2/service_instances/{instance_id}/service_bindings/{binding_id}", apiHandler.Bind).Methods(http.MethodPut)
router.HandleFunc("/v2/service_instances/{instance_id}/service_bindings/{binding_id}", apiHandler.GetBinding).Methods(http.MethodGet)
router.HandleFunc("/v2/service_instances/{instance_id}/service_bindings/{binding_id}", apiHandler.Unbind).Methods(http.MethodDelete)
router := httputil.NewRouter()
router.HandleFunc("PUT /v2/service_instances/{instance_id}/service_bindings/{binding_id}", apiHandler.Bind)
router.HandleFunc("GET /v2/service_instances/{instance_id}/service_bindings/{binding_id}", apiHandler.GetBinding)
router.HandleFunc("DELETE /v2/service_instances/{instance_id}/service_bindings/{binding_id}", apiHandler.Unbind)
httpServer = httptest.NewServer(router)
defer httpServer.Close()

Expand Down
34 changes: 16 additions & 18 deletions cmd/broker/broker_suite_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,37 +6,31 @@ import (
"encoding/json"
"fmt"
"io"
"log/slog"
"net/http"
"net/http/httptest"
"os"
"reflect"
"testing"
"time"

"github.com/kyma-project/kyma-environment-broker/internal/customresources"

"github.com/kyma-project/kyma-environment-broker/internal/storage/dberr"

imv1 "github.com/kyma-project/infrastructure-manager/api/v1"

"github.com/kyma-project/kyma-environment-broker/internal/kubeconfig"
"github.com/kyma-project/kyma-environment-broker/internal/metricsv2"
"github.com/prometheus/client_golang/prometheus/promhttp"
"github.com/prometheus/client_golang/prometheus/testutil"

"code.cloudfoundry.org/lager"
"github.com/google/uuid"
"github.com/gorilla/mux"
"github.com/kyma-project/control-plane/components/provisioner/pkg/gqlschema"
imv1 "github.com/kyma-project/infrastructure-manager/api/v1"
"github.com/kyma-project/kyma-environment-broker/common/gardener"
"github.com/kyma-project/kyma-environment-broker/common/orchestration"
"github.com/kyma-project/kyma-environment-broker/internal"
"github.com/kyma-project/kyma-environment-broker/internal/broker"
kebConfig "github.com/kyma-project/kyma-environment-broker/internal/config"
"github.com/kyma-project/kyma-environment-broker/internal/customresources"
"github.com/kyma-project/kyma-environment-broker/internal/edp"
"github.com/kyma-project/kyma-environment-broker/internal/event"
"github.com/kyma-project/kyma-environment-broker/internal/expiration"
"github.com/kyma-project/kyma-environment-broker/internal/fixture"
"github.com/kyma-project/kyma-environment-broker/internal/httputil"
"github.com/kyma-project/kyma-environment-broker/internal/kubeconfig"
kcMock "github.com/kyma-project/kyma-environment-broker/internal/kubeconfig/automock"
"github.com/kyma-project/kyma-environment-broker/internal/metricsv2"
"github.com/kyma-project/kyma-environment-broker/internal/notification"
kebOrchestration "github.com/kyma-project/kyma-environment-broker/internal/orchestration"
orchestrate "github.com/kyma-project/kyma-environment-broker/internal/orchestration/handlers"
Expand All @@ -47,9 +41,12 @@ import (
"github.com/kyma-project/kyma-environment-broker/internal/provisioner"
kebRuntime "github.com/kyma-project/kyma-environment-broker/internal/runtime"
"github.com/kyma-project/kyma-environment-broker/internal/storage"
"github.com/pivotal-cf/brokerapi/v8/domain"
"github.com/pivotal-cf/brokerapi/v8/domain/apiresponses"
"github.com/kyma-project/kyma-environment-broker/internal/storage/dberr"
"github.com/pivotal-cf/brokerapi/v11/domain"
"github.com/pivotal-cf/brokerapi/v11/domain/apiresponses"
"github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus/promhttp"
"github.com/prometheus/client_golang/prometheus/testutil"
"github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
Expand Down Expand Up @@ -96,7 +93,7 @@ type BrokerSuiteTest struct {
gardenerClient dynamic.Interface

httpServer *httptest.Server
router *mux.Router
router *httputil.Router

t *testing.T
inputBuilderFactory input.CreatorForPlan
Expand Down Expand Up @@ -231,7 +228,7 @@ func NewBrokerSuiteTestWithConfig(t *testing.T, cfg *Config, version ...string)
storageCleanup: storageCleanup,
provisionerClient: provisionerClient,
gardenerClient: gardenerClient,
router: mux.NewRouter(),
router: httputil.NewRouter(),
t: t,
inputBuilderFactory: inputFactory,
k8sKcp: cli,
Expand Down Expand Up @@ -387,8 +384,9 @@ func (s *BrokerSuiteTest) CreateAPI(inputFactory broker.PlanValidator, cfg *Conf
var fakeKcpK8sClient = fake.NewClientBuilder().Build()
kcBuilder := &kcMock.KcBuilder{}
kcBuilder.On("Build", nil).Return("--kubeconfig file", nil)
logger := slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{Level: LogLevel})).With("source", "api")
createAPI(s.router, servicesConfig, inputFactory, cfg, db, provisioningQueue, deprovisionQueue, updateQueue,
lager.NewLogger("api"), logs, planDefaults, kcBuilder, skrK8sClientProvider, skrK8sClientProvider, gardenerClient, fakeKcpK8sClient, eventBroker)
logger, logs, planDefaults, kcBuilder, skrK8sClientProvider, skrK8sClientProvider, gardenerClient, fakeKcpK8sClient, eventBroker)

s.httpServer = httptest.NewServer(s.router)
}
Expand Down
2 changes: 1 addition & 1 deletion cmd/broker/deprovisioning_suite_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ import (
"github.com/kyma-project/kyma-environment-broker/internal/process"
"github.com/kyma-project/kyma-environment-broker/internal/provisioner"
"github.com/kyma-project/kyma-environment-broker/internal/storage"
"github.com/pivotal-cf/brokerapi/v8/domain"
"github.com/pivotal-cf/brokerapi/v11/domain"
"github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
Expand Down
2 changes: 1 addition & 1 deletion cmd/broker/deprovisioning_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ import (
"github.com/google/uuid"
"github.com/kyma-project/kyma-environment-broker/common/runtime"
"github.com/kyma-project/kyma-environment-broker/internal"
"github.com/pivotal-cf/brokerapi/v8/domain"
"github.com/pivotal-cf/brokerapi/v11/domain"
"github.com/stretchr/testify/assert"
)

Expand Down
2 changes: 1 addition & 1 deletion cmd/broker/expiration_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import (
"github.com/google/uuid"
"github.com/kyma-project/kyma-environment-broker/internal"
"github.com/kyma-project/kyma-environment-broker/internal/broker"
"github.com/pivotal-cf/brokerapi/v8/domain"
"github.com/pivotal-cf/brokerapi/v11/domain"
"github.com/stretchr/testify/assert"
)

Expand Down
77 changes: 37 additions & 40 deletions cmd/broker/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,27 +3,17 @@ package main
import (
"context"
"fmt"
"io"
"log/slog"
"net/http"
"os"
gruntime "runtime"
"runtime/pprof"
"sort"
"time"

"k8s.io/client-go/kubernetes"

imv1 "github.com/kyma-project/infrastructure-manager/api/v1"

"github.com/kyma-project/kyma-environment-broker/internal/expiration"
"github.com/kyma-project/kyma-environment-broker/internal/metricsv2"
"github.com/kyma-project/kyma-environment-broker/internal/whitelist"

"code.cloudfoundry.org/lager"
"github.com/dlmiddlecote/sqlstats"
shoot "github.com/gardener/gardener/pkg/apis/core/v1beta1"
"github.com/gorilla/handlers"
"github.com/gorilla/mux"
imv1 "github.com/kyma-project/infrastructure-manager/api/v1"
"github.com/kyma-project/kyma-environment-broker/common/gardener"
"github.com/kyma-project/kyma-environment-broker/common/hyperscaler"
orchestrationExt "github.com/kyma-project/kyma-environment-broker/common/orchestration"
Expand All @@ -37,9 +27,11 @@ import (
"github.com/kyma-project/kyma-environment-broker/internal/event"
"github.com/kyma-project/kyma-environment-broker/internal/events"
eventshandler "github.com/kyma-project/kyma-environment-broker/internal/events/handler"
"github.com/kyma-project/kyma-environment-broker/internal/expiration"
"github.com/kyma-project/kyma-environment-broker/internal/health"
"github.com/kyma-project/kyma-environment-broker/internal/httputil"
"github.com/kyma-project/kyma-environment-broker/internal/kubeconfig"
"github.com/kyma-project/kyma-environment-broker/internal/metricsv2"
"github.com/kyma-project/kyma-environment-broker/internal/middleware"
"github.com/kyma-project/kyma-environment-broker/internal/notification"
"github.com/kyma-project/kyma-environment-broker/internal/orchestration"
Expand All @@ -53,20 +45,24 @@ import (
"github.com/kyma-project/kyma-environment-broker/internal/storage/dbmodel"
"github.com/kyma-project/kyma-environment-broker/internal/suspension"
"github.com/kyma-project/kyma-environment-broker/internal/swagger"
"github.com/kyma-project/kyma-environment-broker/internal/whitelist"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promhttp"
"github.com/sirupsen/logrus"
"github.com/vrischmann/envconfig"
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/dynamic"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/kubernetes/scheme"
"k8s.io/client-go/rest"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/apiutil"
"sigs.k8s.io/controller-runtime/pkg/client/config"
)

var LogLevel = new(slog.LevelVar)

// Config holds configuration for the whole application
type Config struct {
// DbInMemory allows to use memory storage instead of the postgres one.
Expand All @@ -81,7 +77,7 @@ type Config struct {
// DevelopmentMode if set to true then errors are returned in http
// responses, otherwise errors are only logged and generic message
// is returned to client.
// Currently works only with /info endpoints.
// Currently, works only with /info endpoints.
DevelopmentMode bool `envconfig:"default=false"`

// DumpProvisionerRequests enables dumping Provisioner requests. Must be disabled on Production environments
Expand Down Expand Up @@ -179,24 +175,24 @@ const (
startStageName = "start"
)

func periodicProfile(logger lager.Logger, profiler ProfilerConfig) {
func periodicProfile(logger *slog.Logger, profiler ProfilerConfig) {
if profiler.Memory == false {
return
}
logger.Info(fmt.Sprintf("Starting periodic profiler %v", profiler))
if err := os.MkdirAll(profiler.Path, os.ModePerm); err != nil {
logger.Error(fmt.Sprintf("Failed to create dir %v for profile storage", profiler.Path), err)
logger.Error(fmt.Sprintf("Failed to create dir %v for profile storage", profiler.Path), "error", err)
}
for {
profName := fmt.Sprintf("%v/mem-%v.pprof", profiler.Path, time.Now().Unix())
logger.Info(fmt.Sprintf("Creating periodic memory profile %v", profName))
profFile, err := os.Create(profName)
if err != nil {
logger.Error(fmt.Sprintf("Creating periodic memory profile %v failed", profName), err)
logger.Error(fmt.Sprintf("Creating periodic memory profile %v failed", profName), "error", err)
}
err = pprof.Lookup("allocs").WriteTo(profFile, 0)
if err != nil {
logger.Error(fmt.Sprintf("Failed to write periodic memory profile to %v file", profName), err)
logger.Error(fmt.Sprintf("Failed to write periodic memory profile to %v file", profName), "error", err)
}
gruntime.GC()
time.Sleep(profiler.Sampling)
Expand Down Expand Up @@ -231,11 +227,15 @@ func main() {
if cfg.LogLevel != "" {
l, _ := logrus.ParseLevel(cfg.LogLevel)
logs.SetLevel(l)
if cfg.LogLevel == "debug" {
LogLevel.Set(slog.LevelDebug)
}
}

cfg.OrchestrationConfig.KubernetesVersion = cfg.Provisioner.KubernetesVersion

// create logger
logger := lager.NewLogger("kyma-env-broker")
logger := slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{Level: LogLevel})).With("source", "kyma-env-broker")

logger.Info("Starting Kyma Environment Broker")

Expand Down Expand Up @@ -331,7 +331,7 @@ func main() {
kcBuilder := kubeconfig.NewBuilder(provisionerClient, kcpK8sClient, skrK8sClientProvider)

// create server
router := mux.NewRouter()
router := httputil.NewRouter()
createAPI(router, servicesConfig, inputFactory, &cfg, db, provisionQueue, deprovisionQueue, updateQueue, logger, logs,
inputFactory.GetPlanDefaults, kcBuilder, skrK8sClientProvider, skrK8sClientProvider, gardenerClient, kcpK8sClient, eventBroker)

Expand Down Expand Up @@ -387,12 +387,19 @@ func main() {
expirationHandler := expiration.NewHandler(db.Instances(), db.Operations(), deprovisionQueue, logs)
expirationHandler.AttachRoutes(router)

router.StrictSlash(true).PathPrefix("/").Handler(http.StripPrefix("/", http.FileServer(http.Dir("/swagger"))))
svr := handlers.CustomLoggingHandler(os.Stdout, router, func(writer io.Writer, params handlers.LogFormatterParams) {
logs.Infof("Call handled: method=%s url=%s statusCode=%d size=%d", params.Request.Method, params.URL.Path, params.StatusCode, params.Size)
})
/*
** to replace with own solution
router.StrictSlash(true).PathPrefix("/").Handler(http.StripPrefix("/", http.FileServer(http.Dir("/swagger"))))
*/

/*
** to replace with own solution
svr := handlers.CustomLoggingHandler(os.Stdout, router, func(writer io.Writer, params handlers.LogFormatterParams) {
logs.Infof("Call handled: method=%s url=%s statusCode=%d size=%d", params.Request.Method, params.URL.Path, params.StatusCode, params.Size)
})
*/

fatalOnError(http.ListenAndServe(cfg.Host+":"+cfg.Port, svr), logs)
fatalOnError(http.ListenAndServe(cfg.Host+":"+cfg.Port, router), logs)
}

func logConfiguration(logs *logrus.Logger, cfg Config) {
Expand All @@ -411,21 +418,14 @@ func logConfiguration(logs *logrus.Logger, cfg Config) {
logs.Infof("Is UpdateCustomResourcesLabelsOnAccountMove enabled: %t", cfg.Broker.UpdateCustomResourcesLabelsOnAccountMove)
}

func createAPI(router *mux.Router, servicesConfig broker.ServicesConfig, planValidator broker.PlanValidator, cfg *Config, db storage.BrokerStorage,
provisionQueue, deprovisionQueue, updateQueue *process.Queue, logger lager.Logger, logs logrus.FieldLogger, planDefaults broker.PlanDefaults, kcBuilder kubeconfig.KcBuilder, clientProvider K8sClientProvider, kubeconfigProvider KubeconfigProvider, gardenerClient, kcpK8sClient client.Client, publisher event.Publisher) {
func createAPI(router *httputil.Router, servicesConfig broker.ServicesConfig, planValidator broker.PlanValidator, cfg *Config, db storage.BrokerStorage,
provisionQueue, deprovisionQueue, updateQueue *process.Queue, logger *slog.Logger, logs logrus.FieldLogger, planDefaults broker.PlanDefaults, kcBuilder kubeconfig.KcBuilder, clientProvider K8sClientProvider, kubeconfigProvider KubeconfigProvider, gardenerClient, kcpK8sClient client.Client, publisher event.Publisher) {

suspensionCtxHandler := suspension.NewContextUpdateHandler(db.Operations(), provisionQueue, deprovisionQueue, logs)

defaultPlansConfig, err := servicesConfig.DefaultPlansConfig()
fatalOnError(err, logs)

debugSink, err := lager.NewRedactingSink(lager.NewWriterSink(os.Stdout, lager.DEBUG), []string{"instance-details"}, []string{})
fatalOnError(err, logs)
logger.RegisterSink(debugSink)
errorSink, err := lager.NewRedactingSink(lager.NewWriterSink(os.Stderr, lager.ERROR), []string{"instance-details"}, []string{})
fatalOnError(err, logs)
logger.RegisterSink(errorSink)

freemiumGlobalAccountIds, err := whitelist.ReadWhitelistedGlobalAccountIdsFromFile(cfg.FreemiumWhitelistedGlobalAccountsFilePath)
fatalOnError(err, logs)
logs.Infof("Number of globalAccountIds for unlimited freeemium: %d\n", len(freemiumGlobalAccountIds))
Expand Down Expand Up @@ -456,13 +456,10 @@ func createAPI(router *mux.Router, servicesConfig broker.ServicesConfig, planVal

router.Use(middleware.AddRegionToContext(cfg.DefaultRequestRegion))
router.Use(middleware.AddProviderToContext())
for _, prefix := range []string{
"/oauth/", // oauth2 handled by Ory
"/oauth/{region}/", // oauth2 handled by Ory with region
} {
route := router.PathPrefix(prefix).Subrouter()
broker.AttachRoutes(route, kymaEnvBroker, logger, cfg.Broker.Binding.CreateBindingTimeout)
}

// oauth2 handled by Ory
prefixes := []string{"/oauth", "/oauth/{region}"}
broker.AttachRoutes(router, kymaEnvBroker, logger, cfg.Broker.Binding.CreateBindingTimeout, prefixes)

respWriter := httputil.NewResponseWriter(logs, cfg.DevelopmentMode)
runtimesInfoHandler := appinfo.NewRuntimeInfoHandler(db.Instances(), db.Operations(), defaultPlansConfig, cfg.DefaultRequestRegion, respWriter)
Expand Down
Loading
Loading