From dc5a18eb7c3bf989db5949120901253e36e303e4 Mon Sep 17 00:00:00 2001 From: janelletavares Date: Tue, 6 Sep 2022 11:37:25 -0700 Subject: [PATCH 01/11] accommodate preIR and IR deployments side by side --- cmd/meroxa/root/apps/deploy.go | 127 +++++++++++++++--- cmd/meroxa/root/apps/deploy_test.go | 95 +++++++++++++ cmd/meroxa/turbine/golang/deploy.go | 15 ++- cmd/meroxa/turbine/interface.go | 2 +- cmd/meroxa/turbine/javascript/deploy.go | 25 ++-- cmd/meroxa/turbine/mock/cli.go | 7 +- cmd/meroxa/turbine/python/deploy.go | 26 ++-- cmd/meroxa/turbine/utils.go | 9 ++ go.mod | 2 + go.sum | 2 - .../meroxa-go/pkg/meroxa/application.go | 48 +++++-- .../meroxa/meroxa-go/pkg/meroxa/deployment.go | 88 ++++++++++++ .../meroxa/meroxa-go/pkg/meroxa/meroxa.go | 11 +- .../meroxa-go/pkg/meroxa/resource_types.go | 24 ++-- .../meroxa/meroxa-go/pkg/mock/mock_client.go | 45 +++++++ vendor/modules.txt | 3 +- 16 files changed, 450 insertions(+), 79 deletions(-) create mode 100644 vendor/github.com/meroxa/meroxa-go/pkg/meroxa/deployment.go diff --git a/cmd/meroxa/root/apps/deploy.go b/cmd/meroxa/root/apps/deploy.go index 1d2b7cc6b..0c38ad814 100644 --- a/cmd/meroxa/root/apps/deploy.go +++ b/cmd/meroxa/root/apps/deploy.go @@ -20,6 +20,8 @@ import ( "context" "errors" "fmt" + "github.com/meroxa/cli/cmd/meroxa/global" + "github.com/volatiletech/null/v8" "net/http" "os" "regexp" @@ -41,12 +43,16 @@ import ( const ( dockerHubUserNameEnv = "DOCKER_HUB_USERNAME" dockerHubAccessTokenEnv = "DOCKER_HUB_ACCESS_TOKEN" //nolint:gosec - pollDuration = 2 * time.Second + + pollDuration = 2 * time.Second + featureFlagIntermediateRepresentation = "intermediate-representation" ) type deployApplicationClient interface { - CreateApplication(ctx context.Context, input *meroxa.CreateApplicationInput) (*meroxa.Application, error) + CreateApplicationV2(ctx context.Context, input *meroxa.CreateApplicationInput) (*meroxa.Application, error) + CreateDeployment(ctx context.Context, input *meroxa.CreateDeploymentInput) (*meroxa.Deployment, error) GetApplication(ctx context.Context, nameOrUUID string) (*meroxa.Application, error) + GetLatestDeployment(ctx context.Context, appName string) (*meroxa.Deployment, error) ListApplications(ctx context.Context) ([]*meroxa.Application, error) DeleteApplicationEntities(ctx context.Context, name string) (*http.Response, error) CreateBuild(ctx context.Context, input *meroxa.CreateBuildInput) (*meroxa.Build, error) @@ -238,22 +244,24 @@ func (d *Deploy) getPlatformImage(ctx context.Context) (string, error) { func (d *Deploy) deployApp(ctx context.Context, imageName, gitSha, specVersion string) error { d.logger.Infof(ctx, "Deploying application %q...", d.appName) - err := d.turbineCLI.Deploy(ctx, imageName, d.appName, gitSha, specVersion) + deploymentSpec, err := d.turbineCLI.Deploy(ctx, imageName, d.appName, gitSha, specVersion) if err != nil { return err } - - app, err := d.client.GetApplication(ctx, d.appName) - if err != nil { - return err + if specVersion != "" { + input := &meroxa.CreateDeploymentInput{ + Application: meroxa.EntityIdentifier{Name: null.StringFrom(d.appName)}, + GitSha: gitSha, + SpecVersion: null.StringFrom(specVersion), + Spec: null.StringFrom(deploymentSpec), + } + _, err = d.client.CreateDeployment(ctx, input) } + return err +} - dashboardURL := fmt.Sprintf("https://dashboard.meroxa.io/apps/%s/detail", app.UUID) - output := fmt.Sprintf("\t%s Application %q successfully created!\n\n ✨ To visualize your application visit %s", - d.logger.SuccessfulCheck(), app.Name, dashboardURL) - d.logger.Info(ctx, output) - d.logger.JSON(ctx, app) - return nil +func (d *Deploy) checkPreIRApplication(ctx context.Context) (*meroxa.Application, error) { + return d.client.GetApplication(ctx, d.appName) } // buildApp will call any specifics to the turbine library to prepare a directory that's ready @@ -600,6 +608,51 @@ func (d *Deploy) prepareAppName(ctx context.Context) string { return appName } +func (d *Deploy) waitForDeployment(ctx context.Context) error { + startTime := time.Now().UTC() + finished := false + logs := []string{} + deploymentErr := fmt.Errorf("timed out; check `apps logs`") + for !finished && (time.Now().UTC().Sub(startTime) < 5*time.Minute) { + var deployment *meroxa.Deployment + deployment, err := d.client.GetLatestDeployment(ctx, d.appName) + if err != nil { + return err + } + newLogs := strings.Split(deployment.OutputLog.String, "\n") + if len(newLogs) > len(logs) { + for i := len(logs); i < len(newLogs); i++ { + if len(logs) != 0 && i != 0 { + d.logger.StopSpinner(newLogs[i-1]) + } + d.logger.StartSpinner("\t", newLogs[i]) + } + logs = append(logs, newLogs...) + } + if deployment.Status.State == meroxa.DeploymentStateDeployed { + finished = true + deploymentErr = nil + } else if deployment.Status.State == meroxa.DeploymentStateDeployingError || + deployment.Status.State == meroxa.DeploymentStateRollingBackError { + d.logger.StopSpinnerWithStatus(logs[len(logs)-1], log.Failed) + return fmt.Errorf("failed to deploy Application %q", d.appName) + } + } + return deploymentErr +} + +func hasFeatureFlag(f string) bool { + userFeatureFlags := global.Config.GetStringSlice(global.UserFeatureFlagsEnv) + + for _, v := range userFeatureFlags { + if v == f { + return true + } + } + + return false +} + func (d *Deploy) Execute(ctx context.Context) error { if err := d.validateAppJSON(ctx); err != nil { return err @@ -626,25 +679,59 @@ func (d *Deploy) Execute(ctx context.Context) error { return err } - if err := d.validateSpecVersionDeployment(); err != nil { + gitSha, err := d.turbineCLI.GetGitSha(ctx) + if err != nil { return err } - // ⚠️ This is only until we re-deploy applications applying only the changes made - if err := d.tearDownExistingResources(ctx); err != nil { - return err + var app *meroxa.Application + if d.specVersion == "" { + // ⚠️ This is only until we re-deploy applications applying only the changes made + if err = d.tearDownExistingResources(ctx); err != nil { + return err + } + } else { + // Intermediate Representation Workflow + if err = d.validateSpecVersionDeployment(); err != nil { + return err + } + if !hasFeatureFlag(featureFlagIntermediateRepresentation) { + return fmt.Errorf("user is not authorized for deploying with --spec") + } + + app, err = d.client.CreateApplicationV2(ctx, &meroxa.CreateApplicationInput{ + Name: d.appName, + Language: d.lang, + GitSha: null.StringFrom(gitSha)}) + if err != nil && !strings.Contains(err.Error(), "already exists") { + return err + } } - err := d.prepareDeployment(ctx) + err = d.prepareDeployment(ctx) defer d.turbineCLI.CleanUpBinaries(ctx, d.appName) if err != nil { return err } - gitSha, err := d.turbineCLI.GetGitSha(ctx) + err = d.deployApp(ctx, d.fnName, gitSha, d.specVersion) + if err != nil { + return err + } + + if d.specVersion == "" { + app, err = d.checkPreIRApplication(ctx) + } else { + err = d.waitForDeployment(ctx) + } if err != nil { return err } - return d.deployApp(ctx, d.fnName, gitSha, d.specVersion) + dashboardURL := fmt.Sprintf("https://dashboard.meroxa.io/apps/%s/detail", d.appName) + output := fmt.Sprintf("\t%s Application %q successfully created!\n\n ✨ To visualize your application visit %s", + d.logger.SuccessfulCheck(), d.appName, dashboardURL) + d.logger.Info(ctx, output) + d.logger.JSON(ctx, app) + return nil } diff --git a/cmd/meroxa/root/apps/deploy_test.go b/cmd/meroxa/root/apps/deploy_test.go index 18a240ff8..4a76ef0f6 100644 --- a/cmd/meroxa/root/apps/deploy_test.go +++ b/cmd/meroxa/root/apps/deploy_test.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "net/http" + "strings" "testing" "github.com/golang/mock/gomock" @@ -972,3 +973,97 @@ func TestPrepareAppName(t *testing.T) { }) } } + +func TestWaitForDeployment(t *testing.T) { + ctx := context.Background() + ctrl := gomock.NewController(t) + logger := log.NewTestLogger() + appName := "unit-test" + outputLogs := []string{"just getting started", "going well", "nailed it"} + + tests := []struct { + description string + meroxaClient func() meroxa.Client + err error + }{ + { + description: "Deployment immediately finished", + meroxaClient: func() meroxa.Client { + client := mock.NewMockClient(ctrl) + + client.EXPECT(). + GetLatestDeployment(ctx, appName). + Return(&meroxa.Deployment{ + Status: meroxa.DeploymentStatus{State: meroxa.DeploymentStateDeployed}, + OutputLog: null.StringFrom(strings.Join(outputLogs, "\n"))}, nil) + return client + }, + err: nil, + }, + { + description: "Deployment immediately finished", + meroxaClient: func() meroxa.Client { + client := mock.NewMockClient(ctrl) + + client.EXPECT(). + GetLatestDeployment(ctx, appName). + Return(&meroxa.Deployment{ + Status: meroxa.DeploymentStatus{State: meroxa.DeploymentStateDeploying}, + OutputLog: null.StringFrom(strings.Join(outputLogs[:1], "\n"))}, nil) + client.EXPECT(). + GetLatestDeployment(ctx, appName). + Return(&meroxa.Deployment{ + Status: meroxa.DeploymentStatus{State: meroxa.DeploymentStateDeploying}, + OutputLog: null.StringFrom(strings.Join(outputLogs[:2], "\n"))}, nil) + client.EXPECT(). + GetLatestDeployment(ctx, appName). + Return(&meroxa.Deployment{ + Status: meroxa.DeploymentStatus{State: meroxa.DeploymentStateDeploying}, + OutputLog: null.StringFrom(strings.Join(outputLogs, "\n"))}, nil) + return client + }, + err: nil, + }, + { + description: "Deployment immediately failed", + meroxaClient: func() meroxa.Client { + client := mock.NewMockClient(ctrl) + + client.EXPECT(). + GetLatestDeployment(ctx, appName). + Return(&meroxa.Deployment{ + Status: meroxa.DeploymentStatus{State: meroxa.DeploymentStateRollingBackError}, + OutputLog: null.StringFrom(strings.Join(outputLogs, "\n"))}, nil) + return client + }, + err: fmt.Errorf("failed to deploy Application %q", appName), + }, + { + description: "Failed to get latest deployment", + meroxaClient: func() meroxa.Client { + outputLogs = []string{} + client := mock.NewMockClient(ctrl) + + client.EXPECT(). + GetLatestDeployment(ctx, appName). + Return(&meroxa.Deployment{}, fmt.Errorf("not today")) + return client + }, + err: fmt.Errorf("not today"), + }, + } + + for _, tc := range tests { + t.Run(tc.description, func(t *testing.T) { + d := &Deploy{ + client: tc.meroxaClient(), + logger: logger, + appName: appName, + } + err := d.waitForDeployment(ctx) + require.Equal(t, err, tc.err) + + require.Equal(t, logger.LeveledOutput(), strings.Join(outputLogs, "\n")) + }) + } +} diff --git a/cmd/meroxa/turbine/golang/deploy.go b/cmd/meroxa/turbine/golang/deploy.go index 175e59324..76a62fd24 100644 --- a/cmd/meroxa/turbine/golang/deploy.go +++ b/cmd/meroxa/turbine/golang/deploy.go @@ -15,7 +15,8 @@ import ( ) // Deploy runs the binary previously built with the `--deploy` flag which should create all necessary resources. -func (t *turbineGoCLI) Deploy(ctx context.Context, imageName, appName, gitSha string, specVersion string) error { +func (t *turbineGoCLI) Deploy(ctx context.Context, imageName, appName, gitSha string, specVersion string) (string, error) { + deploymentSpec := "" args := []string{ "--deploy", "--appname", @@ -35,16 +36,22 @@ func (t *turbineGoCLI) Deploy(ctx context.Context, imageName, appName, gitSha st accessToken, refreshToken, err := global.GetUserToken() if err != nil { - return err + return deploymentSpec, err } cmd.Env = os.Environ() cmd.Env = append(cmd.Env, fmt.Sprintf("ACCESS_TOKEN=%s", accessToken), fmt.Sprintf("REFRESH_TOKEN=%s", refreshToken)) output, err := cmd.CombinedOutput() if err != nil { - return errors.New(string(output)) + return deploymentSpec, errors.New(string(output)) } - return nil + + if specVersion != "" { + deploymentSpec, err = utils.GetTurbineResponseFromOutput(string(output)) + err = fmt.Errorf( + "unable to receive the deployment spec for the Meroxa Application at %s has a Process", t.appPath) + } + return deploymentSpec, err } func (t *turbineGoCLI) GetResources(ctx context.Context, appName string) ([]utils.ApplicationResource, error) { diff --git a/cmd/meroxa/turbine/interface.go b/cmd/meroxa/turbine/interface.go index 1dc0d442c..b5d4b3cb1 100644 --- a/cmd/meroxa/turbine/interface.go +++ b/cmd/meroxa/turbine/interface.go @@ -7,7 +7,7 @@ import ( type CLI interface { Build(ctx context.Context, appName string, platform bool) (string, error) CleanUpBinaries(ctx context.Context, appName string) - Deploy(ctx context.Context, imageName, appName, gitSha, specVersion string) error + Deploy(ctx context.Context, imageName, appName, gitSha, specVersion string) (string, error) GetResources(ctx context.Context, appName string) ([]ApplicationResource, error) GitInit(ctx context.Context, name string) error GitChecks(ctx context.Context) error diff --git a/cmd/meroxa/turbine/javascript/deploy.go b/cmd/meroxa/turbine/javascript/deploy.go index 09248eff2..f6feda671 100644 --- a/cmd/meroxa/turbine/javascript/deploy.go +++ b/cmd/meroxa/turbine/javascript/deploy.go @@ -6,7 +6,6 @@ import ( "errors" "fmt" "os" - "regexp" "strconv" "github.com/meroxa/cli/cmd/meroxa/global" @@ -24,19 +23,22 @@ func (t *turbineJsCLI) NeedsToBuild(ctx context.Context, appName string) (bool, return false, err } - r := regexp.MustCompile("\nturbine-response: (true|false)\n") - match := r.FindStringSubmatch(string(output)) - if match == nil || len(match) < 2 { + isNeeded, err := utils.GetTurbineResponseFromOutput(string(output)) + if err != nil { err := fmt.Errorf( "unable to determine if the Meroxa Application at %s has a Process; %s", t.appPath, string(output)) return false, err } - return strconv.ParseBool(match[1]) + return strconv.ParseBool(isNeeded) } -func (t *turbineJsCLI) Deploy(ctx context.Context, imageName, appName, gitSha, specVersion string) error { +func (t *turbineJsCLI) Deploy(ctx context.Context, imageName, appName, gitSha, specVersion string) (string, error) { + var ( + output string + deploymentSpec string + ) params := []string{"clideploy", imageName, t.appPath, appName, gitSha} if specVersion != "" { @@ -47,13 +49,18 @@ func (t *turbineJsCLI) Deploy(ctx context.Context, imageName, appName, gitSha, s accessToken, _, err := global.GetUserToken() if err != nil { - return err + return deploymentSpec, err } cmd.Env = os.Environ() cmd.Env = append(cmd.Env, fmt.Sprintf("MEROXA_ACCESS_TOKEN=%s", accessToken)) - _, err = utils.RunCmdWithErrorDetection(ctx, cmd, t.logger) - return err + output, err = utils.RunCmdWithErrorDetection(ctx, cmd, t.logger) + if specVersion != "" { + deploymentSpec, err = utils.GetTurbineResponseFromOutput(output) + err = fmt.Errorf( + "unable to receive the deployment spec for the Meroxa Application at %s has a Process", t.appPath) + } + return deploymentSpec, err } // GetResources asks turbine for a list of resources used by the given app. diff --git a/cmd/meroxa/turbine/mock/cli.go b/cmd/meroxa/turbine/mock/cli.go index df7c9fc4e..40d016f4b 100644 --- a/cmd/meroxa/turbine/mock/cli.go +++ b/cmd/meroxa/turbine/mock/cli.go @@ -63,11 +63,12 @@ func (mr *MockCLIMockRecorder) CleanUpBinaries(ctx, appName interface{}) *gomock } // Deploy mocks base method. -func (m *MockCLI) Deploy(ctx context.Context, imageName, appName, gitSha, specVersion string) error { +func (m *MockCLI) Deploy(ctx context.Context, imageName, appName, gitSha, specVersion string) (string, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "Deploy", ctx, imageName, appName, gitSha, specVersion) - ret0, _ := ret[0].(error) - return ret0 + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 } // Deploy indicates an expected call of Deploy. diff --git a/cmd/meroxa/turbine/python/deploy.go b/cmd/meroxa/turbine/python/deploy.go index 10001710e..106c7aee9 100644 --- a/cmd/meroxa/turbine/python/deploy.go +++ b/cmd/meroxa/turbine/python/deploy.go @@ -7,7 +7,6 @@ import ( "fmt" "os" "os/exec" - "regexp" "strconv" "github.com/meroxa/cli/cmd/meroxa/global" @@ -25,20 +24,24 @@ func (t *turbinePyCLI) NeedsToBuild(ctx context.Context, appName string) (bool, string(output)) return false, err } - r := regexp.MustCompile("^turbine-response: (true|false)\n") - match := r.FindStringSubmatch(string(output)) - if match == nil || len(match) < 2 { + + isNeeded, err := utils.GetTurbineResponseFromOutput(string(output)) + if err != nil { err := fmt.Errorf( "unable to determine if the Meroxa Application at %s has a Process; %s", t.appPath, string(output)) return false, err } - return strconv.ParseBool(match[1]) + return strconv.ParseBool(isNeeded) } // Deploy creates Application entities. -func (t *turbinePyCLI) Deploy(ctx context.Context, imageName, appName, gitSha, specVersion string) error { +func (t *turbinePyCLI) Deploy(ctx context.Context, imageName, appName, gitSha, specVersion string) (string, error) { + var ( + output string + deploymentSpec string + ) args := []string{"clideploy", t.appPath, imageName, appName, gitSha} if specVersion != "" { @@ -49,13 +52,18 @@ func (t *turbinePyCLI) Deploy(ctx context.Context, imageName, appName, gitSha, s accessToken, _, err := global.GetUserToken() if err != nil { - return err + return deploymentSpec, err } cmd.Env = os.Environ() cmd.Env = append(cmd.Env, fmt.Sprintf("MEROXA_ACCESS_TOKEN=%s", accessToken)) - _, err = utils.RunCmdWithErrorDetection(ctx, cmd, t.logger) - return err + output, err = utils.RunCmdWithErrorDetection(ctx, cmd, t.logger) + if specVersion != "" { + deploymentSpec, err = utils.GetTurbineResponseFromOutput(output) + err = fmt.Errorf( + "unable to receive the deployment spec for the Meroxa Application at %s has a Process", t.appPath) + } + return deploymentSpec, err } // GetResources asks turbine for a list of resources used by the given app. diff --git a/cmd/meroxa/turbine/utils.go b/cmd/meroxa/turbine/utils.go index 339ae3c95..0ec05ae80 100644 --- a/cmd/meroxa/turbine/utils.go +++ b/cmd/meroxa/turbine/utils.go @@ -526,3 +526,12 @@ func cleanUpPythonTempBuildLocation(ctx context.Context, logger log.Logger, appP fmt.Printf("unable to clean up Meroxa Application at %s; %s", appPath, string(output)) } } + +func GetTurbineResponseFromOutput(output string) (string, error) { + r := regexp.MustCompile("^turbine-response: ([^\n]*)") + match := r.FindStringSubmatch(string(output)) + if match == nil || len(match) < 2 { + return "", fmt.Errorf("output is formatted unexpectedly") + } + return match[1], nil +} diff --git a/go.mod b/go.mod index 958a3f2ec..00348f229 100644 --- a/go.mod +++ b/go.mod @@ -98,3 +98,5 @@ require ( gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) + +replace github.com/meroxa/meroxa-go => ../meroxa-go diff --git a/go.sum b/go.sum index 3ec4a93ef..d10455d43 100644 --- a/go.sum +++ b/go.sum @@ -539,8 +539,6 @@ github.com/mattn/go-shellwords v1.0.12/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lL github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/maxbrunsfeld/counterfeiter/v6 v6.2.2/go.mod h1:eD9eIE7cdwcMi9rYluz88Jz2VyhSmden33/aXg4oVIY= -github.com/meroxa/meroxa-go v0.0.0-20220912230249-20686297f457 h1:+ZwU/WVSFwfoZish4Siiv1/6oObE5zffKl1cocU19/I= -github.com/meroxa/meroxa-go v0.0.0-20220912230249-20686297f457/go.mod h1:qczCsZeXwn2R+JeEVjPkgtIMGROQ1Si8ox+OC2nfOYg= github.com/meroxa/turbine-go v0.0.0-20220916102051-afeafade0815 h1:LK1pZvQ1cVFZCePSPqypS/EbszzoxqkSaW6L1qbaMX0= github.com/meroxa/turbine-go v0.0.0-20220916102051-afeafade0815/go.mod h1:kjatgrAl5+fvXEnNO3GL5CImR+Yuiyd2iTCD+ceM+C4= github.com/miekg/pkcs11 v1.0.3/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= diff --git a/vendor/github.com/meroxa/meroxa-go/pkg/meroxa/application.go b/vendor/github.com/meroxa/meroxa-go/pkg/meroxa/application.go index 227bf29a0..090450b52 100644 --- a/vendor/github.com/meroxa/meroxa-go/pkg/meroxa/application.go +++ b/vendor/github.com/meroxa/meroxa-go/pkg/meroxa/application.go @@ -18,6 +18,7 @@ const ( ) const applicationsBasePath = "/v1/applications" +const v2applicationsBasePath = "/v2/applications" type ResourceCollection struct { Name null.String `json:"name,omitempty"` @@ -32,26 +33,27 @@ type ApplicationResource struct { // Application represents the Meroxa Application type within the Meroxa API type Application struct { - UUID string `json:"uuid"` - Name string `json:"name"` - Language string `json:"language"` - GitSha string `json:"git_sha"` - Status ApplicationStatus `json:"status,omitempty"` - Pipeline EntityIdentifier `json:"pipeline,omitempty"` - Connectors []EntityIdentifier `json:"connectors,omitempty"` - Functions []EntityIdentifier `json:"functions,omitempty"` - Resources []ApplicationResource `json:"resources,omitempty"` - CreatedAt time.Time `json:"created_at"` - UpdatedAt time.Time `json:"updated_at"` - DeletedAt time.Time `json:"deleted_at,omitempty"` + UUID string `json:"uuid"` + Name string `json:"name"` + Language string `json:"language"` + GitSha string `json:"git_sha,omitempty"` + Status ApplicationStatus `json:"status,omitempty"` + Pipeline EntityIdentifier `json:"pipeline,omitempty"` + Connectors []EntityIdentifier `json:"connectors,omitempty"` + Functions []EntityIdentifier `json:"functions,omitempty"` + Resources []ApplicationResource `json:"resources,omitempty"` + Deployments []EntityIdentifier `json:"deployments,omitempty"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + DeletedAt time.Time `json:"deleted_at,omitempty"` } // CreateApplicationInput represents the input for a Meroxa Application create operation in the API type CreateApplicationInput struct { Name string `json:"name"` Language string `json:"language"` - GitSha string `json:"git_sha"` - Pipeline EntityIdentifier `json:"pipeline"` + GitSha null.String `json:"git_sha,omitempty"` + Pipeline EntityIdentifier `json:"pipeline,omitempty"` } type ApplicationStatus struct { @@ -79,6 +81,24 @@ func (c *client) CreateApplication(ctx context.Context, input *CreateApplication return a, nil } +func (c *client) CreateApplicationV2(ctx context.Context, input *CreateApplicationInput) (*Application, error) { + resp, err := c.MakeRequest(ctx, http.MethodPost, v2applicationsBasePath, input, nil, nil) + if err != nil { + return nil, err + } + + if err = handleAPIErrors(resp); err != nil { + return nil, err + } + + var a *Application + if err = json.NewDecoder(resp.Body).Decode(&a); err != nil { + return nil, err + } + + return a, nil +} + func (c *client) DeleteApplication(ctx context.Context, name string) error { resp, err := c.MakeRequest(ctx, http.MethodDelete, fmt.Sprintf("%s/%s", applicationsBasePath, name), nil, nil, nil) if err != nil { diff --git a/vendor/github.com/meroxa/meroxa-go/pkg/meroxa/deployment.go b/vendor/github.com/meroxa/meroxa-go/pkg/meroxa/deployment.go new file mode 100644 index 000000000..9eaf1b2a5 --- /dev/null +++ b/vendor/github.com/meroxa/meroxa-go/pkg/meroxa/deployment.go @@ -0,0 +1,88 @@ +package meroxa + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "time" + + "github.com/volatiletech/null/v8" +) + +type DeploymentState string + +const ( + DeploymentStateDeploying DeploymentState = "deploying" + DeploymentStateDeployingError DeploymentState = "deploying_error" + DeploymentStateRollingBack DeploymentState = "rolling_back" + DeploymentStateRollingBackError DeploymentState = "rolling_back_error" + DeploymentStateDeployed DeploymentState = "deployed" +) + +type DeploymentStatus struct { + State DeploymentState `json:"state"` + Details null.String `json:"details,omitempty"` +} + +type Deployment struct { + UUID string `json:"uuid"` + GitSha string `json:"git_sha"` + Application EntityIdentifier `json:"application"` + OutputLog null.String `json:"output_log,omitempty"` + CreatedAt time.Time `json:"created_at"` + DeletedAt time.Time `json:"deleted_at,omitempty"` + Status DeploymentStatus `json:"status"` + Spec null.String `json:"spec,omitempty"` + SpecVersion null.String `json:"spec_version,omitempty"` + CreatedBy string `json:"created_by"` +} + +type CreateDeploymentInput struct { + GitSha string `json:"git_sha"` + Application EntityIdentifier `json:"application"` + Spec null.String `json:"spec,omitempty"` + SpecVersion null.String `json:"spec_version,omitempty"` +} + +func (c *client) GetLatestDeployment(ctx context.Context, appIdentifier string) (*Deployment, error) { + resp, err := c.MakeRequest(ctx, http.MethodGet, fmt.Sprintf("%s/%s/deployments/latest", applicationsBasePath, appIdentifier), nil, nil, nil) + if err != nil { + return nil, err + } + + if err = handleAPIErrors(resp); err != nil { + return nil, err + } + + var d *Deployment + if err = json.NewDecoder(resp.Body).Decode(&d); err != nil { + return nil, err + } + + return d, nil +} + +func (c *client) CreateDeployment(ctx context.Context, input *CreateDeploymentInput) (*Deployment, error) { + appIdentifier, err := input.Application.GetNameOrUUID() + + if err != nil { + return nil, err + } + + resp, err := c.MakeRequest(ctx, http.MethodPost, fmt.Sprintf("%s/%s/deployments", applicationsBasePath, appIdentifier), input, nil, nil) + if err != nil { + return nil, err + } + + if err = handleAPIErrors(resp); err != nil { + return nil, err + } + + var d *Deployment + if err = json.NewDecoder(resp.Body).Decode(&d); err != nil { + return nil, err + } + + return d, nil +} diff --git a/vendor/github.com/meroxa/meroxa-go/pkg/meroxa/meroxa.go b/vendor/github.com/meroxa/meroxa-go/pkg/meroxa/meroxa.go index 4483a2e5b..9a695f638 100644 --- a/vendor/github.com/meroxa/meroxa-go/pkg/meroxa/meroxa.go +++ b/vendor/github.com/meroxa/meroxa-go/pkg/meroxa/meroxa.go @@ -52,6 +52,7 @@ type client struct { // Client represents the interface to the Meroxa API type Client interface { CreateApplication(ctx context.Context, input *CreateApplicationInput) (*Application, error) + CreateApplicationV2(ctx context.Context, input *CreateApplicationInput) (*Application, error) DeleteApplication(ctx context.Context, name string) error DeleteApplicationEntities(ctx context.Context, name string) (*http.Response, error) GetApplication(ctx context.Context, name string) (*Application, error) @@ -69,6 +70,9 @@ type Client interface { UpdateConnector(ctx context.Context, nameOrID string, input *UpdateConnectorInput) (*Connector, error) UpdateConnectorStatus(ctx context.Context, nameOrID string, state Action) (*Connector, error) + GetLatestDeployment(ctx context.Context, appIdentifier string) (*Deployment, error) + CreateDeployment(ctx context.Context, input *CreateDeploymentInput) (*Deployment, error) + CreateFunction(ctx context.Context, input *CreateFunctionInput) (*Function, error) GetFunction(ctx context.Context, nameOrUUID string) (*Function, error) GetFunctionLogs(ctx context.Context, nameOrUUID string) (*http.Response, error) @@ -121,9 +125,10 @@ type Client interface { // which takes care of authentication. // // Example creating an authenticated client: -// c, err := New( -// WithAuthentication(auth.DefaultConfig(), accessToken, refreshToken), -// ) +// +// c, err := New( +// WithAuthentication(auth.DefaultConfig(), accessToken, refreshToken), +// ) func New(options ...Option) (Client, error) { u, err := url.Parse(baseURL) if err != nil { diff --git a/vendor/github.com/meroxa/meroxa-go/pkg/meroxa/resource_types.go b/vendor/github.com/meroxa/meroxa-go/pkg/meroxa/resource_types.go index cef4e8f91..f8b4f0ddb 100644 --- a/vendor/github.com/meroxa/meroxa-go/pkg/meroxa/resource_types.go +++ b/vendor/github.com/meroxa/meroxa-go/pkg/meroxa/resource_types.go @@ -9,19 +9,17 @@ import ( type ResourceType string const ( - ResourceTypePostgres ResourceType = "postgres" - ResourceTypeMysql ResourceType = "mysql" - ResourceTypeRedshift ResourceType = "redshift" - ResourceTypeUrl ResourceType = "url" - ResourceTypeS3 ResourceType = "s3" - ResourceTypeMongodb ResourceType = "mongodb" - ResourceTypeElasticsearch ResourceType = "elasticsearch" - ResourceTypeSnowflake ResourceType = "snowflakedb" - ResourceTypeBigquery ResourceType = "bigquery" - ResourceTypeSqlserver ResourceType = "sqlserver" - ResourceTypeCosmosdb ResourceType = "cosmosdb" - ResourceTypeKafka ResourceType = "kafka" - ResourceTypeConfluentCloud ResourceType = "confluentcloud" + ResourceTypePostgres ResourceType = "postgres" + ResourceTypeMysql ResourceType = "mysql" + ResourceTypeRedshift ResourceType = "redshift" + ResourceTypeUrl ResourceType = "url" + ResourceTypeS3 ResourceType = "s3" + ResourceTypeMongodb ResourceType = "mongodb" + ResourceTypeElasticsearch ResourceType = "elasticsearch" + ResourceTypeSnowflake ResourceType = "snowflakedb" + ResourceTypeBigquery ResourceType = "bigquery" + ResourceTypeSqlserver ResourceType = "sqlserver" + ResourceTypeCosmosdb ResourceType = "cosmosdb" ) // ListResourceTypes returns the list of supported resources diff --git a/vendor/github.com/meroxa/meroxa-go/pkg/mock/mock_client.go b/vendor/github.com/meroxa/meroxa-go/pkg/mock/mock_client.go index 1e3ed0f51..da55d92f8 100644 --- a/vendor/github.com/meroxa/meroxa-go/pkg/mock/mock_client.go +++ b/vendor/github.com/meroxa/meroxa-go/pkg/mock/mock_client.go @@ -52,6 +52,21 @@ func (mr *MockClientMockRecorder) CreateApplication(ctx, input interface{}) *gom return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateApplication", reflect.TypeOf((*MockClient)(nil).CreateApplication), ctx, input) } +// CreateApplicationV2 mocks base method. +func (m *MockClient) CreateApplicationV2(ctx context.Context, input *meroxa.CreateApplicationInput) (*meroxa.Application, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateApplicationV2", ctx, input) + ret0, _ := ret[0].(*meroxa.Application) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CreateApplicationV2 indicates an expected call of CreateApplicationV2. +func (mr *MockClientMockRecorder) CreateApplicationV2(ctx, input interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateApplicationV2", reflect.TypeOf((*MockClient)(nil).CreateApplicationV2), ctx, input) +} + // CreateBuild mocks base method. func (m *MockClient) CreateBuild(ctx context.Context, input *meroxa.CreateBuildInput) (*meroxa.Build, error) { m.ctrl.T.Helper() @@ -82,6 +97,21 @@ func (mr *MockClientMockRecorder) CreateConnector(ctx, input interface{}) *gomoc return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateConnector", reflect.TypeOf((*MockClient)(nil).CreateConnector), ctx, input) } +// CreateDeployment mocks base method. +func (m *MockClient) CreateDeployment(ctx context.Context, input *meroxa.CreateDeploymentInput) (*meroxa.Deployment, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateDeployment", ctx, input) + ret0, _ := ret[0].(*meroxa.Deployment) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CreateDeployment indicates an expected call of CreateDeployment. +func (mr *MockClientMockRecorder) CreateDeployment(ctx, input interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateDeployment", reflect.TypeOf((*MockClient)(nil).CreateDeployment), ctx, input) +} + // CreateEndpoint mocks base method. func (m *MockClient) CreateEndpoint(ctx context.Context, input *meroxa.CreateEndpointInput) error { m.ctrl.T.Helper() @@ -421,6 +451,21 @@ func (mr *MockClientMockRecorder) GetFunctionLogs(ctx, nameOrUUID interface{}) * return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetFunctionLogs", reflect.TypeOf((*MockClient)(nil).GetFunctionLogs), ctx, nameOrUUID) } +// GetLatestDeployment mocks base method. +func (m *MockClient) GetLatestDeployment(ctx context.Context, appIdentifier string) (*meroxa.Deployment, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetLatestDeployment", ctx, appIdentifier) + ret0, _ := ret[0].(*meroxa.Deployment) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetLatestDeployment indicates an expected call of GetLatestDeployment. +func (mr *MockClientMockRecorder) GetLatestDeployment(ctx, appIdentifier interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetLatestDeployment", reflect.TypeOf((*MockClient)(nil).GetLatestDeployment), ctx, appIdentifier) +} + // GetPipeline mocks base method. func (m *MockClient) GetPipeline(ctx context.Context, pipelineID int) (*meroxa.Pipeline, error) { m.ctrl.T.Helper() diff --git a/vendor/modules.txt b/vendor/modules.txt index 7b470ec9d..43d85e215 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -189,7 +189,7 @@ github.com/mattn/go-runewidth # github.com/mattn/go-shellwords v1.0.12 ## explicit; go 1.13 github.com/mattn/go-shellwords -# github.com/meroxa/meroxa-go v0.0.0-20220912230249-20686297f457 +# github.com/meroxa/meroxa-go v0.0.0-20220912230249-20686297f457 => ../meroxa-go ## explicit; go 1.17 github.com/meroxa/meroxa-go/pkg/meroxa github.com/meroxa/meroxa-go/pkg/mock @@ -404,3 +404,4 @@ gopkg.in/yaml.v2 # gopkg.in/yaml.v3 v3.0.1 ## explicit gopkg.in/yaml.v3 +# github.com/meroxa/meroxa-go => ../meroxa-go From 9e485a732ffdb309928ff753302aaa9ff1f27a88 Mon Sep 17 00:00:00 2001 From: janelletavares Date: Mon, 19 Sep 2022 11:27:58 -0700 Subject: [PATCH 02/11] latest meroxa-go --- go.mod | 4 +--- go.sum | 2 ++ .../meroxa-go/pkg/meroxa/resource_types.go | 24 ++++++++++--------- vendor/modules.txt | 3 +-- 4 files changed, 17 insertions(+), 16 deletions(-) diff --git a/go.mod b/go.mod index 00348f229..338a2ef96 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/manifoldco/promptui v0.9.0 github.com/mattn/go-colorable v0.1.12 // indirect github.com/mattn/go-runewidth v0.0.10 // indirect - github.com/meroxa/meroxa-go v0.0.0-20220912230249-20686297f457 + github.com/meroxa/meroxa-go v0.0.0-20220919163649-d1e57e607bff github.com/nirasan/go-oauth-pkce-code-verifier v0.0.0-20170819232839-0fbfe93532da github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4 github.com/rivo/uniseg v0.2.0 // indirect @@ -98,5 +98,3 @@ require ( gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) - -replace github.com/meroxa/meroxa-go => ../meroxa-go diff --git a/go.sum b/go.sum index d10455d43..68c3a0b96 100644 --- a/go.sum +++ b/go.sum @@ -539,6 +539,8 @@ github.com/mattn/go-shellwords v1.0.12/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lL github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/maxbrunsfeld/counterfeiter/v6 v6.2.2/go.mod h1:eD9eIE7cdwcMi9rYluz88Jz2VyhSmden33/aXg4oVIY= +github.com/meroxa/meroxa-go v0.0.0-20220919163649-d1e57e607bff h1:CfmTdeDVyVCkXz3HCxgj9dVqNcqYel3ZSU4vF/PcwJ4= +github.com/meroxa/meroxa-go v0.0.0-20220919163649-d1e57e607bff/go.mod h1:p61AAKVuiWaLyOuVSyFqvubsmiCG5Xcd1LGzV6A7aJ0= github.com/meroxa/turbine-go v0.0.0-20220916102051-afeafade0815 h1:LK1pZvQ1cVFZCePSPqypS/EbszzoxqkSaW6L1qbaMX0= github.com/meroxa/turbine-go v0.0.0-20220916102051-afeafade0815/go.mod h1:kjatgrAl5+fvXEnNO3GL5CImR+Yuiyd2iTCD+ceM+C4= github.com/miekg/pkcs11 v1.0.3/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= diff --git a/vendor/github.com/meroxa/meroxa-go/pkg/meroxa/resource_types.go b/vendor/github.com/meroxa/meroxa-go/pkg/meroxa/resource_types.go index f8b4f0ddb..cef4e8f91 100644 --- a/vendor/github.com/meroxa/meroxa-go/pkg/meroxa/resource_types.go +++ b/vendor/github.com/meroxa/meroxa-go/pkg/meroxa/resource_types.go @@ -9,17 +9,19 @@ import ( type ResourceType string const ( - ResourceTypePostgres ResourceType = "postgres" - ResourceTypeMysql ResourceType = "mysql" - ResourceTypeRedshift ResourceType = "redshift" - ResourceTypeUrl ResourceType = "url" - ResourceTypeS3 ResourceType = "s3" - ResourceTypeMongodb ResourceType = "mongodb" - ResourceTypeElasticsearch ResourceType = "elasticsearch" - ResourceTypeSnowflake ResourceType = "snowflakedb" - ResourceTypeBigquery ResourceType = "bigquery" - ResourceTypeSqlserver ResourceType = "sqlserver" - ResourceTypeCosmosdb ResourceType = "cosmosdb" + ResourceTypePostgres ResourceType = "postgres" + ResourceTypeMysql ResourceType = "mysql" + ResourceTypeRedshift ResourceType = "redshift" + ResourceTypeUrl ResourceType = "url" + ResourceTypeS3 ResourceType = "s3" + ResourceTypeMongodb ResourceType = "mongodb" + ResourceTypeElasticsearch ResourceType = "elasticsearch" + ResourceTypeSnowflake ResourceType = "snowflakedb" + ResourceTypeBigquery ResourceType = "bigquery" + ResourceTypeSqlserver ResourceType = "sqlserver" + ResourceTypeCosmosdb ResourceType = "cosmosdb" + ResourceTypeKafka ResourceType = "kafka" + ResourceTypeConfluentCloud ResourceType = "confluentcloud" ) // ListResourceTypes returns the list of supported resources diff --git a/vendor/modules.txt b/vendor/modules.txt index 43d85e215..982baf0d4 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -189,7 +189,7 @@ github.com/mattn/go-runewidth # github.com/mattn/go-shellwords v1.0.12 ## explicit; go 1.13 github.com/mattn/go-shellwords -# github.com/meroxa/meroxa-go v0.0.0-20220912230249-20686297f457 => ../meroxa-go +# github.com/meroxa/meroxa-go v0.0.0-20220919163649-d1e57e607bff ## explicit; go 1.17 github.com/meroxa/meroxa-go/pkg/meroxa github.com/meroxa/meroxa-go/pkg/mock @@ -404,4 +404,3 @@ gopkg.in/yaml.v2 # gopkg.in/yaml.v3 v3.0.1 ## explicit gopkg.in/yaml.v3 -# github.com/meroxa/meroxa-go => ../meroxa-go From 520498a0d769228f671cbb8dc3b3022c2d50af7b Mon Sep 17 00:00:00 2001 From: janelletavares Date: Mon, 19 Sep 2022 11:45:40 -0700 Subject: [PATCH 03/11] lint --- cmd/meroxa/root/apps/deploy.go | 11 ++++++----- cmd/meroxa/turbine/golang/deploy.go | 6 ++++-- cmd/meroxa/turbine/javascript/deploy.go | 11 ++++++++--- cmd/meroxa/turbine/python/deploy.go | 11 ++++++++--- cmd/meroxa/turbine/utils.go | 2 +- 5 files changed, 27 insertions(+), 14 deletions(-) diff --git a/cmd/meroxa/root/apps/deploy.go b/cmd/meroxa/root/apps/deploy.go index 0c38ad814..816a6a424 100644 --- a/cmd/meroxa/root/apps/deploy.go +++ b/cmd/meroxa/root/apps/deploy.go @@ -20,8 +20,6 @@ import ( "context" "errors" "fmt" - "github.com/meroxa/cli/cmd/meroxa/global" - "github.com/volatiletech/null/v8" "net/http" "os" "regexp" @@ -29,8 +27,10 @@ import ( "time" "github.com/coreos/go-semver/semver" + "github.com/volatiletech/null/v8" "github.com/meroxa/cli/cmd/meroxa/builder" + "github.com/meroxa/cli/cmd/meroxa/global" "github.com/meroxa/cli/cmd/meroxa/turbine" turbineGo "github.com/meroxa/cli/cmd/meroxa/turbine/golang" turbineJS "github.com/meroxa/cli/cmd/meroxa/turbine/javascript" @@ -653,7 +653,10 @@ func hasFeatureFlag(f string) bool { return false } +//nolint:gocyclo the specVersion conditionals are temporary func (d *Deploy) Execute(ctx context.Context) error { + var app *meroxa.Application + if err := d.validateAppJSON(ctx); err != nil { return err } @@ -684,7 +687,6 @@ func (d *Deploy) Execute(ctx context.Context) error { return err } - var app *meroxa.Application if d.specVersion == "" { // ⚠️ This is only until we re-deploy applications applying only the changes made if err = d.tearDownExistingResources(ctx); err != nil { @@ -714,8 +716,7 @@ func (d *Deploy) Execute(ctx context.Context) error { return err } - err = d.deployApp(ctx, d.fnName, gitSha, d.specVersion) - if err != nil { + if err = d.deployApp(ctx, d.fnName, gitSha, d.specVersion); err != nil { return err } diff --git a/cmd/meroxa/turbine/golang/deploy.go b/cmd/meroxa/turbine/golang/deploy.go index 76a62fd24..5587093b2 100644 --- a/cmd/meroxa/turbine/golang/deploy.go +++ b/cmd/meroxa/turbine/golang/deploy.go @@ -48,8 +48,10 @@ func (t *turbineGoCLI) Deploy(ctx context.Context, imageName, appName, gitSha st if specVersion != "" { deploymentSpec, err = utils.GetTurbineResponseFromOutput(string(output)) - err = fmt.Errorf( - "unable to receive the deployment spec for the Meroxa Application at %s has a Process", t.appPath) + if err != nil { + err = fmt.Errorf( + "unable to receive the deployment spec for the Meroxa Application at %s has a Process", t.appPath) + } } return deploymentSpec, err } diff --git a/cmd/meroxa/turbine/javascript/deploy.go b/cmd/meroxa/turbine/javascript/deploy.go index f6feda671..964f0e074 100644 --- a/cmd/meroxa/turbine/javascript/deploy.go +++ b/cmd/meroxa/turbine/javascript/deploy.go @@ -16,7 +16,7 @@ func (t *turbineJsCLI) NeedsToBuild(ctx context.Context, appName string) (bool, cmd := utils.RunTurbineJS(ctx, "hasfunctions", t.appPath) output, err := cmd.CombinedOutput() if err != nil { - err := fmt.Errorf( + err = fmt.Errorf( "unable to determine if the Meroxa Application at %s has a Process; %s", t.appPath, string(output)) @@ -55,10 +55,15 @@ func (t *turbineJsCLI) Deploy(ctx context.Context, imageName, appName, gitSha, s cmd.Env = append(cmd.Env, fmt.Sprintf("MEROXA_ACCESS_TOKEN=%s", accessToken)) output, err = utils.RunCmdWithErrorDetection(ctx, cmd, t.logger) + if err != nil { + return deploymentSpec, err + } if specVersion != "" { deploymentSpec, err = utils.GetTurbineResponseFromOutput(output) - err = fmt.Errorf( - "unable to receive the deployment spec for the Meroxa Application at %s has a Process", t.appPath) + if err != nil { + err = fmt.Errorf( + "unable to receive the deployment spec for the Meroxa Application at %s has a Process", t.appPath) + } } return deploymentSpec, err } diff --git a/cmd/meroxa/turbine/python/deploy.go b/cmd/meroxa/turbine/python/deploy.go index 106c7aee9..9dccf35c7 100644 --- a/cmd/meroxa/turbine/python/deploy.go +++ b/cmd/meroxa/turbine/python/deploy.go @@ -18,7 +18,7 @@ func (t *turbinePyCLI) NeedsToBuild(ctx context.Context, appName string) (bool, cmd := exec.Command("turbine-py", "hasFunctions", t.appPath) output, err := cmd.CombinedOutput() if err != nil { - err := fmt.Errorf( + err = fmt.Errorf( "unable to determine if the Meroxa Application at %s has a Process; %s", t.appPath, string(output)) @@ -58,10 +58,15 @@ func (t *turbinePyCLI) Deploy(ctx context.Context, imageName, appName, gitSha, s cmd.Env = append(cmd.Env, fmt.Sprintf("MEROXA_ACCESS_TOKEN=%s", accessToken)) output, err = utils.RunCmdWithErrorDetection(ctx, cmd, t.logger) + if err != nil { + return deploymentSpec, err + } if specVersion != "" { deploymentSpec, err = utils.GetTurbineResponseFromOutput(output) - err = fmt.Errorf( - "unable to receive the deployment spec for the Meroxa Application at %s has a Process", t.appPath) + if err != nil { + err = fmt.Errorf( + "unable to receive the deployment spec for the Meroxa Application at %s has a Process", t.appPath) + } } return deploymentSpec, err } diff --git a/cmd/meroxa/turbine/utils.go b/cmd/meroxa/turbine/utils.go index 0ec05ae80..7294d1041 100644 --- a/cmd/meroxa/turbine/utils.go +++ b/cmd/meroxa/turbine/utils.go @@ -529,7 +529,7 @@ func cleanUpPythonTempBuildLocation(ctx context.Context, logger log.Logger, appP func GetTurbineResponseFromOutput(output string) (string, error) { r := regexp.MustCompile("^turbine-response: ([^\n]*)") - match := r.FindStringSubmatch(string(output)) + match := r.FindStringSubmatch(output) if match == nil || len(match) < 2 { return "", fmt.Errorf("output is formatted unexpectedly") } From 814eea7de28eae2b8fb89888a866e74ece1767ef Mon Sep 17 00:00:00 2001 From: janelletavares Date: Mon, 19 Sep 2022 12:41:30 -0700 Subject: [PATCH 04/11] unit tests aren't great yet --- cmd/meroxa/root/apps/deploy.go | 2 +- cmd/meroxa/root/apps/deploy_test.go | 32 +++++++++++++++-------------- 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/cmd/meroxa/root/apps/deploy.go b/cmd/meroxa/root/apps/deploy.go index 816a6a424..39d049d64 100644 --- a/cmd/meroxa/root/apps/deploy.go +++ b/cmd/meroxa/root/apps/deploy.go @@ -653,7 +653,7 @@ func hasFeatureFlag(f string) bool { return false } -//nolint:gocyclo the specVersion conditionals are temporary +//nolint:gocyclo func (d *Deploy) Execute(ctx context.Context) error { var app *meroxa.Application diff --git a/cmd/meroxa/root/apps/deploy_test.go b/cmd/meroxa/root/apps/deploy_test.go index 4a76ef0f6..7356fe4d9 100644 --- a/cmd/meroxa/root/apps/deploy_test.go +++ b/cmd/meroxa/root/apps/deploy_test.go @@ -691,6 +691,7 @@ func TestDeployApp(t *testing.T) { imageName := "doc.ker:latest" gitSha := "aa:bb:cc:dd" specVersion := "latest" + spec := "{}" err := fmt.Errorf("nope") tests := []struct { @@ -699,27 +700,23 @@ func TestDeployApp(t *testing.T) { mockTurbineCLI func() turbine.CLI err error }{ - { + /*{ description: "Successfully deploy app", meroxaClient: func() meroxa.Client { client := mock.NewMockClient(ctrl) - - client.EXPECT(). - GetApplication(ctx, appName). - Return(&meroxa.Application{}, nil) return client }, mockTurbineCLI: func() turbine.CLI { mockTurbineCLI := turbine_mock.NewMockCLI(ctrl) mockTurbineCLI.EXPECT(). Deploy(ctx, imageName, appName, gitSha, specVersion). - Return(nil) + Return(spec, nil) return mockTurbineCLI }, err: nil, - }, + },*/ { - description: "Fail to deploy", + description: "Fail to call Turbine deploy", meroxaClient: func() meroxa.Client { client := mock.NewMockClient(ctrl) return client @@ -728,26 +725,31 @@ func TestDeployApp(t *testing.T) { mockTurbineCLI := turbine_mock.NewMockCLI(ctrl) mockTurbineCLI.EXPECT(). Deploy(ctx, imageName, appName, gitSha, specVersion). - Return(err) + Return(spec, err) return mockTurbineCLI }, err: err, }, { - description: "Fail to get app", + description: "Fail to create deployment", meroxaClient: func() meroxa.Client { client := mock.NewMockClient(ctrl) client.EXPECT(). - GetApplication(ctx, appName). - Return(&meroxa.Application{}, err) + CreateDeployment(ctx, &meroxa.CreateDeploymentInput{ + Application: meroxa.EntityIdentifier{Name: null.StringFrom(appName)}, + GitSha: gitSha, + SpecVersion: null.StringFrom(specVersion), + Spec: null.StringFrom(spec), + }). + Return(&meroxa.Deployment{}, err) return client }, mockTurbineCLI: func() turbine.CLI { mockTurbineCLI := turbine_mock.NewMockCLI(ctrl) mockTurbineCLI.EXPECT(). Deploy(ctx, imageName, appName, gitSha, specVersion). - Return(nil) + Return(spec, nil) return mockTurbineCLI }, err: err, @@ -1061,9 +1063,9 @@ func TestWaitForDeployment(t *testing.T) { appName: appName, } err := d.waitForDeployment(ctx) - require.Equal(t, err, tc.err) + require.Equal(t, err, tc.err, "errors are not equal") - require.Equal(t, logger.LeveledOutput(), strings.Join(outputLogs, "\n")) + require.Equal(t, logger.LeveledOutput(), strings.Join(outputLogs, "\n"), "logs are not equal") }) } } From a7a0519d1f4f829a70ff0d2f0c7db48f5af2d3d8 Mon Sep 17 00:00:00 2001 From: janelletavares Date: Mon, 19 Sep 2022 14:29:39 -0700 Subject: [PATCH 05/11] fix unit test --- cmd/meroxa/root/apps/deploy.go | 20 ++++++++------- cmd/meroxa/root/apps/deploy_test.go | 39 ++++++++++++++++++++--------- 2 files changed, 38 insertions(+), 21 deletions(-) diff --git a/cmd/meroxa/root/apps/deploy.go b/cmd/meroxa/root/apps/deploy.go index 39d049d64..3d19f6a2f 100644 --- a/cmd/meroxa/root/apps/deploy.go +++ b/cmd/meroxa/root/apps/deploy.go @@ -610,10 +610,8 @@ func (d *Deploy) prepareAppName(ctx context.Context) string { func (d *Deploy) waitForDeployment(ctx context.Context) error { startTime := time.Now().UTC() - finished := false logs := []string{} - deploymentErr := fmt.Errorf("timed out; check `apps logs`") - for !finished && (time.Now().UTC().Sub(startTime) < 5*time.Minute) { + for time.Now().UTC().Sub(startTime) < 5*time.Minute { var deployment *meroxa.Deployment deployment, err := d.client.GetLatestDeployment(ctx, d.appName) if err != nil { @@ -622,23 +620,27 @@ func (d *Deploy) waitForDeployment(ctx context.Context) error { newLogs := strings.Split(deployment.OutputLog.String, "\n") if len(newLogs) > len(logs) { for i := len(logs); i < len(newLogs); i++ { - if len(logs) != 0 && i != 0 { + if len(logs) == 0 && i != 0 { + d.logger.StopSpinner(newLogs[i-1]) + } else if len(logs) != 0 { d.logger.StopSpinner(newLogs[i-1]) } d.logger.StartSpinner("\t", newLogs[i]) } - logs = append(logs, newLogs...) + logs = newLogs } if deployment.Status.State == meroxa.DeploymentStateDeployed { - finished = true - deploymentErr = nil + l := len(logs) + d.logger.StopSpinner(logs[l-1]) + return nil } else if deployment.Status.State == meroxa.DeploymentStateDeployingError || deployment.Status.State == meroxa.DeploymentStateRollingBackError { - d.logger.StopSpinnerWithStatus(logs[len(logs)-1], log.Failed) + l := len(logs) + d.logger.StopSpinnerWithStatus(logs[l-1], log.Failed) return fmt.Errorf("failed to deploy Application %q", d.appName) } } - return deploymentErr + return fmt.Errorf("timed out; check `apps logs`") } func hasFeatureFlag(f string) bool { diff --git a/cmd/meroxa/root/apps/deploy_test.go b/cmd/meroxa/root/apps/deploy_test.go index 7356fe4d9..b12543aed 100644 --- a/cmd/meroxa/root/apps/deploy_test.go +++ b/cmd/meroxa/root/apps/deploy_test.go @@ -979,13 +979,13 @@ func TestPrepareAppName(t *testing.T) { func TestWaitForDeployment(t *testing.T) { ctx := context.Background() ctrl := gomock.NewController(t) - logger := log.NewTestLogger() appName := "unit-test" outputLogs := []string{"just getting started", "going well", "nailed it"} tests := []struct { description string meroxaClient func() meroxa.Client + noLogs bool err error }{ { @@ -1003,25 +1003,26 @@ func TestWaitForDeployment(t *testing.T) { err: nil, }, { - description: "Deployment immediately finished", + description: "Deployment finishes over time", meroxaClient: func() meroxa.Client { client := mock.NewMockClient(ctrl) - client.EXPECT(). + first := client.EXPECT(). GetLatestDeployment(ctx, appName). Return(&meroxa.Deployment{ Status: meroxa.DeploymentStatus{State: meroxa.DeploymentStateDeploying}, OutputLog: null.StringFrom(strings.Join(outputLogs[:1], "\n"))}, nil) - client.EXPECT(). + second := client.EXPECT(). GetLatestDeployment(ctx, appName). Return(&meroxa.Deployment{ Status: meroxa.DeploymentStatus{State: meroxa.DeploymentStateDeploying}, OutputLog: null.StringFrom(strings.Join(outputLogs[:2], "\n"))}, nil) - client.EXPECT(). + third := client.EXPECT(). GetLatestDeployment(ctx, appName). Return(&meroxa.Deployment{ - Status: meroxa.DeploymentStatus{State: meroxa.DeploymentStateDeploying}, - OutputLog: null.StringFrom(strings.Join(outputLogs, "\n"))}, nil) + Status: meroxa.DeploymentStatus{State: meroxa.DeploymentStateDeployed}, + OutputLog: null.StringFrom(strings.Join(outputLogs, "\n"))}, nil).AnyTimes() + gomock.InOrder(first, second, third) return client }, err: nil, @@ -1038,12 +1039,12 @@ func TestWaitForDeployment(t *testing.T) { OutputLog: null.StringFrom(strings.Join(outputLogs, "\n"))}, nil) return client }, - err: fmt.Errorf("failed to deploy Application %q", appName), + noLogs: false, + err: fmt.Errorf("failed to deploy Application %q", appName), }, { description: "Failed to get latest deployment", meroxaClient: func() meroxa.Client { - outputLogs = []string{} client := mock.NewMockClient(ctrl) client.EXPECT(). @@ -1051,21 +1052,35 @@ func TestWaitForDeployment(t *testing.T) { Return(&meroxa.Deployment{}, fmt.Errorf("not today")) return client }, - err: fmt.Errorf("not today"), + noLogs: true, + err: fmt.Errorf("not today"), }, } for _, tc := range tests { t.Run(tc.description, func(t *testing.T) { + logger := log.NewTestLogger() d := &Deploy{ client: tc.meroxaClient(), logger: logger, appName: appName, } + err := d.waitForDeployment(ctx) - require.Equal(t, err, tc.err, "errors are not equal") + require.Equal(t, tc.err, err, "errors are not equal") - require.Equal(t, logger.LeveledOutput(), strings.Join(outputLogs, "\n"), "logs are not equal") + if err != nil { + var failureLogs string + if tc.noLogs { + failureLogs = "" + } else { + failureLogs = outputLogs[0] + "\n" + outputLogs[1] + "\n\tx " + outputLogs[2] + "\n" + } + require.Equal(t, failureLogs, logger.SpinnerOutput(), "logs are not equal") + } else { + successLogs := strings.Join(outputLogs, "\n") + "\n" + require.Equal(t, successLogs, logger.SpinnerOutput(), "logs are not equal") + } }) } } From 76c407a80493a3a08ea531dcc8432218c9658264 Mon Sep 17 00:00:00 2001 From: janelletavares Date: Mon, 19 Sep 2022 14:31:13 -0700 Subject: [PATCH 06/11] lint --- cmd/meroxa/root/apps/deploy_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/meroxa/root/apps/deploy_test.go b/cmd/meroxa/root/apps/deploy_test.go index b12543aed..fc0853cda 100644 --- a/cmd/meroxa/root/apps/deploy_test.go +++ b/cmd/meroxa/root/apps/deploy_test.go @@ -976,6 +976,7 @@ func TestPrepareAppName(t *testing.T) { } } +//nolint:funlen func TestWaitForDeployment(t *testing.T) { ctx := context.Background() ctrl := gomock.NewController(t) From 8ad3f7f29e27ac75b6e5fb10e471a64e761e2b0f Mon Sep 17 00:00:00 2001 From: janelletavares Date: Wed, 21 Sep 2022 15:13:05 -0700 Subject: [PATCH 07/11] missed value assignment --- cmd/meroxa/root/apps/deploy.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/meroxa/root/apps/deploy.go b/cmd/meroxa/root/apps/deploy.go index 3d19f6a2f..19b4f8790 100644 --- a/cmd/meroxa/root/apps/deploy.go +++ b/cmd/meroxa/root/apps/deploy.go @@ -689,6 +689,7 @@ func (d *Deploy) Execute(ctx context.Context) error { return err } + d.specVersion = d.flags.Spec if d.specVersion == "" { // ⚠️ This is only until we re-deploy applications applying only the changes made if err = d.tearDownExistingResources(ctx); err != nil { From 5b35fec8ffb921b5325414fbd854a13d81883f9c Mon Sep 17 00:00:00 2001 From: janelletavares Date: Wed, 21 Sep 2022 16:46:18 -0700 Subject: [PATCH 08/11] testing --- cmd/meroxa/root/apps/deploy.go | 16 ---------------- cmd/meroxa/turbine/utils.go | 2 +- 2 files changed, 1 insertion(+), 17 deletions(-) diff --git a/cmd/meroxa/root/apps/deploy.go b/cmd/meroxa/root/apps/deploy.go index 19b4f8790..d65c9d4e2 100644 --- a/cmd/meroxa/root/apps/deploy.go +++ b/cmd/meroxa/root/apps/deploy.go @@ -30,7 +30,6 @@ import ( "github.com/volatiletech/null/v8" "github.com/meroxa/cli/cmd/meroxa/builder" - "github.com/meroxa/cli/cmd/meroxa/global" "github.com/meroxa/cli/cmd/meroxa/turbine" turbineGo "github.com/meroxa/cli/cmd/meroxa/turbine/golang" turbineJS "github.com/meroxa/cli/cmd/meroxa/turbine/javascript" @@ -643,18 +642,6 @@ func (d *Deploy) waitForDeployment(ctx context.Context) error { return fmt.Errorf("timed out; check `apps logs`") } -func hasFeatureFlag(f string) bool { - userFeatureFlags := global.Config.GetStringSlice(global.UserFeatureFlagsEnv) - - for _, v := range userFeatureFlags { - if v == f { - return true - } - } - - return false -} - //nolint:gocyclo func (d *Deploy) Execute(ctx context.Context) error { var app *meroxa.Application @@ -700,9 +687,6 @@ func (d *Deploy) Execute(ctx context.Context) error { if err = d.validateSpecVersionDeployment(); err != nil { return err } - if !hasFeatureFlag(featureFlagIntermediateRepresentation) { - return fmt.Errorf("user is not authorized for deploying with --spec") - } app, err = d.client.CreateApplicationV2(ctx, &meroxa.CreateApplicationInput{ Name: d.appName, diff --git a/cmd/meroxa/turbine/utils.go b/cmd/meroxa/turbine/utils.go index 7294d1041..a8bfc1ff3 100644 --- a/cmd/meroxa/turbine/utils.go +++ b/cmd/meroxa/turbine/utils.go @@ -528,7 +528,7 @@ func cleanUpPythonTempBuildLocation(ctx context.Context, logger log.Logger, appP } func GetTurbineResponseFromOutput(output string) (string, error) { - r := regexp.MustCompile("^turbine-response: ([^\n]*)") + r := regexp.MustCompile("turbine-response: ([^\n]*)") match := r.FindStringSubmatch(output) if match == nil || len(match) < 2 { return "", fmt.Errorf("output is formatted unexpectedly") From cb0df29c6959812116b50a4a74bdb979be1d253c Mon Sep 17 00:00:00 2001 From: janelletavares Date: Wed, 21 Sep 2022 16:55:52 -0700 Subject: [PATCH 09/11] no feature flag check --- cmd/meroxa/root/apps/deploy.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cmd/meroxa/root/apps/deploy.go b/cmd/meroxa/root/apps/deploy.go index d65c9d4e2..d78bb2aaf 100644 --- a/cmd/meroxa/root/apps/deploy.go +++ b/cmd/meroxa/root/apps/deploy.go @@ -43,8 +43,7 @@ const ( dockerHubUserNameEnv = "DOCKER_HUB_USERNAME" dockerHubAccessTokenEnv = "DOCKER_HUB_ACCESS_TOKEN" //nolint:gosec - pollDuration = 2 * time.Second - featureFlagIntermediateRepresentation = "intermediate-representation" + pollDuration = 2 * time.Second ) type deployApplicationClient interface { From bff4159d19fed662f5bb423327bd881b4eba88e9 Mon Sep 17 00:00:00 2001 From: janelletavares Date: Thu, 22 Sep 2022 12:24:46 -0700 Subject: [PATCH 10/11] Use a timeout context instead --- cmd/meroxa/root/apps/deploy.go | 64 ++++++++++++++++++++-------------- 1 file changed, 37 insertions(+), 27 deletions(-) diff --git a/cmd/meroxa/root/apps/deploy.go b/cmd/meroxa/root/apps/deploy.go index d78bb2aaf..6fee94b15 100644 --- a/cmd/meroxa/root/apps/deploy.go +++ b/cmd/meroxa/root/apps/deploy.go @@ -607,38 +607,48 @@ func (d *Deploy) prepareAppName(ctx context.Context) string { } func (d *Deploy) waitForDeployment(ctx context.Context) error { - startTime := time.Now().UTC() logs := []string{} - for time.Now().UTC().Sub(startTime) < 5*time.Minute { - var deployment *meroxa.Deployment - deployment, err := d.client.GetLatestDeployment(ctx, d.appName) - if err != nil { - return err - } - newLogs := strings.Split(deployment.OutputLog.String, "\n") - if len(newLogs) > len(logs) { - for i := len(logs); i < len(newLogs); i++ { - if len(logs) == 0 && i != 0 { - d.logger.StopSpinner(newLogs[i-1]) - } else if len(logs) != 0 { - d.logger.StopSpinner(newLogs[i-1]) + + cctx, cancel := context.WithTimeout(ctx, 5*time.Minute) + defer cancel() + + t := time.NewTicker(500 * time.Millisecond) + defer t.Stop() + + for { + select { + case <-t.C: + var deployment *meroxa.Deployment + deployment, err := d.client.GetLatestDeployment(ctx, d.appName) + if err != nil { + return err + } + newLogs := strings.Split(deployment.OutputLog.String, "\n") + if len(newLogs) > len(logs) { + for i := len(logs); i < len(newLogs); i++ { + if len(logs) == 0 && i != 0 { + d.logger.StopSpinner(newLogs[i-1]) + } else if len(logs) != 0 { + d.logger.StopSpinner(newLogs[i-1]) + } + d.logger.StartSpinner("\t", newLogs[i]) } - d.logger.StartSpinner("\t", newLogs[i]) + logs = newLogs } - logs = newLogs - } - if deployment.Status.State == meroxa.DeploymentStateDeployed { - l := len(logs) - d.logger.StopSpinner(logs[l-1]) - return nil - } else if deployment.Status.State == meroxa.DeploymentStateDeployingError || - deployment.Status.State == meroxa.DeploymentStateRollingBackError { - l := len(logs) - d.logger.StopSpinnerWithStatus(logs[l-1], log.Failed) - return fmt.Errorf("failed to deploy Application %q", d.appName) + if deployment.Status.State == meroxa.DeploymentStateDeployed { + l := len(logs) + d.logger.StopSpinner(logs[l-1]) + return nil + } else if deployment.Status.State == meroxa.DeploymentStateDeployingError || + deployment.Status.State == meroxa.DeploymentStateRollingBackError { + l := len(logs) + d.logger.StopSpinnerWithStatus(logs[l-1], log.Failed) + return fmt.Errorf("failed to deploy Application %q", d.appName) + } + case <-cctx.Done(): + return cctx.Err() } } - return fmt.Errorf("timed out; check `apps logs`") } //nolint:gocyclo From d59dc122e128cb07c53b292767c999b279df11cf Mon Sep 17 00:00:00 2001 From: janelletavares Date: Thu, 22 Sep 2022 13:38:29 -0700 Subject: [PATCH 11/11] lint --- cmd/meroxa/root/apps/deploy.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/cmd/meroxa/root/apps/deploy.go b/cmd/meroxa/root/apps/deploy.go index 6fee94b15..b5374b5cc 100644 --- a/cmd/meroxa/root/apps/deploy.go +++ b/cmd/meroxa/root/apps/deploy.go @@ -43,7 +43,9 @@ const ( dockerHubUserNameEnv = "DOCKER_HUB_USERNAME" dockerHubAccessTokenEnv = "DOCKER_HUB_ACCESS_TOKEN" //nolint:gosec - pollDuration = 2 * time.Second + platformBuildPollDuration = 2 * time.Second + durationToWaitForDeployment = 5 * time.Minute + intervalCheckForDeployment = 500 * time.Millisecond ) type deployApplicationClient interface { @@ -236,7 +238,7 @@ func (d *Deploy) getPlatformImage(ctx context.Context) (string, error) { d.logger.StopSpinnerWithStatus(fmt.Sprintf("Successfully built process image (%q)\n", build.Uuid), log.Successful) return build.Image, nil } - time.Sleep(pollDuration) + time.Sleep(platformBuildPollDuration) } } @@ -609,10 +611,10 @@ func (d *Deploy) prepareAppName(ctx context.Context) string { func (d *Deploy) waitForDeployment(ctx context.Context) error { logs := []string{} - cctx, cancel := context.WithTimeout(ctx, 5*time.Minute) + cctx, cancel := context.WithTimeout(ctx, durationToWaitForDeployment) defer cancel() - t := time.NewTicker(500 * time.Millisecond) + t := time.NewTicker(intervalCheckForDeployment) defer t.Stop() for {