From 2338d4a3b125fd39114a244f66de254034d2dc1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Barroso?= Date: Mon, 14 Mar 2022 18:14:59 +0100 Subject: [PATCH 1/9] feat: check for functions --- cmd/meroxa/turbine_cli/go_lang.go | 82 ++++++++++++++++++++++++------- go.mod | 2 + go.sum | 2 - vendor/modules.txt | 3 +- 4 files changed, 68 insertions(+), 21 deletions(-) diff --git a/cmd/meroxa/turbine_cli/go_lang.go b/cmd/meroxa/turbine_cli/go_lang.go index e7290ce4e..fbb89440c 100644 --- a/cmd/meroxa/turbine_cli/go_lang.go +++ b/cmd/meroxa/turbine_cli/go_lang.go @@ -9,6 +9,7 @@ import ( "os" "os/exec" "path" + "regexp" "time" "github.com/meroxa/cli/cmd/meroxa/global" @@ -26,8 +27,8 @@ type GoDeploy struct { DockerHubAccessTokenEnv string } -// buildGoApp will create a go binary with a specific name on a specific path. -func buildGoApp(ctx context.Context, l log.Logger, appPath, appName string, platform bool) error { +// buildBinary will create a go binary with a specific name on a specific path. +func buildBinary(ctx context.Context, l log.Logger, appPath, appName string, platform bool) error { var cmd *exec.Cmd if platform { @@ -39,17 +40,18 @@ func buildGoApp(ctx context.Context, l log.Logger, appPath, appName string, plat stdout, err := cmd.CombinedOutput() if err != nil { - l.Error(ctx, string(stdout)) + l.Errorf(ctx, string(stdout)) return err } return nil } -// deployApp deploys the image previously built. -func deployApp(ctx context.Context, l log.Logger, appPath, appName, imageName string) error { +// runDeployApp runs the binary previously built with the `--deploy` flag which should create all necessary resources +func runDeployApp(ctx context.Context, l log.Logger, appPath, appName, imageName string) error { l.Infof(ctx, "Deploying application %q...", appName) + // TODO: Check here if imageName is "" cmd := exec.Command(appPath+"/"+appName, "--deploy", "--imagename", imageName) // nolint:gosec accessToken, refreshToken, err := global.GetUserToken() if err != nil { @@ -167,7 +169,7 @@ func RunGoApp(ctx context.Context, appPath string, l log.Logger) error { appName := path.Base(appPath) // building is a requirement prior to running for go apps - err := buildGoApp(ctx, l, appPath, appName, false) + err := buildBinary(ctx, l, appPath, appName, false) if err != nil { return err } @@ -182,34 +184,78 @@ func RunGoApp(ctx context.Context, appPath string, l log.Logger) error { out, err := cmd.CombinedOutput() if err != nil { l.Error(ctx, err.Error()) + return err } l.Info(ctx, string(out)) return nil } -func (gd *GoDeploy) DeployGoApp(ctx context.Context, appPath string, l log.Logger) error { - appName := path.Base(appPath) - fqImageName := gd.DockerHubUserNameEnv + "/" + appName - err := gd.buildImage(ctx, l, appPath, fqImageName) - if err != nil { - l.Errorf(ctx, "unable to build image; %q\n%s", fqImageName, err) - } +// needsToBuild reads from the Turbine application to determine whether it needs to be built or not +// this is currently based on the number of functions +func needsToBuild(appPath, appName string) (bool, error) { + cmd := exec.Command(appPath+"/"+appName, "--listfunctions") - err = gd.pushImage(l, fqImageName) + accessToken, refreshToken, err := global.GetUserToken() if err != nil { - l.Errorf(ctx, "unable to push image; %q\n%s", fqImageName, err) + return false, err } + cmd.Env = os.Environ() + cmd.Env = append(cmd.Env, fmt.Sprintf("ACCESS_TOKEN=%s", accessToken), fmt.Sprintf("REFRESH_TOKEN=%s", refreshToken)) + + re := regexp.MustCompile(`\[(.*?)]`) + stdout, err := cmd.CombinedOutput() + + // stdout is expected as `"2022/03/14 17:33:06 available functions: []` where within [], there will be each function + hasFunctions := len(re.FindAllString(string(stdout), -1)) > 1 + return hasFunctions, nil +} + +// DeployGoApp takes care of all the necessary steps to deploy a Turbine application +// 1. Build binary +// 2. Build image +// 3. Push image +// 4. Run Turbine deploy +func (gd *GoDeploy) DeployGoApp(ctx context.Context, appPath string, l log.Logger) error { + var fqImageName string + appName := path.Base(appPath) - err = buildGoApp(ctx, l, appPath, appName, true) + err := buildBinary(ctx, l, appPath, appName, true) if err != nil { return err } - err = deployApp(ctx, l, appPath, appName, fqImageName) + // to determine whether we need to call POST /sources and POST /build + if ok, err := needsToBuild(appPath, appName); ok { + if err != nil { + l.Errorf(ctx, err.Error()) + return err + } + + //fqImageName will be eventually taken from the build endpoint + fqImageName = gd.DockerHubUserNameEnv + "/" + appName + + err = gd.buildImage(ctx, l, appPath, fqImageName) + if err != nil { + l.Errorf(ctx, "unable to build image; %q\n%s", fqImageName, err) + return err + } + + // this will go away when using the new service + err = gd.pushImage(l, fqImageName) + if err != nil { + l.Errorf(ctx, "unable to push image; %q\n%s", fqImageName, err) + return err + } + + // TODO: Keep polling for building statsu until it's successful + } + + // creates all resources + err = runDeployApp(ctx, l, appPath, appName, fqImageName) if err != nil { l.Errorf(ctx, "unable to deploy app; %s", err) + return err } - return nil } diff --git a/go.mod b/go.mod index 7df6d2904..f43d721e3 100644 --- a/go.mod +++ b/go.mod @@ -88,3 +88,5 @@ require ( gopkg.in/ini.v1 v1.62.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect ) + +replace github.com/meroxa/turbine => ../turbine diff --git a/go.sum b/go.sum index 4c8745739..7bebfc0b5 100644 --- a/go.sum +++ b/go.sum @@ -577,8 +577,6 @@ github.com/meroxa/funtime v0.0.0-20220113012133-85e6e898fc73/go.mod h1:K2y2GvcA4 github.com/meroxa/meroxa-go v0.0.0-20220208195203-71ddc3133fab/go.mod h1:HDFszURCM1cOpKE699o5Hs0T2tEIXqY+vFcsur3RiwY= github.com/meroxa/meroxa-go v0.0.0-20220309001705-0d80e029000d h1:tnD5FSfL51tHXrQAwj1M5azcQ20zcGvjZogXbnRXxlg= github.com/meroxa/meroxa-go v0.0.0-20220309001705-0d80e029000d/go.mod h1:BsqYa9jqfyGOAgfkggfK567b2Ahkb+RCH3lXDQGgrh8= -github.com/meroxa/turbine v0.0.0-20220308221505-9789faa0043c h1:RkyAOSpyPhSqxMava2XhqzvPzn5ia6db+T6iWS87c+E= -github.com/meroxa/turbine v0.0.0-20220308221505-9789faa0043c/go.mod h1:4A2E9icHDi3x4Flp9CunWyRBvNkZy6h8MS8zLvWUlmw= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/pkcs11 v1.0.3/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4= diff --git a/vendor/modules.txt b/vendor/modules.txt index 96a46b743..53e7b7539 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -187,7 +187,7 @@ github.com/mattn/go-shellwords ## explicit; go 1.17 github.com/meroxa/meroxa-go/pkg/meroxa github.com/meroxa/meroxa-go/pkg/mock -# github.com/meroxa/turbine v0.0.0-20220308221505-9789faa0043c +# github.com/meroxa/turbine v0.0.0-20220308221505-9789faa0043c => ../turbine ## explicit; go 1.17 github.com/meroxa/turbine/deploy github.com/meroxa/turbine/init @@ -361,3 +361,4 @@ gopkg.in/ini.v1 # gopkg.in/yaml.v2 v2.4.0 ## explicit; go 1.15 gopkg.in/yaml.v2 +# github.com/meroxa/turbine => ../turbine From 08c2efe2325fb2fc77ae6309322b7fa426d71163 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Barroso?= Date: Tue, 15 Mar 2022 16:31:16 +0100 Subject: [PATCH 2/9] feat: Make local deployments configurable --- cmd/meroxa/root/apps/deploy.go | 60 ++++++++-- cmd/meroxa/root/apps/deploy_test.go | 164 ++++++++++++++++++++++++++++ cmd/meroxa/turbine_cli/go_lang.go | 1 + 3 files changed, 217 insertions(+), 8 deletions(-) diff --git a/cmd/meroxa/root/apps/deploy.go b/cmd/meroxa/root/apps/deploy.go index 680981d14..d12365e57 100644 --- a/cmd/meroxa/root/apps/deploy.go +++ b/cmd/meroxa/root/apps/deploy.go @@ -44,7 +44,9 @@ type createApplicationClient interface { type Deploy struct { flags struct { - Path string `long:"path" description:"path to the app directory"` + Path string `long:"path" description:"path to the app directory (default is local directory)"` + DockerHubUserName string `long:"docker-hub-username" description:"DockerHub username to use to build and deploy the application image"` + DockerHubAccessToken string `long:"docker-hub-access-token" description:"DockerHub access token to use to build and deploy the application image"` } client createApplicationClient @@ -105,15 +107,56 @@ func (d *Deploy) getDockerHubAccessTokenEnv() string { return d.config.GetString(dockerHubAccessTokenEnv) } -func (d *Deploy) checkRequiredEnvVars() error { - if d.getDockerHubUserNameEnv() == "" || d.getDockerHubAccessTokenEnv() == "" { - msg := fmt.Sprintf("both %q and %q are required to be set to deploy your application", dockerHubUserNameEnv, dockerHubAccessTokenEnv) - return errors.New(msg) +func (d *Deploy) validateDockerHubFlags() error { + if d.flags.DockerHubUserName != "" { + d.goDeploy.DockerHubUserNameEnv = d.flags.DockerHubUserName + if d.flags.DockerHubAccessToken == "" { + return errors.New("--docker-hub-access-token is required when --docker-hub-username is present") + } } - d.goDeploy.DockerHubUserNameEnv = d.getDockerHubUserNameEnv() - d.goDeploy.DockerHubAccessTokenEnv = d.getDockerHubAccessTokenEnv() + if d.flags.DockerHubAccessToken != "" { + d.goDeploy.DockerHubAccessTokenEnv = d.flags.DockerHubAccessToken + if d.flags.DockerHubUserName == "" { + return errors.New("--docker-hub-username is required when --docker-hub-access-token is present") + } + } + return nil +} +func (d *Deploy) validateDockerHubEnvVars() error { + if d.getDockerHubUserNameEnv() != "" { + d.goDeploy.DockerHubUserNameEnv = d.getDockerHubUserNameEnv() + if d.getDockerHubAccessTokenEnv() == "" { + return fmt.Errorf("%s is required when %s is present", dockerHubAccessTokenEnv, dockerHubUserNameEnv) + } + } + if d.getDockerHubAccessTokenEnv() != "" { + d.goDeploy.DockerHubAccessTokenEnv = d.getDockerHubAccessTokenEnv() + if d.getDockerHubUserNameEnv() == "" { + return fmt.Errorf("%s is required when %s is present", dockerHubUserNameEnv, dockerHubAccessTokenEnv) + } + } + return nil +} + +func (d *Deploy) validateLocalDeploymentConfig() error { + // Check if users had an old configuration where DockerHub credentials were set as environment variables + err := d.validateDockerHubEnvVars() + if err != nil { + return err + } + + // Check if users are making use of DockerHub flags + err = d.validateDockerHubFlags() + if err != nil { + return err + } + + // If there are DockerHub credentials, we consider it a local deployment + if d.goDeploy.DockerHubUserNameEnv != "" && d.goDeploy.DockerHubAccessTokenEnv != "" { + d.goDeploy.LocalDeployment = true + } return nil } @@ -182,7 +225,8 @@ func (d *Deploy) gitChecks(ctx context.Context) error { } func (d *Deploy) Execute(ctx context.Context) error { - err := d.checkRequiredEnvVars() + // validateLocalDeploymentConfig will look for DockerHub credentials to determine whether it's a local deployment or not + err := d.validateLocalDeploymentConfig() if err != nil { return err } diff --git a/cmd/meroxa/root/apps/deploy_test.go b/cmd/meroxa/root/apps/deploy_test.go index 5b87b1ac1..4635c099b 100644 --- a/cmd/meroxa/root/apps/deploy_test.go +++ b/cmd/meroxa/root/apps/deploy_test.go @@ -3,11 +3,14 @@ package apps import ( "context" "encoding/json" + "errors" "fmt" "os" "reflect" "testing" + "github.com/meroxa/cli/config" + "github.com/volatiletech/null/v8" "github.com/meroxa/meroxa-go/pkg/meroxa" @@ -29,6 +32,8 @@ func TestDeployAppFlags(t *testing.T) { hidden bool }{ {name: "path", required: false}, + {name: "docker-hub-username", required: false}, + {name: "docker-hub-access-token", required: false}, } c := builder.BuildCobraCommand(&Deploy{}) @@ -57,6 +62,165 @@ func TestDeployAppFlags(t *testing.T) { } } +func TestValidateDockerHubFlags(t *testing.T) { + tests := []struct { + desc string + dockerHubUserName string + dockerHubAccessToken string + err error + }{ + { + desc: "Neither DockerHub flags are present", + dockerHubUserName: "", + dockerHubAccessToken: "", + err: nil, + }, + { + desc: "DockerHubUserName is specified, but DockerHubAccessToken isn't", + dockerHubUserName: "my-docker-hub-username", + dockerHubAccessToken: "", + err: errors.New("--docker-hub-access-token is required when --docker-hub-username is present"), + }, + { + desc: "DockerHubAccessToken is specified, but DockerHubUserName isn't", + dockerHubUserName: "", + dockerHubAccessToken: "my-docker-hub-access-token", + err: errors.New("--docker-hub-username is required when --docker-hub-access-token is present"), + }, + { + desc: "BothDockerHubAccessToken and DockerHubUserName are specified", + dockerHubUserName: "my-docker-hub-username", + dockerHubAccessToken: "my-docker-hub-access-token", + err: nil, + }, + } + + for _, tc := range tests { + t.Run(tc.desc, func(t *testing.T) { + d := &Deploy{} + d.flags.DockerHubUserName = tc.dockerHubUserName + d.flags.DockerHubAccessToken = tc.dockerHubAccessToken + err := d.validateDockerHubFlags() + + if err != nil && tc.err.Error() != err.Error() { + t.Fatalf("expected %v, got %v", tc.err, err) + } + + if err == nil { + if d.goDeploy.DockerHubUserNameEnv != tc.dockerHubUserName { + t.Fatalf("expected DockerHubUserNameEnv to be %q, got %q", tc.dockerHubUserName, d.goDeploy.DockerHubUserNameEnv) + } + + if d.goDeploy.DockerHubAccessTokenEnv != tc.dockerHubAccessToken { + t.Fatalf("expected DockerHubAccessTokenEnv to be %q, got %q", tc.dockerHubAccessToken, d.goDeploy.DockerHubAccessTokenEnv) + } + } + }) + } +} + +func TestValidateDockerHubEnVars(t *testing.T) { + tests := []struct { + desc string + dockerHubUserName string + dockerHubAccessToken string + err error + }{ + { + desc: "Neither DockerHub flags are present", + dockerHubUserName: "", + dockerHubAccessToken: "", + err: nil, + }, + { + desc: "DockerHubUserName is specified, but DockerHubAccessToken isn't", + dockerHubUserName: "my-docker-hub-username", + dockerHubAccessToken: "", + err: fmt.Errorf("%s is required when %s is present", dockerHubAccessTokenEnv, dockerHubUserNameEnv), + }, + { + desc: "DockerHubAccessToken is specified, but DockerHubUserName isn't", + dockerHubUserName: "", + dockerHubAccessToken: "my-docker-hub-access-token", + err: fmt.Errorf("%s is required when %s is present", dockerHubUserNameEnv, dockerHubAccessTokenEnv), + }, + { + desc: "BothDockerHubAccessToken and DockerHubUserName are specified", + dockerHubUserName: "my-docker-hub-username", + dockerHubAccessToken: "my-docker-hub-access-token", + err: nil, + }, + } + + for _, tc := range tests { + t.Run(tc.desc, func(t *testing.T) { + cfg := config.NewInMemoryConfig() + + d := &Deploy{ + config: cfg, + } + d.config.Set(dockerHubUserNameEnv, tc.dockerHubUserName) + d.config.Set(dockerHubAccessTokenEnv, tc.dockerHubAccessToken) + err := d.validateDockerHubEnvVars() + + if err != nil && tc.err.Error() != err.Error() { + t.Fatalf("expected %v, got %v", tc.err, err) + } + + if err == nil { + if d.goDeploy.DockerHubUserNameEnv != tc.dockerHubUserName { + t.Fatalf("expected DockerHubUserNameEnv to be %q, got %q", tc.dockerHubUserName, d.goDeploy.DockerHubUserNameEnv) + } + + if d.goDeploy.DockerHubAccessTokenEnv != tc.dockerHubAccessToken { + t.Fatalf("expected DockerHubAccessTokenEnv to be %q, got %q", tc.dockerHubAccessToken, d.goDeploy.DockerHubAccessTokenEnv) + } + } + }) + } +} + +func TestValidateLocalDeploymentConfig(t *testing.T) { + tests := []struct { + desc string + dockerHubUserName string + dockerHubAccessToken string + localDeployment bool + }{ + { + desc: "Neither DockerHub flags are present", + dockerHubUserName: "", + dockerHubAccessToken: "", + localDeployment: false, + }, + { + desc: "BothDockerHubAccessToken and DockerHubUserName are specified", + dockerHubUserName: "my-docker-hub-username", + dockerHubAccessToken: "my-docker-hub-access-token", + localDeployment: true, + }, + } + + for _, tc := range tests { + t.Run(tc.desc, func(t *testing.T) { + cfg := config.NewInMemoryConfig() + d := &Deploy{ + config: cfg, + } + d.flags.DockerHubUserName = tc.dockerHubUserName + d.flags.DockerHubAccessToken = tc.dockerHubAccessToken + d.config.Set(dockerHubUserNameEnv, tc.dockerHubUserName) + d.config.Set(dockerHubAccessTokenEnv, tc.dockerHubAccessToken) + + err := d.validateLocalDeploymentConfig() + + if err == nil && d.goDeploy.LocalDeployment != tc.localDeployment { + t.Fatalf("expected localDeployment to be %v, got %v", tc.localDeployment, d.goDeploy.LocalDeployment) + } + }) + } +} + const tempAppDir = "test-app" func TestCreateApplication(t *testing.T) { diff --git a/cmd/meroxa/turbine_cli/go_lang.go b/cmd/meroxa/turbine_cli/go_lang.go index fbb89440c..f1ff0b539 100644 --- a/cmd/meroxa/turbine_cli/go_lang.go +++ b/cmd/meroxa/turbine_cli/go_lang.go @@ -25,6 +25,7 @@ import ( type GoDeploy struct { DockerHubUserNameEnv string DockerHubAccessTokenEnv string + LocalDeployment bool } // buildBinary will create a go binary with a specific name on a specific path. From 103d3dad70f8e669d4d8641710ecb6cb1e708ff6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Barroso?= Date: Tue, 15 Mar 2022 18:37:31 +0100 Subject: [PATCH 3/9] chore: Refactor in preparation for remote deploys --- cmd/meroxa/root/apps/deploy.go | 15 ++- cmd/meroxa/root/apps/run.go | 8 +- cmd/meroxa/turbine_cli/golang/build.go | 28 +++++ .../{go_lang.go => golang/deploy.go} | 109 ++---------------- .../turbine_cli/golang/local_deployment.go | 58 ++++++++++ cmd/meroxa/turbine_cli/golang/run.go | 38 ++++++ cmd/meroxa/turbine_cli/javascript/build.go | 21 ++++ .../{js_lang.go => javascript/deploy.go} | 17 +-- .../turbine_cli/{turbine_cli.go => utils.go} | 0 9 files changed, 172 insertions(+), 122 deletions(-) create mode 100644 cmd/meroxa/turbine_cli/golang/build.go rename cmd/meroxa/turbine_cli/{go_lang.go => golang/deploy.go} (61%) create mode 100644 cmd/meroxa/turbine_cli/golang/local_deployment.go create mode 100644 cmd/meroxa/turbine_cli/golang/run.go create mode 100644 cmd/meroxa/turbine_cli/javascript/build.go rename cmd/meroxa/turbine_cli/{js_lang.go => javascript/deploy.go} (50%) rename cmd/meroxa/turbine_cli/{turbine_cli.go => utils.go} (100%) diff --git a/cmd/meroxa/root/apps/deploy.go b/cmd/meroxa/root/apps/deploy.go index d12365e57..95fc2fc9c 100644 --- a/cmd/meroxa/root/apps/deploy.go +++ b/cmd/meroxa/root/apps/deploy.go @@ -24,6 +24,10 @@ import ( "os/exec" "strings" + turbineGo "github.com/meroxa/cli/cmd/meroxa/turbine_cli/golang" + + turbineJS "github.com/meroxa/cli/cmd/meroxa/turbine_cli/javascript" + "github.com/volatiletech/null/v8" "github.com/meroxa/cli/cmd/meroxa/builder" @@ -54,7 +58,7 @@ type Deploy struct { logger log.Logger path string lang string - goDeploy turbineCLI.GoDeploy + goDeploy turbineGo.Deploy } var ( @@ -164,6 +168,7 @@ func (d *Deploy) Logger(logger log.Logger) { d.logger = logger } +// TODO: Move this to each turbine library func (d *Deploy) createApplication(ctx context.Context) error { appName, err := turbineCLI.GetAppNameFromAppJSON(d.path) if err != nil { @@ -244,9 +249,13 @@ func (d *Deploy) Execute(ctx context.Context) error { switch d.lang { case GoLang: - err = d.goDeploy.DeployGoApp(ctx, d.path, d.logger) + // The only reason Deploy is scoped this other way is so we can have the Docker Credentials + // Maybe that function should take care of checking type of deployment, only passing flags + // and environment variables + // err = turbineGo.Deploy(ctx, d.path, d.logger) + err = d.goDeploy.Deploy(ctx, d.path, d.logger) case "js", JavaScript, NodeJs: - err = turbineCLI.DeployJSApp(ctx, d.path, d.logger) + err = turbineJS.Deploy(ctx, d.path, d.logger) default: return fmt.Errorf("language %q not supported. %s", d.lang, LanguageNotSupportedError) } diff --git a/cmd/meroxa/root/apps/run.go b/cmd/meroxa/root/apps/run.go index d57934e76..f5aaf4d42 100644 --- a/cmd/meroxa/root/apps/run.go +++ b/cmd/meroxa/root/apps/run.go @@ -21,6 +21,10 @@ import ( "fmt" "path/filepath" + turbineGo "github.com/meroxa/cli/cmd/meroxa/turbine_cli/golang" + + turbineJS "github.com/meroxa/cli/cmd/meroxa/turbine_cli/javascript" + "github.com/meroxa/cli/cmd/meroxa/builder" turbineCLI "github.com/meroxa/cli/cmd/meroxa/turbine_cli" "github.com/meroxa/cli/log" @@ -83,9 +87,9 @@ func (r *Run) Execute(ctx context.Context) error { switch lang { case GoLang: - return turbineCLI.RunGoApp(ctx, r.path, r.logger) + return turbineGo.Run(ctx, r.path, r.logger) case "js", JavaScript, NodeJs: - return turbineCLI.BuildJSApp(ctx, r.logger) + return turbineJS.Build(ctx, r.logger) default: return fmt.Errorf("language %q not supported. %s", lang, LanguageNotSupportedError) } diff --git a/cmd/meroxa/turbine_cli/golang/build.go b/cmd/meroxa/turbine_cli/golang/build.go new file mode 100644 index 000000000..4ff3e8cfe --- /dev/null +++ b/cmd/meroxa/turbine_cli/golang/build.go @@ -0,0 +1,28 @@ +package turbineGo + +import ( + "context" + "os/exec" + + "github.com/meroxa/cli/log" +) + +// buildBinary will create a go binary with a specific name on a specific path. +func buildBinary(ctx context.Context, l log.Logger, appPath, appName string, platform bool) error { + var cmd *exec.Cmd + + if platform { + cmd = exec.Command("go", "build", "--tags", "platform", "-o", appName, "./...") + } else { + cmd = exec.Command("go", "build", "-o", appName, "./...") + } + cmd.Dir = appPath + + stdout, err := cmd.CombinedOutput() + if err != nil { + l.Errorf(ctx, string(stdout)) + return err + } + + return nil +} diff --git a/cmd/meroxa/turbine_cli/go_lang.go b/cmd/meroxa/turbine_cli/golang/deploy.go similarity index 61% rename from cmd/meroxa/turbine_cli/go_lang.go rename to cmd/meroxa/turbine_cli/golang/deploy.go index f1ff0b539..3972b9ac3 100644 --- a/cmd/meroxa/turbine_cli/go_lang.go +++ b/cmd/meroxa/turbine_cli/golang/deploy.go @@ -1,53 +1,29 @@ -package turbinecli +package turbineGo import ( "context" - "encoding/base64" - "encoding/json" "fmt" "io" "os" "os/exec" "path" "regexp" - "time" - "github.com/meroxa/cli/cmd/meroxa/global" + turbine "github.com/meroxa/turbine/deploy" "github.com/docker/docker/api/types" "github.com/docker/docker/client" "github.com/docker/docker/pkg/archive" - turbine "github.com/meroxa/turbine/deploy" - + "github.com/meroxa/cli/cmd/meroxa/global" "github.com/meroxa/cli/log" ) -type GoDeploy struct { +type Deploy struct { DockerHubUserNameEnv string DockerHubAccessTokenEnv string LocalDeployment bool } -// buildBinary will create a go binary with a specific name on a specific path. -func buildBinary(ctx context.Context, l log.Logger, appPath, appName string, platform bool) error { - var cmd *exec.Cmd - - if platform { - cmd = exec.Command("go", "build", "--tags", "platform", "-o", appName, "./...") - } else { - cmd = exec.Command("go", "build", "-o", appName, "./...") - } - cmd.Dir = appPath - - stdout, err := cmd.CombinedOutput() - if err != nil { - l.Errorf(ctx, string(stdout)) - return err - } - - return nil -} - // runDeployApp runs the binary previously built with the `--deploy` flag which should create all necessary resources func runDeployApp(ctx context.Context, l log.Logger, appPath, appName, imageName string) error { l.Infof(ctx, "Deploying application %q...", appName) @@ -72,50 +48,7 @@ func runDeployApp(ctx context.Context, l log.Logger, appPath, appName, imageName return nil } -func (gd *GoDeploy) getAuthConfig() string { - dhUsername := gd.DockerHubUserNameEnv - dhAccessToken := gd.DockerHubAccessTokenEnv - authConfig := types.AuthConfig{ - Username: dhUsername, - Password: dhAccessToken, - ServerAddress: "https://index.docker.io/v1/", - } - authConfigBytes, _ := json.Marshal(authConfig) - return base64.URLEncoding.EncodeToString(authConfigBytes) -} - -func (gd *GoDeploy) pushImage(l log.Logger, imageName string) error { - cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation()) - if err != nil { - return err - } - authConfig := gd.getAuthConfig() - ctx, cancel := context.WithTimeout(context.Background(), time.Second*120) // nolint:gomnd - defer cancel() - - l.Infof(ctx, "pushing image %q to container registry", imageName) - opts := types.ImagePushOptions{RegistryAuth: authConfig} - rd, err := cli.ImagePush(ctx, imageName, opts) - if err != nil { - return err - } - defer func(rd io.ReadCloser) { - err = rd.Close() - if err != nil { - l.Error(ctx, err.Error()) - } - }(rd) - - _, err = io.Copy(os.Stdout, rd) - if err != nil { - return err - } - l.Info(ctx, "image successfully pushed to container registry!") - - return nil -} - -func (*GoDeploy) buildImage(ctx context.Context, l log.Logger, pwd, imageName string) error { +func (*Deploy) buildImage(ctx context.Context, l log.Logger, pwd, imageName string) error { l.Infof(ctx, "Building image %q located at %q", imageName, pwd) cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation()) if err != nil { @@ -164,34 +97,6 @@ func (*GoDeploy) buildImage(ctx context.Context, l log.Logger, pwd, imageName st return nil } -// RunGoApp will build a go binary and will run it. -func RunGoApp(ctx context.Context, appPath string, l log.Logger) error { - // grab current location to use it as project name - appName := path.Base(appPath) - - // building is a requirement prior to running for go apps - err := buildBinary(ctx, l, appPath, appName, false) - if err != nil { - return err - } - - err = os.Chdir(appPath) - if err != nil { - return err - } - - cmd := exec.Command("./" + appName) //nolint:gosec - - out, err := cmd.CombinedOutput() - if err != nil { - l.Error(ctx, err.Error()) - return err - } - - l.Info(ctx, string(out)) - return nil -} - // needsToBuild reads from the Turbine application to determine whether it needs to be built or not // this is currently based on the number of functions func needsToBuild(appPath, appName string) (bool, error) { @@ -212,12 +117,12 @@ func needsToBuild(appPath, appName string) (bool, error) { return hasFunctions, nil } -// DeployGoApp takes care of all the necessary steps to deploy a Turbine application +// Deploy takes care of all the necessary steps to deploy a Turbine application // 1. Build binary // 2. Build image // 3. Push image // 4. Run Turbine deploy -func (gd *GoDeploy) DeployGoApp(ctx context.Context, appPath string, l log.Logger) error { +func (gd *Deploy) Deploy(ctx context.Context, appPath string, l log.Logger) error { var fqImageName string appName := path.Base(appPath) diff --git a/cmd/meroxa/turbine_cli/golang/local_deployment.go b/cmd/meroxa/turbine_cli/golang/local_deployment.go new file mode 100644 index 000000000..72bca755e --- /dev/null +++ b/cmd/meroxa/turbine_cli/golang/local_deployment.go @@ -0,0 +1,58 @@ +// Package turbineGo TODO: Reorganize this in a different pkg +package turbineGo + +import ( + "context" + "encoding/base64" + "encoding/json" + "io" + "os" + "time" + + "github.com/docker/docker/api/types" + "github.com/docker/docker/client" + "github.com/meroxa/cli/log" +) + +func (gd *Deploy) getAuthConfig() string { + dhUsername := gd.DockerHubUserNameEnv + dhAccessToken := gd.DockerHubAccessTokenEnv + authConfig := types.AuthConfig{ + Username: dhUsername, + Password: dhAccessToken, + ServerAddress: "https://index.docker.io/v1/", + } + authConfigBytes, _ := json.Marshal(authConfig) + return base64.URLEncoding.EncodeToString(authConfigBytes) +} + +func (gd *Deploy) pushImage(l log.Logger, imageName string) error { + cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation()) + if err != nil { + return err + } + authConfig := gd.getAuthConfig() + ctx, cancel := context.WithTimeout(context.Background(), time.Second*120) // nolint:gomnd + defer cancel() + + l.Infof(ctx, "pushing image %q to container registry", imageName) + opts := types.ImagePushOptions{RegistryAuth: authConfig} + rd, err := cli.ImagePush(ctx, imageName, opts) + if err != nil { + return err + } + defer func(rd io.ReadCloser) { + err = rd.Close() + if err != nil { + l.Error(ctx, err.Error()) + } + }(rd) + + _, err = io.Copy(os.Stdout, rd) + if err != nil { + return err + } + l.Info(ctx, "image successfully pushed to container registry!") + + return nil +} diff --git a/cmd/meroxa/turbine_cli/golang/run.go b/cmd/meroxa/turbine_cli/golang/run.go new file mode 100644 index 000000000..27a189a40 --- /dev/null +++ b/cmd/meroxa/turbine_cli/golang/run.go @@ -0,0 +1,38 @@ +package turbineGo + +import ( + "context" + "os" + "os/exec" + "path" + + "github.com/meroxa/cli/log" +) + +// Run will build a go binary and will run it. +func Run(ctx context.Context, appPath string, l log.Logger) error { + // grab current location to use it as project name + appName := path.Base(appPath) + + // building is a requirement prior to running for go apps + err := buildBinary(ctx, l, appPath, appName, false) + if err != nil { + return err + } + + err = os.Chdir(appPath) + if err != nil { + return err + } + + cmd := exec.Command("./" + appName) //nolint:gosec + + out, err := cmd.CombinedOutput() + if err != nil { + l.Error(ctx, err.Error()) + return err + } + + l.Info(ctx, string(out)) + return nil +} diff --git a/cmd/meroxa/turbine_cli/javascript/build.go b/cmd/meroxa/turbine_cli/javascript/build.go new file mode 100644 index 000000000..698819265 --- /dev/null +++ b/cmd/meroxa/turbine_cli/javascript/build.go @@ -0,0 +1,21 @@ +package turbineJS + +import ( + "context" + "os/exec" + + "github.com/meroxa/cli/log" +) + +// Build calls turbine-js to build an application. +func Build(ctx context.Context, l log.Logger) error { + // TODO: Handle the requirement of https://github.com/meroxa/turbine-js.git being installed + // cd into the path first + cmd := exec.Command("npx", "turbine", "test") + stdout, err := cmd.CombinedOutput() + if err != nil { + return err + } + l.Info(ctx, string(stdout)) + return nil +} diff --git a/cmd/meroxa/turbine_cli/js_lang.go b/cmd/meroxa/turbine_cli/javascript/deploy.go similarity index 50% rename from cmd/meroxa/turbine_cli/js_lang.go rename to cmd/meroxa/turbine_cli/javascript/deploy.go index 427c683ee..4abb007ed 100644 --- a/cmd/meroxa/turbine_cli/js_lang.go +++ b/cmd/meroxa/turbine_cli/javascript/deploy.go @@ -1,4 +1,4 @@ -package turbinecli +package turbineJS import ( "context" @@ -10,20 +10,7 @@ import ( "github.com/meroxa/cli/log" ) -// BuildJSApp calls turbine-js to build an application. -func BuildJSApp(ctx context.Context, l log.Logger) error { - // TODO: Handle the requirement of https://github.com/meroxa/turbine-js.git being installed - // cd into the path first - cmd := exec.Command("npx", "turbine", "test") - stdout, err := cmd.CombinedOutput() - if err != nil { - return err - } - l.Info(ctx, string(stdout)) - return nil -} - -func DeployJSApp(ctx context.Context, path string, l log.Logger) error { +func Deploy(ctx context.Context, path string, l log.Logger) error { cmd := exec.Command("npx", "turbine", "deploy", path) accessToken, _, err := global.GetUserToken() diff --git a/cmd/meroxa/turbine_cli/turbine_cli.go b/cmd/meroxa/turbine_cli/utils.go similarity index 100% rename from cmd/meroxa/turbine_cli/turbine_cli.go rename to cmd/meroxa/turbine_cli/utils.go From c2d51882db1e1c4526423588cdcee3ac8dfaf488 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Barroso?= Date: Wed, 16 Mar 2022 12:34:53 +0100 Subject: [PATCH 4/9] feat: Separate local vs platform deployment --- cmd/meroxa/turbine_cli/golang/deploy.go | 89 ++++--------------- .../turbine_cli/golang/local_deployment.go | 73 +++++++++++++++ .../turbine_cli/golang/platform_deployment.go | 16 ++++ 3 files changed, 104 insertions(+), 74 deletions(-) create mode 100644 cmd/meroxa/turbine_cli/golang/platform_deployment.go diff --git a/cmd/meroxa/turbine_cli/golang/deploy.go b/cmd/meroxa/turbine_cli/golang/deploy.go index 3972b9ac3..62776460c 100644 --- a/cmd/meroxa/turbine_cli/golang/deploy.go +++ b/cmd/meroxa/turbine_cli/golang/deploy.go @@ -3,17 +3,11 @@ package turbineGo import ( "context" "fmt" - "io" "os" "os/exec" "path" "regexp" - turbine "github.com/meroxa/turbine/deploy" - - "github.com/docker/docker/api/types" - "github.com/docker/docker/client" - "github.com/docker/docker/pkg/archive" "github.com/meroxa/cli/cmd/meroxa/global" "github.com/meroxa/cli/log" ) @@ -27,9 +21,14 @@ type Deploy struct { // runDeployApp runs the binary previously built with the `--deploy` flag which should create all necessary resources func runDeployApp(ctx context.Context, l log.Logger, appPath, appName, imageName string) error { l.Infof(ctx, "Deploying application %q...", appName) + var cmd *exec.Cmd + + if imageName != "" { + cmd = exec.Command(appPath+"/"+appName, "--deploy", "--imagename", imageName) // nolint:gosec + } else { + cmd = exec.Command(appPath+"/"+appName, "--deploy") + } - // TODO: Check here if imageName is "" - cmd := exec.Command(appPath+"/"+appName, "--deploy", "--imagename", imageName) // nolint:gosec accessToken, refreshToken, err := global.GetUserToken() if err != nil { return err @@ -48,55 +47,6 @@ func runDeployApp(ctx context.Context, l log.Logger, appPath, appName, imageName return nil } -func (*Deploy) buildImage(ctx context.Context, l log.Logger, pwd, imageName string) error { - l.Infof(ctx, "Building image %q located at %q", imageName, pwd) - cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation()) - if err != nil { - l.Errorf(ctx, "unable to init docker client; %s", err) - } - - // Generate dockerfile - err = turbine.CreateDockerfile(pwd) - if err != nil { - return err - } - - // Read local Dockerfile - tar, err := archive.TarWithOptions(pwd, &archive.TarOptions{ - Compression: archive.Uncompressed, - ExcludePatterns: []string{".git", "fixtures"}, - }) - if err != nil { - l.Errorf(ctx, "unable to create tar; %s", err) - } - - buildOptions := types.ImageBuildOptions{ - Context: tar, - Dockerfile: "Dockerfile", - Remove: true, - Tags: []string{imageName}} - - resp, err := cli.ImageBuild( - ctx, - tar, - buildOptions, - ) - if err != nil { - l.Errorf(ctx, "unable to build docker image; %s", err) - } - defer func(Body io.ReadCloser) { - err = Body.Close() - if err != nil { - l.Errorf(ctx, "unable to close docker build response body; %s", err) - } - }(resp.Body) - _, err = io.Copy(os.Stdout, resp.Body) - if err != nil { - l.Errorf(ctx, "unable to read image build response; %s", err) - } - return nil -} - // needsToBuild reads from the Turbine application to determine whether it needs to be built or not // this is currently based on the number of functions func needsToBuild(appPath, appName string) (bool, error) { @@ -131,30 +81,21 @@ func (gd *Deploy) Deploy(ctx context.Context, appPath string, l log.Logger) erro return err } - // to determine whether we need to call POST /sources and POST /build + // check for image instances if ok, err := needsToBuild(appPath, appName); ok { if err != nil { l.Errorf(ctx, err.Error()) return err } - //fqImageName will be eventually taken from the build endpoint - fqImageName = gd.DockerHubUserNameEnv + "/" + appName - - err = gd.buildImage(ctx, l, appPath, fqImageName) - if err != nil { - l.Errorf(ctx, "unable to build image; %q\n%s", fqImageName, err) - return err + if gd.LocalDeployment { + fqImageName, err = gd.getDockerImageName(ctx, l, appPath, appName) + if err != nil { + return err + } + } else { + fqImageName, err = gd.getPlatformImage(ctx, l) } - - // this will go away when using the new service - err = gd.pushImage(l, fqImageName) - if err != nil { - l.Errorf(ctx, "unable to push image; %q\n%s", fqImageName, err) - return err - } - - // TODO: Keep polling for building statsu until it's successful } // creates all resources diff --git a/cmd/meroxa/turbine_cli/golang/local_deployment.go b/cmd/meroxa/turbine_cli/golang/local_deployment.go index 72bca755e..16d3a2360 100644 --- a/cmd/meroxa/turbine_cli/golang/local_deployment.go +++ b/cmd/meroxa/turbine_cli/golang/local_deployment.go @@ -9,6 +9,9 @@ import ( "os" "time" + "github.com/docker/docker/pkg/archive" + turbine "github.com/meroxa/turbine/deploy" + "github.com/docker/docker/api/types" "github.com/docker/docker/client" "github.com/meroxa/cli/log" @@ -26,6 +29,76 @@ func (gd *Deploy) getAuthConfig() string { return base64.URLEncoding.EncodeToString(authConfigBytes) } +// getDockerImageName Will create the image via DockerHub +func (gd *Deploy) getDockerImageName(ctx context.Context, l log.Logger, appPath, appName string) (string, error) { + //fqImageName will be eventually taken from the build endpoint + fqImageName := gd.DockerHubUserNameEnv + "/" + appName + + err := gd.buildImage(ctx, l, appPath, fqImageName) + if err != nil { + l.Errorf(ctx, "unable to build image; %q\n%s", fqImageName, err) + return "", err + } + + // this will go away when using the new service + err = gd.pushImage(l, fqImageName) + if err != nil { + l.Errorf(ctx, "unable to push image; %q\n%s", fqImageName, err) + return "", err + } + + return fqImageName, nil +} + +func (*Deploy) buildImage(ctx context.Context, l log.Logger, pwd, imageName string) error { + l.Infof(ctx, "Building image %q located at %q", imageName, pwd) + cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation()) + if err != nil { + l.Errorf(ctx, "unable to init docker client; %s", err) + } + + // Generate dockerfile + err = turbine.CreateDockerfile(pwd) + if err != nil { + return err + } + + // Read local Dockerfile + tar, err := archive.TarWithOptions(pwd, &archive.TarOptions{ + Compression: archive.Uncompressed, + ExcludePatterns: []string{".git", "fixtures"}, + }) + if err != nil { + l.Errorf(ctx, "unable to create tar; %s", err) + } + + buildOptions := types.ImageBuildOptions{ + Context: tar, + Dockerfile: "Dockerfile", + Remove: true, + Tags: []string{imageName}} + + resp, err := cli.ImageBuild( + ctx, + tar, + buildOptions, + ) + if err != nil { + l.Errorf(ctx, "unable to build docker image; %s", err) + } + defer func(Body io.ReadCloser) { + err = Body.Close() + if err != nil { + l.Errorf(ctx, "unable to close docker build response body; %s", err) + } + }(resp.Body) + _, err = io.Copy(os.Stdout, resp.Body) + if err != nil { + l.Errorf(ctx, "unable to read image build response; %s", err) + } + return nil +} + func (gd *Deploy) pushImage(l log.Logger, imageName string) error { cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation()) if err != nil { diff --git a/cmd/meroxa/turbine_cli/golang/platform_deployment.go b/cmd/meroxa/turbine_cli/golang/platform_deployment.go new file mode 100644 index 000000000..3666bd168 --- /dev/null +++ b/cmd/meroxa/turbine_cli/golang/platform_deployment.go @@ -0,0 +1,16 @@ +package turbineGo + +import ( + "context" + + "github.com/meroxa/cli/log" +) + +// TODO: Implement +func (gd *Deploy) getPlatformImage(ctx context.Context, l log.Logger) (string, error) { + // Get source (POST /source) + l.Infof(ctx, "POST /source") + // POST /builds + l.Infof(ctx, "POST /builds") + return "", nil +} From 5b9ffe5ffe640f6e4b5d8f12ec8b4acbfed248d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Barroso?= Date: Wed, 16 Mar 2022 17:24:25 +0100 Subject: [PATCH 5/9] feat: Associate pipeline with application --- cmd/meroxa/root/apps/deploy.go | 26 +++++++++++++++------ cmd/meroxa/turbine_cli/golang/deploy.go | 31 ++++++++++++++----------- go.mod | 2 +- vendor/modules.txt | 2 +- 4 files changed, 39 insertions(+), 22 deletions(-) diff --git a/cmd/meroxa/root/apps/deploy.go b/cmd/meroxa/root/apps/deploy.go index 95fc2fc9c..5fa0305d0 100644 --- a/cmd/meroxa/root/apps/deploy.go +++ b/cmd/meroxa/root/apps/deploy.go @@ -22,6 +22,7 @@ import ( "fmt" "os" "os/exec" + "regexp" "strings" turbineGo "github.com/meroxa/cli/cmd/meroxa/turbine_cli/golang" @@ -169,7 +170,7 @@ func (d *Deploy) Logger(logger log.Logger) { } // TODO: Move this to each turbine library -func (d *Deploy) createApplication(ctx context.Context) error { +func (d *Deploy) createApplication(ctx context.Context, pipelineUUID string) error { appName, err := turbineCLI.GetAppNameFromAppJSON(d.path) if err != nil { return err @@ -179,7 +180,7 @@ func (d *Deploy) createApplication(ctx context.Context) error { Name: appName, Language: d.lang, GitSha: "hardcoded", - Pipeline: meroxa.EntityIdentifier{Name: null.StringFrom("default")}, + Pipeline: meroxa.EntityIdentifier{UUID: null.StringFrom(pipelineUUID)}, } d.logger.Infof(ctx, "Creating application %q with language %q...", input.Name, d.lang) @@ -229,7 +230,18 @@ func (d *Deploy) gitChecks(ctx context.Context) error { return os.Chdir(pwd) } +// getPipelineUUID parses the deploy output when it was successful to determine the pipeline UUID to create +func getPipelineUUID(output string) string { + // Example output: + // 2022/03/16 13:21:36 pipeline created: "turbine-pipeline-simple" ("049760a8-a3d2-44d9-b326-0614c09a3f3e") + re := regexp.MustCompile(`pipeline created:."[a-zA-Z]+-[a-zA-Z]+-[a-zA-Z]+".(\([^)]*\))`) + res := re.FindStringSubmatch(output)[1] + res = strings.Trim(res, "()\"") + return res +} + func (d *Deploy) Execute(ctx context.Context) error { + var deployOuput string // validateLocalDeploymentConfig will look for DockerHub credentials to determine whether it's a local deployment or not err := d.validateLocalDeploymentConfig() if err != nil { @@ -249,21 +261,21 @@ func (d *Deploy) Execute(ctx context.Context) error { switch d.lang { case GoLang: - // The only reason Deploy is scoped this other way is so we can have the Docker Credentials + // The only reason Deploy is scoped this other way is, so we can have the Docker Credentials // Maybe that function should take care of checking type of deployment, only passing flags // and environment variables // err = turbineGo.Deploy(ctx, d.path, d.logger) - err = d.goDeploy.Deploy(ctx, d.path, d.logger) + deployOuput, err = d.goDeploy.Deploy(ctx, d.path, d.logger) case "js", JavaScript, NodeJs: err = turbineJS.Deploy(ctx, d.path, d.logger) default: return fmt.Errorf("language %q not supported. %s", d.lang, LanguageNotSupportedError) } - if err != nil { return err } - // Build was successful, we're ready to create the application - return d.createApplication(ctx) + pipelineUUID := getPipelineUUID(deployOuput) + + return d.createApplication(ctx, pipelineUUID) } diff --git a/cmd/meroxa/turbine_cli/golang/deploy.go b/cmd/meroxa/turbine_cli/golang/deploy.go index 62776460c..6558188f5 100644 --- a/cmd/meroxa/turbine_cli/golang/deploy.go +++ b/cmd/meroxa/turbine_cli/golang/deploy.go @@ -19,7 +19,7 @@ type Deploy struct { } // runDeployApp runs the binary previously built with the `--deploy` flag which should create all necessary resources -func runDeployApp(ctx context.Context, l log.Logger, appPath, appName, imageName string) error { +func runDeployApp(ctx context.Context, l log.Logger, appPath, appName, imageName string) (string, error) { l.Infof(ctx, "Deploying application %q...", appName) var cmd *exec.Cmd @@ -31,7 +31,7 @@ func runDeployApp(ctx context.Context, l log.Logger, appPath, appName, imageName accessToken, refreshToken, err := global.GetUserToken() if err != nil { - return err + return "", err } cmd.Env = os.Environ() cmd.Env = append(cmd.Env, fmt.Sprintf("ACCESS_TOKEN=%s", accessToken), fmt.Sprintf("REFRESH_TOKEN=%s", refreshToken)) @@ -39,12 +39,12 @@ func runDeployApp(ctx context.Context, l log.Logger, appPath, appName, imageName stdout, err := cmd.CombinedOutput() if err != nil { l.Errorf(ctx, string(stdout)) - return err + return "", err } l.Info(ctx, string(stdout)) l.Info(ctx, "deploy complete!") - return nil + return string(stdout), nil } // needsToBuild reads from the Turbine application to determine whether it needs to be built or not @@ -59,11 +59,13 @@ func needsToBuild(appPath, appName string) (bool, error) { cmd.Env = os.Environ() cmd.Env = append(cmd.Env, fmt.Sprintf("ACCESS_TOKEN=%s", accessToken), fmt.Sprintf("REFRESH_TOKEN=%s", refreshToken)) - re := regexp.MustCompile(`\[(.*?)]`) + // TODO: Implement in Turbine something that returns a boolean rather than having to regex its output + re := regexp.MustCompile(`\[(.+?)]`) stdout, err := cmd.CombinedOutput() // stdout is expected as `"2022/03/14 17:33:06 available functions: []` where within [], there will be each function - hasFunctions := len(re.FindAllString(string(stdout), -1)) > 1 + hasFunctions := len(re.FindAllString(string(stdout), -1)) > 0 + return hasFunctions, nil } @@ -72,37 +74,40 @@ func needsToBuild(appPath, appName string) (bool, error) { // 2. Build image // 3. Push image // 4. Run Turbine deploy -func (gd *Deploy) Deploy(ctx context.Context, appPath string, l log.Logger) error { +func (gd *Deploy) Deploy(ctx context.Context, appPath string, l log.Logger) (string, error) { var fqImageName string appName := path.Base(appPath) err := buildBinary(ctx, l, appPath, appName, true) if err != nil { - return err + return "", err } // check for image instances if ok, err := needsToBuild(appPath, appName); ok { if err != nil { l.Errorf(ctx, err.Error()) - return err + return "", err } if gd.LocalDeployment { fqImageName, err = gd.getDockerImageName(ctx, l, appPath, appName) + l.Infof(ctx, "fqImageName: %q", fqImageName) if err != nil { - return err + return "", err } } else { fqImageName, err = gd.getPlatformImage(ctx, l) + // Returns so it doesn't error the next step + return "", fmt.Errorf("using build service not working at the moment, please use your own dockerhub credentials in the meantime") } } // creates all resources - err = runDeployApp(ctx, l, appPath, appName, fqImageName) + output, err := runDeployApp(ctx, l, appPath, appName, fqImageName) if err != nil { l.Errorf(ctx, "unable to deploy app; %s", err) - return err + return output, err } - return nil + return output, nil } diff --git a/go.mod b/go.mod index f43d721e3..1debe0a41 100644 --- a/go.mod +++ b/go.mod @@ -27,7 +27,7 @@ require ( require ( github.com/docker/docker v20.10.12+incompatible github.com/mattn/go-shellwords v1.0.12 - github.com/meroxa/turbine v0.0.0-20220308221505-9789faa0043c + github.com/meroxa/turbine v0.0.0-20220315122820-92658c705129 github.com/volatiletech/null/v8 v8.1.2 ) diff --git a/vendor/modules.txt b/vendor/modules.txt index 53e7b7539..370d576b0 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -187,7 +187,7 @@ github.com/mattn/go-shellwords ## explicit; go 1.17 github.com/meroxa/meroxa-go/pkg/meroxa github.com/meroxa/meroxa-go/pkg/mock -# github.com/meroxa/turbine v0.0.0-20220308221505-9789faa0043c => ../turbine +# github.com/meroxa/turbine v0.0.0-20220315122820-92658c705129 => ../turbine ## explicit; go 1.17 github.com/meroxa/turbine/deploy github.com/meroxa/turbine/init From 9dc3e7c790f864e8f01cc1e9e97249678ebd9c39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Barroso?= Date: Wed, 16 Mar 2022 17:27:11 +0100 Subject: [PATCH 6/9] fix: make sure it logs out the error on apps init --- cmd/meroxa/root/apps/init.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/meroxa/root/apps/init.go b/cmd/meroxa/root/apps/init.go index 76f390396..2ada93a2f 100644 --- a/cmd/meroxa/root/apps/init.go +++ b/cmd/meroxa/root/apps/init.go @@ -97,6 +97,7 @@ func (i *Init) Execute(ctx context.Context) error { cmd := exec.Command("npx", "turbine", "generate", name, i.path) stdout, err := cmd.CombinedOutput() if err != nil { + i.logger.Error(ctx, string(stdout)) return err } i.logger.Info(ctx, string(stdout)) From c8ebd765a8cfae054a5752894712a4a000c614cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Barroso?= Date: Wed, 16 Mar 2022 17:54:01 +0100 Subject: [PATCH 7/9] chore: uses latest turbine --- go.mod | 2 -- go.sum | 2 ++ vendor/modules.txt | 3 +-- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 1debe0a41..43c4b05a8 100644 --- a/go.mod +++ b/go.mod @@ -88,5 +88,3 @@ require ( gopkg.in/ini.v1 v1.62.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect ) - -replace github.com/meroxa/turbine => ../turbine diff --git a/go.sum b/go.sum index 7bebfc0b5..090379fe5 100644 --- a/go.sum +++ b/go.sum @@ -577,6 +577,8 @@ github.com/meroxa/funtime v0.0.0-20220113012133-85e6e898fc73/go.mod h1:K2y2GvcA4 github.com/meroxa/meroxa-go v0.0.0-20220208195203-71ddc3133fab/go.mod h1:HDFszURCM1cOpKE699o5Hs0T2tEIXqY+vFcsur3RiwY= github.com/meroxa/meroxa-go v0.0.0-20220309001705-0d80e029000d h1:tnD5FSfL51tHXrQAwj1M5azcQ20zcGvjZogXbnRXxlg= github.com/meroxa/meroxa-go v0.0.0-20220309001705-0d80e029000d/go.mod h1:BsqYa9jqfyGOAgfkggfK567b2Ahkb+RCH3lXDQGgrh8= +github.com/meroxa/turbine v0.0.0-20220315122820-92658c705129 h1:AQcYzxVny/O2Sa+qEGbVmlmmAUIZLvI6Byyh7g8az8o= +github.com/meroxa/turbine v0.0.0-20220315122820-92658c705129/go.mod h1:4A2E9icHDi3x4Flp9CunWyRBvNkZy6h8MS8zLvWUlmw= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/pkcs11 v1.0.3/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4= diff --git a/vendor/modules.txt b/vendor/modules.txt index 370d576b0..b2624e376 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -187,7 +187,7 @@ github.com/mattn/go-shellwords ## explicit; go 1.17 github.com/meroxa/meroxa-go/pkg/meroxa github.com/meroxa/meroxa-go/pkg/mock -# github.com/meroxa/turbine v0.0.0-20220315122820-92658c705129 => ../turbine +# github.com/meroxa/turbine v0.0.0-20220315122820-92658c705129 ## explicit; go 1.17 github.com/meroxa/turbine/deploy github.com/meroxa/turbine/init @@ -361,4 +361,3 @@ gopkg.in/ini.v1 # gopkg.in/yaml.v2 v2.4.0 ## explicit; go 1.15 gopkg.in/yaml.v2 -# github.com/meroxa/turbine => ../turbine From 93f24d49b64c8f544147f712255bd17c448f73ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Barroso?= Date: Wed, 16 Mar 2022 17:56:45 +0100 Subject: [PATCH 8/9] fix: test --- cmd/meroxa/root/apps/deploy_test.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cmd/meroxa/root/apps/deploy_test.go b/cmd/meroxa/root/apps/deploy_test.go index 4635c099b..2d9f444f6 100644 --- a/cmd/meroxa/root/apps/deploy_test.go +++ b/cmd/meroxa/root/apps/deploy_test.go @@ -230,6 +230,7 @@ func TestCreateApplication(t *testing.T) { logger := log.NewTestLogger() name := "my-application" lang := GoLang + pipelineUUID := "5d0c9667-1626-4ffd-9a94-fab4092eec5a" // Create application locally path, err := initLocalApp(name) @@ -244,7 +245,7 @@ func TestCreateApplication(t *testing.T) { Name: name, Language: lang, GitSha: "hardcoded", - Pipeline: meroxa.EntityIdentifier{Name: null.StringFrom("default")}, + Pipeline: meroxa.EntityIdentifier{UUID: null.StringFrom(pipelineUUID)}, } a := &meroxa.Application{ @@ -268,7 +269,7 @@ func TestCreateApplication(t *testing.T) { lang: lang, } - err = d.createApplication(ctx) + err = d.createApplication(ctx, pipelineUUID) if err != nil { t.Fatalf("not expected error, got \"%s\"", err.Error()) From 990492e16570fc1db6ef39a2000ccb5547fec697 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Barroso?= Date: Wed, 16 Mar 2022 18:52:12 +0100 Subject: [PATCH 9/9] fix: linting errors --- cmd/meroxa/root/apps/deploy.go | 12 +++++------ cmd/meroxa/turbine_cli/golang/build.go | 2 +- cmd/meroxa/turbine_cli/golang/deploy.go | 21 ++++++++++++------- .../turbine_cli/golang/local_deployment.go | 8 +++---- .../turbine_cli/golang/platform_deployment.go | 4 ++-- cmd/meroxa/turbine_cli/golang/run.go | 2 +- cmd/meroxa/turbine_cli/javascript/build.go | 2 +- cmd/meroxa/turbine_cli/javascript/deploy.go | 2 +- 8 files changed, 29 insertions(+), 24 deletions(-) diff --git a/cmd/meroxa/root/apps/deploy.go b/cmd/meroxa/root/apps/deploy.go index 5fa0305d0..f66d97e46 100644 --- a/cmd/meroxa/root/apps/deploy.go +++ b/cmd/meroxa/root/apps/deploy.go @@ -50,8 +50,8 @@ type createApplicationClient interface { type Deploy struct { flags struct { Path string `long:"path" description:"path to the app directory (default is local directory)"` - DockerHubUserName string `long:"docker-hub-username" description:"DockerHub username to use to build and deploy the application image"` - DockerHubAccessToken string `long:"docker-hub-access-token" description:"DockerHub access token to use to build and deploy the application image"` + DockerHubUserName string `long:"docker-hub-username" description:"DockerHub username to use to build and deploy the app"` + DockerHubAccessToken string `long:"docker-hub-access-token" description:"DockerHub access token to use to build and deploy the app"` } client createApplicationClient @@ -169,7 +169,7 @@ func (d *Deploy) Logger(logger log.Logger) { d.logger = logger } -// TODO: Move this to each turbine library +// TODO: Move this to each turbine library. func (d *Deploy) createApplication(ctx context.Context, pipelineUUID string) error { appName, err := turbineCLI.GetAppNameFromAppJSON(d.path) if err != nil { @@ -230,10 +230,10 @@ func (d *Deploy) gitChecks(ctx context.Context) error { return os.Chdir(pwd) } -// getPipelineUUID parses the deploy output when it was successful to determine the pipeline UUID to create +// getPipelineUUID parses the deploy output when it was successful to determine the pipeline UUID to create. func getPipelineUUID(output string) string { // Example output: - // 2022/03/16 13:21:36 pipeline created: "turbine-pipeline-simple" ("049760a8-a3d2-44d9-b326-0614c09a3f3e") + // 2022/03/16 13:21:36 pipeline created: "turbine-pipeline-simple" ("049760a8-a3d2-44d9-b326-0614c09a3f3e"). re := regexp.MustCompile(`pipeline created:."[a-zA-Z]+-[a-zA-Z]+-[a-zA-Z]+".(\([^)]*\))`) res := re.FindStringSubmatch(output)[1] res = strings.Trim(res, "()\"") @@ -242,7 +242,7 @@ func getPipelineUUID(output string) string { func (d *Deploy) Execute(ctx context.Context) error { var deployOuput string - // validateLocalDeploymentConfig will look for DockerHub credentials to determine whether it's a local deployment or not + // validateLocalDeploymentConfig will look for DockerHub credentials to determine whether it's a local deployment or not. err := d.validateLocalDeploymentConfig() if err != nil { return err diff --git a/cmd/meroxa/turbine_cli/golang/build.go b/cmd/meroxa/turbine_cli/golang/build.go index 4ff3e8cfe..8dafca243 100644 --- a/cmd/meroxa/turbine_cli/golang/build.go +++ b/cmd/meroxa/turbine_cli/golang/build.go @@ -1,4 +1,4 @@ -package turbineGo +package turbinego import ( "context" diff --git a/cmd/meroxa/turbine_cli/golang/deploy.go b/cmd/meroxa/turbine_cli/golang/deploy.go index 6558188f5..d70663863 100644 --- a/cmd/meroxa/turbine_cli/golang/deploy.go +++ b/cmd/meroxa/turbine_cli/golang/deploy.go @@ -1,4 +1,4 @@ -package turbineGo +package turbinego import ( "context" @@ -18,7 +18,7 @@ type Deploy struct { LocalDeployment bool } -// runDeployApp runs the binary previously built with the `--deploy` flag which should create all necessary resources +// runDeployApp runs the binary previously built with the `--deploy` flag which should create all necessary resources. func runDeployApp(ctx context.Context, l log.Logger, appPath, appName, imageName string) (string, error) { l.Infof(ctx, "Deploying application %q...", appName) var cmd *exec.Cmd @@ -26,7 +26,7 @@ func runDeployApp(ctx context.Context, l log.Logger, appPath, appName, imageName if imageName != "" { cmd = exec.Command(appPath+"/"+appName, "--deploy", "--imagename", imageName) // nolint:gosec } else { - cmd = exec.Command(appPath+"/"+appName, "--deploy") + cmd = exec.Command(appPath+"/"+appName, "--deploy") // nolint:gosec } accessToken, refreshToken, err := global.GetUserToken() @@ -48,9 +48,9 @@ func runDeployApp(ctx context.Context, l log.Logger, appPath, appName, imageName } // needsToBuild reads from the Turbine application to determine whether it needs to be built or not -// this is currently based on the number of functions +// this is currently based on the number of functions. func needsToBuild(appPath, appName string) (bool, error) { - cmd := exec.Command(appPath+"/"+appName, "--listfunctions") + cmd := exec.Command(appPath+"/"+appName, "--listfunctions") // nolint:gosec accessToken, refreshToken, err := global.GetUserToken() if err != nil { @@ -62,8 +62,11 @@ func needsToBuild(appPath, appName string) (bool, error) { // TODO: Implement in Turbine something that returns a boolean rather than having to regex its output re := regexp.MustCompile(`\[(.+?)]`) stdout, err := cmd.CombinedOutput() + if err != nil { + return false, err + } - // stdout is expected as `"2022/03/14 17:33:06 available functions: []` where within [], there will be each function + // stdout is expected as `"2022/03/14 17:33:06 available functions: []` where within [], there will be each function. hasFunctions := len(re.FindAllString(string(stdout), -1)) > 0 return hasFunctions, nil @@ -84,7 +87,8 @@ func (gd *Deploy) Deploy(ctx context.Context, appPath string, l log.Logger) (str } // check for image instances - if ok, err := needsToBuild(appPath, appName); ok { + var ok bool + if ok, err = needsToBuild(appPath, appName); ok { if err != nil { l.Errorf(ctx, err.Error()) return "", err @@ -97,7 +101,8 @@ func (gd *Deploy) Deploy(ctx context.Context, appPath string, l log.Logger) (str return "", err } } else { - fqImageName, err = gd.getPlatformImage(ctx, l) + // fqImageName, err = gd.getPlatformImage(ctx, l). + _, _ = gd.getPlatformImage(ctx, l) // Returns so it doesn't error the next step return "", fmt.Errorf("using build service not working at the moment, please use your own dockerhub credentials in the meantime") } diff --git a/cmd/meroxa/turbine_cli/golang/local_deployment.go b/cmd/meroxa/turbine_cli/golang/local_deployment.go index 16d3a2360..9634011b0 100644 --- a/cmd/meroxa/turbine_cli/golang/local_deployment.go +++ b/cmd/meroxa/turbine_cli/golang/local_deployment.go @@ -1,5 +1,5 @@ // Package turbineGo TODO: Reorganize this in a different pkg -package turbineGo +package turbinego import ( "context" @@ -29,9 +29,9 @@ func (gd *Deploy) getAuthConfig() string { return base64.URLEncoding.EncodeToString(authConfigBytes) } -// getDockerImageName Will create the image via DockerHub +// getDockerImageName Will create the image via DockerHub. func (gd *Deploy) getDockerImageName(ctx context.Context, l log.Logger, appPath, appName string) (string, error) { - //fqImageName will be eventually taken from the build endpoint + // fqImageName will be eventually taken from the build endpoint. fqImageName := gd.DockerHubUserNameEnv + "/" + appName err := gd.buildImage(ctx, l, appPath, fqImageName) @@ -40,7 +40,7 @@ func (gd *Deploy) getDockerImageName(ctx context.Context, l log.Logger, appPath, return "", err } - // this will go away when using the new service + // this will go away when using the new service. err = gd.pushImage(l, fqImageName) if err != nil { l.Errorf(ctx, "unable to push image; %q\n%s", fqImageName, err) diff --git a/cmd/meroxa/turbine_cli/golang/platform_deployment.go b/cmd/meroxa/turbine_cli/golang/platform_deployment.go index 3666bd168..9b6a662e0 100644 --- a/cmd/meroxa/turbine_cli/golang/platform_deployment.go +++ b/cmd/meroxa/turbine_cli/golang/platform_deployment.go @@ -1,4 +1,4 @@ -package turbineGo +package turbinego import ( "context" @@ -6,7 +6,7 @@ import ( "github.com/meroxa/cli/log" ) -// TODO: Implement +// TODO: Implement. func (gd *Deploy) getPlatformImage(ctx context.Context, l log.Logger) (string, error) { // Get source (POST /source) l.Infof(ctx, "POST /source") diff --git a/cmd/meroxa/turbine_cli/golang/run.go b/cmd/meroxa/turbine_cli/golang/run.go index 27a189a40..518703b24 100644 --- a/cmd/meroxa/turbine_cli/golang/run.go +++ b/cmd/meroxa/turbine_cli/golang/run.go @@ -1,4 +1,4 @@ -package turbineGo +package turbinego import ( "context" diff --git a/cmd/meroxa/turbine_cli/javascript/build.go b/cmd/meroxa/turbine_cli/javascript/build.go index 698819265..622fd936b 100644 --- a/cmd/meroxa/turbine_cli/javascript/build.go +++ b/cmd/meroxa/turbine_cli/javascript/build.go @@ -1,4 +1,4 @@ -package turbineJS +package turbinejs import ( "context" diff --git a/cmd/meroxa/turbine_cli/javascript/deploy.go b/cmd/meroxa/turbine_cli/javascript/deploy.go index 4abb007ed..595ddeaa4 100644 --- a/cmd/meroxa/turbine_cli/javascript/deploy.go +++ b/cmd/meroxa/turbine_cli/javascript/deploy.go @@ -1,4 +1,4 @@ -package turbineJS +package turbinejs import ( "context"