Skip to content

Commit

Permalink
feat: Get image from platform (#283)
Browse files Browse the repository at this point in the history 
* feat: Get image from platform

* lint

* continue cd pattern when creating tar.gz

* chore: return error instead

* fix: bad rebase

Co-authored-by: janelletavares <[email protected]>
  • Loading branch information
@raulb
raulb and janelletavares authored Mar 31, 2022
1 parent c8360a1 commit 25dd8b8
Show file tree
Hide file tree
Showing 129 changed files with 2,481 additions and 508 deletions.
214 changes: 203 additions & 11 deletions cmd/meroxa/root/apps/deploy.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,16 @@ limitations under the License.
package apps

import (
"bytes"
"context"
"errors"
"fmt"
"io"
"net/http"
"os"
"path"
"path/filepath"
"time"

"github.com/volatiletech/null/v8"

Expand All @@ -31,27 +37,33 @@ import (
"github.com/meroxa/cli/config"
"github.com/meroxa/cli/log"
"github.com/meroxa/meroxa-go/pkg/meroxa"
turbine "github.com/meroxa/turbine/deploy"
)

const (
dockerHubUserNameEnv = "DOCKER_HUB_USERNAME"
dockerHubAccessTokenEnv = "DOCKER_HUB_ACCESS_TOKEN" // nolint:gosec
pollDuration = 2 * time.Second
)

type createApplicationClient interface {
type deployApplicationClient interface {
CreateApplication(ctx context.Context, input *meroxa.CreateApplicationInput) (*meroxa.Application, error)
CreateBuild(ctx context.Context, input *meroxa.CreateBuildInput) (*meroxa.Build, error)
CreateSource(ctx context.Context) (*meroxa.Source, error)
GetBuild(ctx context.Context, uuid string) (*meroxa.Build, error)
}

type Deploy struct {
flags struct {
Path string `long:"path" description:"path to the app directory (default is local directory)"`
DockerHubUserName string `long:"docker-hub-username" description:"DockerHub username to use to build and deploy the app"`
DockerHubAccessToken string `long:"docker-hub-access-token" description:"DockerHub access token to use to build and deploy the app"`
DockerHubUserName string `long:"docker-hub-username" description:"DockerHub username to use to build and deploy the app" hidden:"true"` //nolint:lll
DockerHubAccessToken string `long:"docker-hub-access-token" description:"DockerHub access token to use to build and deploy the app" hidden:"true"` //nolint:lll
}

client createApplicationClient
client deployApplicationClient
config config.Config
logger log.Logger
appName string
path string
lang string
goDeploy turbineGo.Deploy
Expand Down Expand Up @@ -189,13 +201,189 @@ func (d *Deploy) createApplication(ctx context.Context, pipelineUUID, gitSha str
return nil
}

// uploadSource creates first a Dockerfile to then, package the entire folder which will be later uploaded
// this should ignore .git files and fixtures/.
func (d *Deploy) uploadSource(ctx context.Context, appPath, url string) error {
// Before creating a .tar.zip, we make sure it contains a Dockerfile.
err := turbine.CreateDockerfile(appPath)
if err != nil {
return err
}

dFile := fmt.Sprintf("turbine-%s.tar.gz", d.appName)

var buf bytes.Buffer
d.logger.Infof(ctx, "Packaging application located at %q...", appPath)
err = turbineCLI.CreateTarAndZipFile(appPath, &buf)
if err != nil {
return err
}

fileToWrite, err := os.OpenFile(dFile, os.O_CREATE|os.O_RDWR, os.FileMode(0777)) //nolint:gomnd
defer func(fileToWrite *os.File) {
err = fileToWrite.Close()
if err != nil {
panic(err.Error())
}
}(fileToWrite)

if err != nil {
return err
}
if _, err = io.Copy(fileToWrite, &buf); err != nil {
return err
}

// We clean up Dockerfile as last step
err = os.Remove(filepath.Join(appPath, "Dockerfile"))
if err != nil {
return err
}

err = d.uploadFile(ctx, dFile, url)
if err != nil {
return err
}

// remove .tar.gz file
return os.Remove(dFile)
}

func (d *Deploy) uploadFile(ctx context.Context, filePath, url string) error {
d.logger.Info(ctx, "Uploading file to our build service...")
fh, err := os.Open(filePath)
if err != nil {
return err
}
defer func(fh *os.File) {
err = fh.Close()
if err != nil {
d.logger.Warn(ctx, err.Error())
}
}(fh)

req, err := http.NewRequestWithContext(ctx, "PUT", url, fh)
if err != nil {
return err
}

fi, err := fh.Stat()
if err != nil {
return err
}

req.Header.Set("Accept", "*/*")
req.Header.Set("Content-Type", "multipart/form-data")
req.Header.Set("Accept-Encoding", "gzip, deflate, br")
req.Header.Set("Connection", "keep-alive")

req.ContentLength = fi.Size()

client := &http.Client{}
res, err := client.Do(req) //nolint:bodyclose
if err != nil {
return err
}
defer func(Body io.ReadCloser) {
err := Body.Close()
d.logger.Infof(ctx, "Uploaded!")
if err != nil {
d.logger.Error(ctx, err.Error())
}
}(res.Body)

return nil
}

func (d *Deploy) getPlatformImage(ctx context.Context, appPath string) (string, error) {
s, err := d.client.CreateSource(ctx)
if err != nil {
return "", err
}

err = d.uploadSource(ctx, appPath, s.PutUrl)
if err != nil {
return "", err
}

sourceBlob := meroxa.SourceBlob{Url: s.GetUrl}
buildInput := &meroxa.CreateBuildInput{SourceBlob: sourceBlob}

build, err := d.client.CreateBuild(ctx, buildInput)
if err != nil {
return "", err
}

fmt.Print("Getting status for build: ", build.Uuid)
for {
fmt.Printf(".")
b, err := d.client.GetBuild(ctx, build.Uuid)
if err != nil {
return "", err
}

switch b.Status.State {
case "error":
return "", fmt.Errorf("build with uuid %q errored ", b.Uuid)
case "complete":
fmt.Println("Image built! ")
return build.Image, nil
}
time.Sleep(pollDuration)
}
}

// Deploy takes care of all the necessary steps to deploy a Turbine application
// 1. Build binary // different for jS
// 2. Build image // common
// 3. Push image // common
// 4. Run Turbine deploy // different
func (d *Deploy) deploy(ctx context.Context, appPath string, l log.Logger) (string, error) {
var fqImageName string
d.appName = path.Base(appPath)

err := turbineGo.BuildBinary(ctx, l, appPath, d.appName, true)
if err != nil {
return "", err
}

var ok bool
// check for image instances
if ok, err = turbineGo.NeedsToBuild(appPath, d.appName); ok {
if err != nil {
l.Errorf(ctx, err.Error())
return "", err
}

if d.goDeploy.LocalDeployment {
fqImageName, err = d.goDeploy.GetDockerImageName(ctx, l, appPath, d.appName)
if err != nil {
return "", err
}
} else {
fqImageName, err = d.getPlatformImage(ctx, appPath)
if err != nil {
return "", err
}
}
}

// creates all resources
output, err := turbineGo.RunDeployApp(ctx, l, appPath, d.appName, fqImageName)
if err != nil {
l.Errorf(ctx, "unable to deploy app; %s", err)
return output, err
}
return output, nil
}

func (d *Deploy) Execute(ctx context.Context) error {
var deployOuput string
// validateLocalDeploymentConfig will look for DockerHub credentials to determine whether it's a local deployment or not.
err := d.validateLocalDeploymentConfig()
if err != nil {
return err
}
var deployOutput string

d.path, err = turbineCLI.GetPath(d.flags.Path)
if err != nil {
Expand All @@ -216,13 +404,17 @@ func (d *Deploy) Execute(ctx context.Context) error {
return err
}

// 1. set up the app structure (CLI does this for any language)
// 2. *depending on the language* call something to create the dockerfile <=
// 3. CLI would handle:
// 3.1 creating the tar.zip,
// 3.2 post /sources
// 3.3 uploading the tar.zip
// 3.4 post /builds
// 4. CLI would call (depending on language) the deploy script <=
switch d.lang {
case GoLang:
// The only reason Deploy is scoped this other way is, so we can have the Docker Credentials
// Maybe that function should take care of checking type of deployment, only passing flags
// and environment variables
// err = turbineGo.Deploy(ctx, d.path, d.logger)
deployOuput, err = d.goDeploy.Deploy(ctx, d.path, d.logger)
deployOutput, err = d.deploy(ctx, d.path, d.logger)
case "js", JavaScript, NodeJs:
err = turbineJS.Deploy(ctx, d.path, d.logger)
default:
Expand All @@ -232,7 +424,7 @@ func (d *Deploy) Execute(ctx context.Context) error {
return err
}

pipelineUUID := turbineCLI.GetPipelineUUID(deployOuput)
pipelineUUID := turbineCLI.GetPipelineUUID(deployOutput)
gitSha, err := turbineCLI.GetGitSha(d.path)
if err != nil {
return err
Expand Down
4 changes: 2 additions & 2 deletions cmd/meroxa/root/apps/deploy_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,8 @@ func TestDeployAppFlags(t *testing.T) {
hidden bool
}{
{name: "path", required: false},
{name: "docker-hub-username", required: false},
{name: "docker-hub-access-token", required: false},
{name: "docker-hub-username", required: false, hidden: true},
{name: "docker-hub-access-token", required: false, hidden: true},
}

c := builder.BuildCobraCommand(&Deploy{})
Expand Down
4 changes: 2 additions & 2 deletions cmd/meroxa/turbine_cli/golang/build.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,8 @@ import (
"github.com/meroxa/cli/log"
)

// buildBinary will create a go binary with a specific name on a specific path.
func buildBinary(ctx context.Context, l log.Logger, appPath, appName string, platform bool) error {
// BuildBinary will create a go binary with a specific name on a specific path.
func BuildBinary(ctx context.Context, l log.Logger, appPath, appName string, platform bool) error {
var cmd *exec.Cmd

if platform {
Expand Down
54 changes: 4 additions & 50 deletions cmd/meroxa/turbine_cli/golang/deploy.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ import (
"fmt"
"os"
"os/exec"
"path"
"regexp"

turbinecli "github.com/meroxa/cli/cmd/meroxa/turbine_cli"
Expand All @@ -20,8 +19,8 @@ type Deploy struct {
LocalDeployment bool
}

// runDeployApp runs the binary previously built with the `--deploy` flag which should create all necessary resources.
func runDeployApp(ctx context.Context, l log.Logger, appPath, appName, imageName string) (string, error) {
// RunDeployApp runs the binary previously built with the `--deploy` flag which should create all necessary resources.
func RunDeployApp(ctx context.Context, l log.Logger, appPath, appName, imageName string) (string, error) {
l.Infof(ctx, "Deploying application %q...", appName)
var cmd *exec.Cmd

Expand All @@ -45,9 +44,9 @@ func runDeployApp(ctx context.Context, l log.Logger, appPath, appName, imageName
return stdout, err
}

// needsToBuild reads from the Turbine application to determine whether it needs to be built or not
// NeedsToBuild reads from the Turbine application to determine whether it needs to be built or not
// this is currently based on the number of functions.
func needsToBuild(appPath, appName string) (bool, error) {
func NeedsToBuild(appPath, appName string) (bool, error) {
cmd := exec.Command(appPath+"/"+appName, "--listfunctions") // nolint:gosec

accessToken, refreshToken, err := global.GetUserToken()
Expand All @@ -69,48 +68,3 @@ func needsToBuild(appPath, appName string) (bool, error) {

return hasFunctions, nil
}

// Deploy takes care of all the necessary steps to deploy a Turbine application
// 1. Build binary
// 2. Build image
// 3. Push image
// 4. Run Turbine deploy
func (gd *Deploy) Deploy(ctx context.Context, appPath string, l log.Logger) (string, error) {
var fqImageName string
appName := path.Base(appPath)

err := buildBinary(ctx, l, appPath, appName, true)
if err != nil {
return "", err
}

// check for image instances
var ok bool
if ok, err = needsToBuild(appPath, appName); ok {
if err != nil {
l.Errorf(ctx, err.Error())
return "", err
}

if gd.LocalDeployment {
fqImageName, err = gd.getDockerImageName(ctx, l, appPath, appName)
l.Infof(ctx, "fqImageName: %q", fqImageName)
if err != nil {
return "", err
}
} else {
// fqImageName, err = gd.getPlatformImage(ctx, l).
_, _ = gd.getPlatformImage(ctx, l)
// Returns so it doesn't error the next step
return "", fmt.Errorf("using build service not working at the moment, please use your own dockerhub credentials in the meantime")
}
}

// creates all resources
output, err := runDeployApp(ctx, l, appPath, appName, fqImageName)
if err != nil {
l.Errorf(ctx, "unable to deploy app; %s", err)
return output, err
}
return output, nil
}
Loading

0 comments on commit 25dd8b8

Please sign in to comment.