forked from flyteorg/flyte
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* adding flytepropeller dep Signed-off-by: David Przybilla <[email protected]> * add compile cmd Signed-off-by: David Przybilla <[email protected]> * printing summary Signed-off-by: David Przybilla <[email protected]> * compile cmd tests Signed-off-by: David Przybilla <[email protected]> This reverts commit 75665de7db90646a3e5d7ec65f64c61739c31955. Signed-off-by: David Przybilla <[email protected]> * ci fix: go 1.18 Signed-off-by: David Przybilla <[email protected]> * compilecmd: fix typos & err handling Signed-off-by: David Przybilla <[email protected]> * compilecmd: more tests Signed-off-by: David Przybilla <[email protected]> * exporting UnMarshalContents Signed-off-by: David Przybilla <[email protected]> * compile uses UnMarshalContents Signed-off-by: David Przybilla <[email protected]> * compile cmd usage examples Signed-off-by: David Przybilla <[email protected]> * compilecmd: popping errors up Signed-off-by: David Przybilla <[email protected]> * compilecmd: add flag config Signed-off-by: David Przybilla <[email protected]> * compilecmd better docstring Signed-off-by: David Przybilla <[email protected]> * compilecmd uses flags config Signed-off-by: David Przybilla <[email protected]> * clean up Signed-off-by: David Przybilla <[email protected]>
- Loading branch information
1 parent
cd55089
commit 0588bb2
Showing
18 changed files
with
719 additions
and
70 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,146 @@ | ||
package compile | ||
|
||
import ( | ||
"context" | ||
"fmt" | ||
"io/ioutil" | ||
"os" | ||
|
||
config "github.com/flyteorg/flytectl/cmd/config/subcommand/compile" | ||
cmdCore "github.com/flyteorg/flytectl/cmd/core" | ||
"github.com/flyteorg/flytectl/cmd/register" | ||
"github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin" | ||
"github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core" | ||
"github.com/flyteorg/flytepropeller/pkg/compiler" | ||
"github.com/flyteorg/flytepropeller/pkg/compiler/common" | ||
) | ||
|
||
// Utility function for compiling a list of Tasks | ||
func compileTasks(tasks []*core.TaskTemplate) ([]*core.CompiledTask, error) { | ||
res := make([]*core.CompiledTask, 0, len(tasks)) | ||
for _, task := range tasks { | ||
compiledTask, err := compiler.CompileTask(task) | ||
if err != nil { | ||
return nil, err | ||
} | ||
res = append(res, compiledTask) | ||
} | ||
return res, nil | ||
} | ||
|
||
/* | ||
Utility to compile a packaged workflow locally. | ||
compilation is done locally so no flyte cluster is required. | ||
*/ | ||
func compileFromPackage(packagePath string) error { | ||
args := []string{packagePath} | ||
fileList, tmpDir, err := register.GetSerializeOutputFiles(context.Background(), args, true) | ||
defer os.RemoveAll(tmpDir) | ||
if err != nil { | ||
fmt.Println("Error found while extracting package..") | ||
return err | ||
} | ||
fmt.Println("Successfully extracted package...") | ||
fmt.Println("Processing Protobuf files...") | ||
workflows := make(map[string]*admin.WorkflowSpec) | ||
plans := make(map[string]*admin.LaunchPlan) | ||
tasks := []*admin.TaskSpec{} | ||
|
||
for _, pbFilePath := range fileList { | ||
rawTsk, err := ioutil.ReadFile(pbFilePath) | ||
if err != nil { | ||
fmt.Printf("error unmarshalling task..") | ||
return err | ||
} | ||
spec, err := register.UnMarshalContents(context.Background(), rawTsk, pbFilePath) | ||
if err != nil { | ||
return err | ||
} | ||
|
||
switch v := spec.(type) { | ||
case *admin.TaskSpec: | ||
tasks = append(tasks, v) | ||
case *admin.WorkflowSpec: | ||
workflows[v.Template.Id.Name] = v | ||
case *admin.LaunchPlan: | ||
plans[v.Id.Name] = v | ||
} | ||
} | ||
|
||
// compile tasks | ||
taskTemplates := []*core.TaskTemplate{} | ||
for _, task := range tasks { | ||
taskTemplates = append(taskTemplates, task.Template) | ||
} | ||
|
||
fmt.Println("\nCompiling tasks...") | ||
compiledTasks, err := compileTasks(taskTemplates) | ||
if err != nil { | ||
fmt.Println("Error while compiling tasks...") | ||
return err | ||
} | ||
|
||
// compile workflows | ||
for wfName, workflow := range workflows { | ||
|
||
fmt.Println("\nCompiling workflow:", wfName) | ||
plan := plans[wfName] | ||
|
||
_, err := compiler.CompileWorkflow(workflow.Template, | ||
workflow.SubWorkflows, | ||
compiledTasks, | ||
[]common.InterfaceProvider{compiler.NewLaunchPlanInterfaceProvider(*plan)}) | ||
if err != nil { | ||
fmt.Println(":( Error Compiling workflow:", wfName) | ||
return err | ||
} | ||
|
||
} | ||
|
||
fmt.Println("All Workflows compiled successfully!") | ||
fmt.Println("\nSummary:") | ||
fmt.Println(len(workflows), " workflows found in package") | ||
fmt.Println(len(tasks), " Tasks found in package") | ||
fmt.Println(len(plans), " Launch plans found in package") | ||
return nil | ||
} | ||
|
||
const ( | ||
compileShort = `Validate flyte packages without registration needed.` | ||
compileLong = ` | ||
Validate workflows by compiling flyte's serialized protobuf files (task, workflows and launch plans). This is useful for testing workflows and tasks without neededing to talk with a flyte cluster. | ||
:: | ||
flytectl compile --file my-flyte-package.tgz | ||
:: | ||
flytectl compile --file /home/user/dags/my-flyte-package.tgz | ||
.. note:: | ||
Input file is a path to a tgz. This file is generated by either pyflyte or jflyte. tgz file contains protobuf files describing workflows, tasks and launch plans. | ||
` | ||
) | ||
|
||
func compile(ctx context.Context, args []string, cmdCtx cmdCore.CommandContext) error { | ||
packageFilePath := config.DefaultCompileConfig.File | ||
if packageFilePath == "" { | ||
return fmt.Errorf("path to package tgz's file is a required flag") | ||
} | ||
return compileFromPackage(packageFilePath) | ||
} | ||
|
||
func CreateCompileCommand() map[string]cmdCore.CommandEntry { | ||
compileResourcesFuncs := map[string]cmdCore.CommandEntry{ | ||
"compile": { | ||
Short: compileShort, | ||
Long: compileLong, | ||
CmdFunc: compile, | ||
PFlagProvider: config.DefaultCompileConfig, | ||
ProjectDomainNotRequired: true, | ||
}, | ||
} | ||
return compileResourcesFuncs | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,71 @@ | ||
package compile | ||
|
||
import ( | ||
"context" | ||
"testing" | ||
|
||
config "github.com/flyteorg/flytectl/cmd/config/subcommand/compile" | ||
cmdCore "github.com/flyteorg/flytectl/cmd/core" | ||
u "github.com/flyteorg/flytectl/cmd/testutils" | ||
"github.com/spf13/cobra" | ||
"github.com/stretchr/testify/assert" | ||
) | ||
|
||
func TestCompileCommand(t *testing.T) { | ||
rootCmd := &cobra.Command{ | ||
Long: "Flytectl is a CLI tool written in Go to interact with the FlyteAdmin service.", | ||
Short: "Flytectl CLI tool", | ||
Use: "flytectl", | ||
DisableAutoGenTag: true, | ||
} | ||
compileCommand := CreateCompileCommand() | ||
cmdCore.AddCommands(rootCmd, compileCommand) | ||
cmdNouns := rootCmd.Commands() | ||
assert.Equal(t, cmdNouns[0].Use, "compile") | ||
assert.Equal(t, cmdNouns[0].Flags().Lookup("file").Name, "file") | ||
// check shorthand | ||
assert.Equal(t, cmdNouns[0].Short, compileShort) | ||
|
||
// compiling via cobra command | ||
compileCfg := config.DefaultCompileConfig | ||
compileCfg.File = "testdata/valid-package.tgz" | ||
var setup = u.Setup | ||
s := setup() | ||
compileCmd := CreateCompileCommand()["compile"] | ||
err := compileCmd.CmdFunc(context.Background(), []string{}, s.CmdCtx) | ||
assert.Nil(t, err, "compiling via cmd returns err") | ||
|
||
// calling command with empty file flag | ||
compileCfg = config.DefaultCompileConfig | ||
compileCfg.File = "" | ||
err = compileCmd.CmdFunc(context.Background(), []string{}, s.CmdCtx) | ||
assert.NotNil(t, err, "calling compile with Empty file flag does not error") | ||
} | ||
|
||
func TestCompilePackage(t *testing.T) { | ||
// valid package contains two workflows | ||
// with three tasks | ||
err := compileFromPackage("testdata/valid-package.tgz") | ||
assert.Nil(t, err, "unable to compile a valid package") | ||
|
||
// invalid gzip header | ||
err = compileFromPackage("testdata/invalid.tgz") | ||
assert.NotNil(t, err, "compiling an invalid package returns no error") | ||
|
||
// invalid workflow, types do not match | ||
err = compileFromPackage("testdata/bad-workflow-package.tgz") | ||
assert.NotNil(t, err, "compilin an invalid workflow returns no error") | ||
|
||
// testing badly serialized task | ||
err = compileFromPackage("testdata/invalidtask.tgz") | ||
assert.NotNil(t, err, "unable to handle invalid task") | ||
|
||
// testing badly serialized launchplan | ||
err = compileFromPackage("testdata/invalidlaunchplan.tgz") | ||
assert.NotNil(t, err, "unable to handle invalid launchplan") | ||
|
||
// testing badly serialized workflow | ||
err = compileFromPackage("testdata/invalidworkflow.tgz") | ||
assert.NotNil(t, err, "unable to handle invalid workflow") | ||
|
||
} |
Binary file not shown.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
invalid tgz file |
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,11 @@ | ||
package compile | ||
|
||
//go:generate pflags Config --default-var DefaultCompileConfig --bind-default-var | ||
var ( | ||
DefaultCompileConfig = &Config{} | ||
) | ||
|
||
// Config stores the flags required by compile command | ||
type Config struct { | ||
File string `json:"file" pflag:",Path to a flyte package file. Flyte packages are tgz files generated by pyflyte or jflyte."` | ||
} |
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Oops, something went wrong.
Oops, something went wrong.