diff --git a/pkg/compiler/requirements.go b/pkg/compiler/requirements.go index 20eeb5ded2..ab1b11a05c 100755 --- a/pkg/compiler/requirements.go +++ b/pkg/compiler/requirements.go @@ -9,24 +9,24 @@ import ( type TaskIdentifier = common.Identifier type LaunchPlanRefIdentifier = common.Identifier -// Represents the set of required resources for a given Workflow's execution. All of the resources should be loaded before -// hand and passed to the compiler. +// WorkflowExecutionRequirements represents the set of required resources for a given Workflow's execution. All the +// resources should be loaded beforehand and passed to the compiler. type WorkflowExecutionRequirements struct { taskIds []TaskIdentifier launchPlanIds []LaunchPlanRefIdentifier } -// Gets a slice of required Task ids to load. +// GetRequiredTaskIds gets a slice of required Task ids to load. func (g WorkflowExecutionRequirements) GetRequiredTaskIds() []TaskIdentifier { return g.taskIds } -// Gets a slice of required Workflow ids to load. +// GetRequiredLaunchPlanIds gets a slice of required Workflow ids to load. func (g WorkflowExecutionRequirements) GetRequiredLaunchPlanIds() []LaunchPlanRefIdentifier { return g.launchPlanIds } -// Computes requirements for a given Workflow. +// GetRequirements computes requirements for a given Workflow. func GetRequirements(fg *core.WorkflowTemplate, subWfs []*core.WorkflowTemplate) (reqs WorkflowExecutionRequirements, err error) { errs := errors.NewCompileErrors() compiledSubWfs := toCompiledWorkflows(subWfs...) diff --git a/pkg/compiler/task_compiler.go b/pkg/compiler/task_compiler.go index 84ed351d31..a310ee31f2 100644 --- a/pkg/compiler/task_compiler.go +++ b/pkg/compiler/task_compiler.go @@ -107,7 +107,7 @@ func compileTaskInternal(task *core.TaskTemplate, errs errors.CompileErrors) com return taskBuilder{flyteTask: task} } -// Task compiler compiles a given Task into an executable Task. It validates all required parameters and ensures a Task +// CompileTask compiles a given Task into an executable Task. It validates all required parameters and ensures a Task // is well-formed. func CompileTask(task *core.TaskTemplate) (*core.CompiledTask, error) { errs := errors.NewCompileErrors() diff --git a/pkg/compiler/test/compiler_test.go b/pkg/compiler/test/compiler_test.go index 0eedb14a4c..8f934aa280 100644 --- a/pkg/compiler/test/compiler_test.go +++ b/pkg/compiler/test/compiler_test.go @@ -3,12 +3,15 @@ package test import ( "encoding/json" "flag" - "fmt" "io/ioutil" "os" "path/filepath" + "reflect" "strings" "testing" + "time" + + "github.com/flyteorg/flytepropeller/pkg/visualize" "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin" @@ -24,14 +27,12 @@ import ( "github.com/flyteorg/flytepropeller/pkg/compiler/common" "github.com/flyteorg/flytepropeller/pkg/compiler/errors" "github.com/flyteorg/flytepropeller/pkg/compiler/transformers/k8s" - "github.com/flyteorg/flytepropeller/pkg/visualize" "github.com/golang/protobuf/jsonpb" "github.com/golang/protobuf/proto" "github.com/stretchr/testify/assert" ) var update = flag.Bool("update", false, "Update .golden files") -var reverse = flag.Bool("reverse", false, "Reverse .golden files") func makeDefaultInputs(iface *core.TypedInterface) *core.LiteralMap { if iface == nil || iface.GetInputs() == nil { @@ -40,8 +41,26 @@ func makeDefaultInputs(iface *core.TypedInterface) *core.LiteralMap { res := make(map[string]*core.Literal, len(iface.GetInputs().Variables)) for inputName, inputVar := range iface.GetInputs().Variables { - val := coreutils.MustMakeDefaultLiteralForType(inputVar.Type) - res[inputName] = val + // A workaround because the coreutils don't support the "StructuredDataSet" type + if reflect.TypeOf(inputVar.Type.Type) == reflect.TypeOf(&core.LiteralType_StructuredDatasetType{}) { + res[inputName] = &core.Literal{ + Value: &core.Literal_Scalar{ + Scalar: &core.Scalar{ + Value: &core.Scalar_StructuredDataset{ + StructuredDataset: &core.StructuredDataset{ + Metadata: &core.StructuredDatasetMetadata{ + StructuredDatasetType: inputVar.Type.Type.(*core.LiteralType_StructuredDatasetType).StructuredDatasetType, + }, + }, + }, + }, + }, + } + } else if reflect.TypeOf(inputVar.Type.Type) == reflect.TypeOf(&core.LiteralType_Simple{}) && inputVar.Type.GetSimple() == core.SimpleType_DATETIME { + res[inputName] = coreutils.MustMakeLiteral(time.UnixMicro(10)) + } else { + res[inputName] = coreutils.MustMakeDefaultLiteralForType(inputVar.Type) + } } return &core.LiteralMap{ @@ -85,28 +104,6 @@ func mustCompileTasks(t *testing.T, tasks []*core.TaskTemplate) []*core.Compiled return compiledTasks } -func marshalProto(t *testing.T, filename string, p proto.Message) { - marshaller := &jsonpb.Marshaler{} - s, err := marshaller.MarshalToString(p) - assert.NoError(t, err) - - if err != nil { - return - } - - originalRaw, err := proto.Marshal(p) - assert.NoError(t, err) - assert.NoError(t, ioutil.WriteFile(strings.Replace(filename, filepath.Ext(filename), ".pb", 1), originalRaw, os.ModePerm)) - - m := map[string]interface{}{} - err = json.Unmarshal([]byte(s), &m) - assert.NoError(t, err) - - b, err := yaml.Marshal(m) - assert.NoError(t, err) - assert.NoError(t, ioutil.WriteFile(strings.Replace(filename, filepath.Ext(filename), ".yaml", 1), b, os.ModePerm)) -} - func TestDynamic(t *testing.T) { errors.SetConfig(errors.Config{IncludeSource: true}) assert.NoError(t, filepath.Walk("testdata/dynamic", func(path string, info os.FileInfo, err error) error { @@ -161,13 +158,10 @@ func TestDynamic(t *testing.T) { t.FailNow() } - inputs := map[string]interface{}{} - for varName, v := range compiledWfc.Primary.Template.Interface.Inputs.Variables { - inputs[varName] = coreutils.MustMakeDefaultLiteralForType(v.Type) - } + inputs := makeDefaultInputs(compiledWfc.Primary.Template.Interface) flyteWf, err := k8s.BuildFlyteWorkflow(compiledWfc, - coreutils.MustMakeLiteral(inputs).GetMap(), + inputs, &core.WorkflowExecutionIdentifier{ Project: "hello", Domain: "domain", @@ -293,286 +287,219 @@ func assertNodeIDsInConnections(t testing.TB, nodeIDsWithDeps, allNodeIDs sets.S return correct } -func TestBranches(t *testing.T) { - errors.SetConfig(errors.Config{IncludeSource: true}) - assert.NoError(t, filepath.Walk("testdata/branch", func(path string, info os.FileInfo, err error) error { - if info.IsDir() { - if filepath.Base(info.Name()) != "branch" { - return filepath.SkipDir - } - - return nil - } +func TestUseCases(t *testing.T) { + runCompileTest(t, "branch") + runCompileTest(t, "snacks-core") +} - t.Run(path, func(t *testing.T) { - // If you want to debug a single use-case. Uncomment this line. - //if !strings.HasSuffix(path, "mycereal_condition_has_no_deps.json") { - // t.SkipNow() - //} +func protoMarshal(v any) ([]byte, error) { + m := jsonpb.Marshaler{} + str, err := m.MarshalToString(v.(proto.Message)) + return []byte(str), err +} - raw, err := ioutil.ReadFile(path) - assert.NoError(t, err) - wf := &core.WorkflowClosure{} - if filepath.Ext(path) == ".json" { - err = jsonpb.UnmarshalString(string(raw), wf) - if !assert.NoError(t, err) { - t.FailNow() - } - } else if filepath.Ext(path) == ".pb" { - m := &jsonpb.Marshaler{ - Indent: " ", - } +func storeOrDiff(t testing.TB, f func(obj any) ([]byte, error), obj any, path string) bool { + raw, err := f(obj) + if !assert.NoError(t, err) { + return false + } - err = proto.Unmarshal(raw, wf) - if !assert.NoError(t, err) { - tsk := &admin.TaskSpec{} - if !assert.NoError(t, proto.Unmarshal(raw, tsk)) { - t.FailNow() - } + if *update { + err = ioutil.WriteFile(path, raw, os.ModePerm) + if !assert.NoError(t, err) { + return false + } - raw, _ := m.MarshalToString(tsk) - err = ioutil.WriteFile(strings.TrimSuffix(path, filepath.Ext(path))+"_task.json", []byte(raw), os.ModePerm) - if !assert.NoError(t, err) { - t.FailNow() - } + } else { + goldenRaw, err := ioutil.ReadFile(path) + if !assert.NoError(t, err) { + return false + } - return - } + if diff := deep.Equal(string(raw), string(goldenRaw)); diff != nil { + t.Errorf("Compiled() Diff = %v\r\n got = %v\r\n want = %v", diff, string(raw), string(goldenRaw)) + } + } - raw, err := m.MarshalToString(wf) - if !assert.NoError(t, err) { - t.FailNow() - } + return true +} - err = ioutil.WriteFile(strings.TrimSuffix(path, filepath.Ext(path))+".json", []byte(raw), os.ModePerm) - if !assert.NoError(t, err) { - t.FailNow() - } - } +func runCompileTest(t *testing.T, dirName string) { + errors.SetConfig(errors.Config{IncludeSource: true}) + // Compile Tasks + t.Run("tasks-"+dirName, func(t *testing.T) { + //t.Parallel() - t.Log("Compiling Workflow") - compiledTasks := mustCompileTasks(t, wf.Tasks) - compiledWfc, err := compiler.CompileWorkflow(wf.Workflow, []*core.WorkflowTemplate{}, compiledTasks, - []common.InterfaceProvider{}) - if !assert.NoError(t, err) { - t.FailNow() - } + paths, err := filepath.Glob("testdata/" + dirName + "/*.pb") + if !assert.NoError(t, err) { + t.FailNow() + } - m := &jsonpb.Marshaler{ - Indent: " ", - } - rawStr, err := m.MarshalToString(compiledWfc) - if !assert.NoError(t, err) { - t.Fail() + for _, p := range paths { + raw, err := ioutil.ReadFile(p) + assert.NoError(t, err) + tsk := &admin.TaskSpec{} + err = proto.Unmarshal(raw, tsk) + if err != nil { + t.Logf("Failed to parse %s as a Task, skipping: %v", p, err) + continue } - compiledFilePath := filepath.Join(filepath.Dir(path), "compiled", filepath.Base(path)) - if *update { - err = ioutil.WriteFile(compiledFilePath, []byte(rawStr), os.ModePerm) - if !assert.NoError(t, err) { - t.Fail() - } - } else { - goldenRaw, err := ioutil.ReadFile(compiledFilePath) - if !assert.NoError(t, err) { - t.Fail() + t.Run(p, func(t *testing.T) { + //t.Parallel() + if !storeOrDiff(t, yaml.Marshal, tsk, strings.TrimSuffix(p, filepath.Ext(p))+"_task.yaml") { + t.FailNow() } - if diff := deep.Equal(rawStr, string(goldenRaw)); diff != nil { - t.Errorf("Compiled() Diff = %v\r\n got = %v\r\n want = %v", diff, rawStr, string(goldenRaw)) + inputTask := tsk.Template + setDefaultFields(inputTask) + task, err := compiler.CompileTask(inputTask) + if !assert.NoError(t, err) { + t.FailNow() } - } - - allNodeIDs := getAllMatchingNodes(compiledWfc.Primary, allNodesPredicate) - nodeIDsWithDeps := getAllMatchingNodes(compiledWfc.Primary, hasPromiseNodePredicate) - if !assertNodeIDsInConnections(t, nodeIDsWithDeps, allNodeIDs, compiledWfc.Primary.Connections) { - t.FailNow() - } - inputs := map[string]interface{}{} - for varName, v := range compiledWfc.Primary.Template.Interface.Inputs.Variables { - inputs[varName] = coreutils.MustMakeDefaultLiteralForType(v.Type) - } - - flyteWf, err := k8s.BuildFlyteWorkflow(compiledWfc, - coreutils.MustMakeLiteral(inputs).GetMap(), - &core.WorkflowExecutionIdentifier{ - Project: "hello", - Domain: "domain", - Name: "name", - }, - "namespace") - if assert.NoError(t, err) { - raw, err := json.MarshalIndent(flyteWf, "", " ") - if assert.NoError(t, err) { - assert.NotEmpty(t, raw) + if !storeOrDiff(t, yaml.Marshal, task, filepath.Join(filepath.Dir(p), "compiled", strings.TrimRight(filepath.Base(p), filepath.Ext(p))+"_task.yaml")) { + t.FailNow() } - k8sObjectFilepath := filepath.Join(filepath.Dir(path), "k8s", filepath.Base(path)) - if *update { - err = ioutil.WriteFile(k8sObjectFilepath, raw, os.ModePerm) - if !assert.NoError(t, err) { - t.Fail() - } - } else { - goldenRaw, err := ioutil.ReadFile(k8sObjectFilepath) - if !assert.NoError(t, err) { - t.Fail() - } - - if diff := deep.Equal(string(raw), string(goldenRaw)); diff != nil { - t.Errorf("K8sObject() Diff = %v\r\n got = %v\r\n want = %v", diff, rawStr, string(goldenRaw)) - } - + if !storeOrDiff(t, protoMarshal, tsk, filepath.Join(filepath.Dir(p), "compiled", strings.TrimRight(filepath.Base(p), filepath.Ext(p))+"_task.json")) { + t.FailNow() } - } - }) - - return nil - })) -} - -func TestReverseEngineerFromYaml(t *testing.T) { - root := "testdata" - errors.SetConfig(errors.Config{IncludeSource: true}) - assert.NoError(t, filepath.Walk(root, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err + }) } + }) - if info.IsDir() { - return nil - } + // Load Compiled Tasks + paths, err := filepath.Glob(filepath.Join("testdata", dirName, "compiled", "*_task.json")) + if !assert.NoError(t, err) { + t.FailNow() + } - if !strings.HasSuffix(path, ".yaml") { - return nil + compiledTasks := make(map[string]*core.CompiledTask, len(paths)) + for _, f := range paths { + raw, err := ioutil.ReadFile(f) + if !assert.NoError(t, err) { + t.FailNow() } - if strings.HasSuffix(path, "-inputs.yaml") { - return nil + tsk := &core.CompiledTask{} + err = jsonpb.UnmarshalString(string(raw), tsk) + if !assert.NoError(t, err) { + t.FailNow() } - ext := ".yaml" + compiledTasks[tsk.Template.Id.String()] = tsk + } - testName := strings.TrimLeft(path, root) - testName = strings.Trim(testName, string(os.PathSeparator)) - testName = strings.TrimSuffix(testName, ext) - testName = strings.Replace(testName, string(os.PathSeparator), "_", -1) + // Compile Workflows + t.Run("workflows-"+dirName, func(t *testing.T) { + //t.Parallel() - t.Run(testName, func(t *testing.T) { - t.Log("Reading from file") - raw, err := ioutil.ReadFile(path) - assert.NoError(t, err) + paths, err = filepath.Glob(filepath.Join("testdata", dirName, "*.pb")) + if !assert.NoError(t, err) { + t.FailNow() + } - raw, err = yaml.YAMLToJSON(raw) + for _, p := range paths { + raw, err := ioutil.ReadFile(p) assert.NoError(t, err) - - t.Log("Unmarshalling Workflow Closure") wf := &core.WorkflowClosure{} - err = jsonpb.UnmarshalString(string(raw), wf) - assert.NoError(t, err) - assert.NotNil(t, wf) + err = proto.Unmarshal(raw, wf) if err != nil { - return + t.Logf("Failed to parse %s as a WorkflowClosure, skipping: %v", p, err) + continue } - t.Log("Compiling Workflow") - compiledWf, err := compiler.CompileWorkflow(wf.Workflow, []*core.WorkflowTemplate{}, mustCompileTasks(t, wf.Tasks), []common.InterfaceProvider{}) - assert.NoError(t, err) - if err != nil { - return - } + t.Run(p, func(t *testing.T) { + //t.Parallel() + if !storeOrDiff(t, yaml.Marshal, wf, strings.TrimSuffix(p, filepath.Ext(p))+"_wf.yaml") { + t.FailNow() + } - inputs := makeDefaultInputs(compiledWf.Primary.Template.GetInterface()) - if *reverse { - marshalProto(t, strings.Replace(path, ext, fmt.Sprintf("-inputs%v", ext), -1), inputs) - } + inputWf := wf.Workflow - t.Log("Building k8s resource") - _, err = k8s.BuildFlyteWorkflow(compiledWf, inputs, nil, "") - assert.NoError(t, err) - if err != nil { - return - } + reqs, err := compiler.GetRequirements(inputWf, nil) + if !assert.NoError(t, err) { + t.FailNow() + } - dotFormat := visualize.ToGraphViz(compiledWf.Primary) - t.Logf("GraphViz Dot: %v\n", dotFormat) + tasks := make([]*core.CompiledTask, 0, len(reqs.GetRequiredTaskIds())) - if *reverse { - marshalProto(t, path, wf) - } - }) + for _, taskID := range reqs.GetRequiredTaskIds() { + compiledTask, found := compiledTasks[taskID.String()] + if !assert.True(t, found, "Could not find compiled task %s", taskID) { + t.FailNow() + } - return nil - })) -} + tasks = append(tasks, compiledTask) + } -func TestCompileAndBuild(t *testing.T) { - root := "testdata" - errors.SetConfig(errors.Config{IncludeSource: true}) - assert.NoError(t, filepath.Walk(root, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } + compiledWfc, err := compiler.CompileWorkflow(inputWf, []*core.WorkflowTemplate{}, tasks, + []common.InterfaceProvider{}) + if !assert.NoError(t, err) { + t.FailNow() + } - if info.IsDir() { - return nil - } + if !storeOrDiff(t, yaml.Marshal, compiledWfc, filepath.Join(filepath.Dir(p), "compiled", strings.TrimRight(filepath.Base(p), filepath.Ext(p))+"_wf.yaml")) { + t.FailNow() + } - if ext := filepath.Ext(path); ext != ".pb" { - return nil - } + if !storeOrDiff(t, protoMarshal, compiledWfc, filepath.Join(filepath.Dir(p), "compiled", strings.TrimRight(filepath.Base(p), filepath.Ext(p))+"_wf.json")) { + t.FailNow() + } - if strings.HasSuffix(path, "-inputs.pb") { - return nil + allNodeIDs := getAllMatchingNodes(compiledWfc.Primary, allNodesPredicate) + nodeIDsWithDeps := getAllMatchingNodes(compiledWfc.Primary, hasPromiseNodePredicate) + if !assertNodeIDsInConnections(t, nodeIDsWithDeps, allNodeIDs, compiledWfc.Primary.Connections) { + t.FailNow() + } + }) } + }) - testName := strings.TrimLeft(path, root) - testName = strings.Trim(testName, string(os.PathSeparator)) - testName = strings.Trim(testName, filepath.Ext(testName)) - testName = strings.Replace(testName, string(os.PathSeparator), "_", -1) - - t.Run(testName, func(t *testing.T) { - t.Log("Reading from file") - raw, err := ioutil.ReadFile(path) - assert.NoError(t, err) + // Build K8s Workflows + t.Run("k8s-"+dirName, func(t *testing.T) { + //t.Parallel() - t.Log("Unmarshalling Workflow Closure") - wf := &core.WorkflowClosure{} - err = proto.Unmarshal(raw, wf) - assert.NoError(t, err) - assert.NotNil(t, wf) - if err != nil { - return - } + paths, err = filepath.Glob(filepath.Join("testdata", dirName, "compiled", "*_wf.json")) + if !assert.NoError(t, err) { + t.FailNow() + } - t.Log("Compiling Workflow") - compiledWf, err := compiler.CompileWorkflow(wf.Workflow, []*core.WorkflowTemplate{}, mustCompileTasks(t, wf.Tasks), []common.InterfaceProvider{}) - assert.NoError(t, err) - if err != nil { - return - } + for _, p := range paths { + t.Run(p, func(t *testing.T) { + //t.Parallel() + raw, err := ioutil.ReadFile(p) + if !assert.NoError(t, err) { + t.FailNow() + } - inputs := makeDefaultInputs(compiledWf.Primary.Template.GetInterface()) - if *update { - marshalProto(t, strings.Replace(path, filepath.Ext(path), fmt.Sprintf("-inputs%v", filepath.Ext(path)), -1), inputs) - } + compiledWfc := &core.CompiledWorkflowClosure{} + if !assert.NoError(t, jsonpb.UnmarshalString(string(raw), compiledWfc)) { + t.FailNow() + } - t.Log("Building k8s resource") - _, err = k8s.BuildFlyteWorkflow(compiledWf, inputs, nil, "") - assert.NoError(t, err) - if err != nil { - return - } + inputs := makeDefaultInputs(compiledWfc.Primary.Template.Interface) - dotFormat := visualize.ToGraphViz(compiledWf.Primary) - t.Logf("GraphViz Dot: %v\n", dotFormat) + dotFormat := visualize.ToGraphViz(compiledWfc.Primary) + t.Logf("GraphViz Dot: %v\n", dotFormat) - if *update { - marshalProto(t, path, wf) - } - }) + flyteWf, err := k8s.BuildFlyteWorkflow(compiledWfc, + inputs, + &core.WorkflowExecutionIdentifier{ + Project: "hello", + Domain: "domain", + Name: "name", + }, + "namespace") + if !assert.NoError(t, err) { + t.FailNow() + } - return nil - })) + if !storeOrDiff(t, yaml.Marshal, flyteWf, filepath.Join(filepath.Dir(filepath.Dir(p)), "k8s", strings.TrimRight(filepath.Base(p), filepath.Ext(p))+".yaml")) { + t.FailNow() + } + }) + } + }) } diff --git a/pkg/compiler/test/testdata/app-workflows-work-one-python-task-w-f.pb b/pkg/compiler/test/testdata/app-workflows-work-one-python-task-w-f.pb index e76e479064..1ef935e330 100755 Binary files a/pkg/compiler/test/testdata/app-workflows-work-one-python-task-w-f.pb and b/pkg/compiler/test/testdata/app-workflows-work-one-python-task-w-f.pb differ diff --git a/pkg/compiler/test/testdata/app-workflows-work-one-python-task-w-f.yaml b/pkg/compiler/test/testdata/app-workflows-work-one-python-task-w-f.yaml index 4da5845fe7..9f5b224d98 100644 --- a/pkg/compiler/test/testdata/app-workflows-work-one-python-task-w-f.yaml +++ b/pkg/compiler/test/testdata/app-workflows-work-one-python-task-w-f.yaml @@ -66,6 +66,18 @@ tasks: value: testValue2 - key: testKey3 value: testValue3 + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 image: myflytecontainer:abc123 resources: {} id: diff --git a/pkg/compiler/test/testdata/snacks-core/000_core.containerization.multi_images.svm_trainer_1.pb b/pkg/compiler/test/testdata/snacks-core/000_core.containerization.multi_images.svm_trainer_1.pb new file mode 100644 index 0000000000..6713b4079a Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/000_core.containerization.multi_images.svm_trainer_1.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/000_core.containerization.multi_images.svm_trainer_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/000_core.containerization.multi_images.svm_trainer_1_task.yaml new file mode 100755 index 0000000000..b199a06874 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/000_core.containerization.multi_images.svm_trainer_1_task.yaml @@ -0,0 +1,63 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.containerization.multi_images + - task-name + - svm_trainer + image: ghcr.io/flyteorg/flytecookbook:core-with-sklearn-baa17ccf39aa667c5950bd713a4366ce7d5fccaf7f85e6be8c07fe4b522f92c3 + resources: {} + id: + name: core.containerization.multi_images.svm_trainer + resource_type: 1 + interface: + inputs: {} + outputs: + variables: + test_features: + description: test_features + type: + Type: + StructuredDatasetType: + format: parquet + test_labels: + description: test_labels + type: + Type: + StructuredDatasetType: + format: parquet + train_features: + description: train_features + type: + Type: + StructuredDatasetType: + format: parquet + train_labels: + description: train_labels + type: + Type: + StructuredDatasetType: + format: parquet + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/001_core.containerization.multi_images.svm_predictor_1.pb b/pkg/compiler/test/testdata/snacks-core/001_core.containerization.multi_images.svm_predictor_1.pb new file mode 100644 index 0000000000..718726930a Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/001_core.containerization.multi_images.svm_predictor_1.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/001_core.containerization.multi_images.svm_predictor_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/001_core.containerization.multi_images.svm_predictor_1_task.yaml new file mode 100755 index 0000000000..70b3faff85 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/001_core.containerization.multi_images.svm_predictor_1_task.yaml @@ -0,0 +1,69 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.containerization.multi_images + - task-name + - svm_predictor + image: ghcr.io/flyteorg/flytecookbook:multi-image-predict-98b125fd57d20594026941c2ebe7ef662e5acb7d6423660a65f493ca2d9aa267 + resources: {} + id: + name: core.containerization.multi_images.svm_predictor + resource_type: 1 + interface: + inputs: + variables: + X_test: + description: X_test + type: + Type: + StructuredDatasetType: + format: parquet + X_train: + description: X_train + type: + Type: + StructuredDatasetType: + format: parquet + y_test: + description: y_test + type: + Type: + StructuredDatasetType: + format: parquet + y_train: + description: y_train + type: + Type: + StructuredDatasetType: + format: parquet + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 2 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/002_core.containerization.multi_images.my_workflow_2.pb b/pkg/compiler/test/testdata/snacks-core/002_core.containerization.multi_images.my_workflow_2.pb new file mode 100644 index 0000000000..aae6e676a1 Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/002_core.containerization.multi_images.my_workflow_2.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/002_core.containerization.multi_images.my_workflow_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/002_core.containerization.multi_images.my_workflow_2_wf.yaml new file mode 100755 index 0000000000..0f3d4e5199 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/002_core.containerization.multi_images.my_workflow_2_wf.yaml @@ -0,0 +1,74 @@ +workflow: + id: + name: core.containerization.multi_images.my_workflow + resource_type: 2 + interface: + inputs: {} + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 2 + metadata: {} + metadata_defaults: {} + nodes: + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.containerization.multi_images.svm_trainer + resource_type: 1 + overrides: {} + id: n0 + metadata: + InterruptibleValue: null + name: svm_trainer + retries: {} + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.containerization.multi_images.svm_predictor + resource_type: 1 + overrides: {} + id: n1 + inputs: + - binding: + Value: + Promise: + node_id: n0 + var: test_features + var: X_test + - binding: + Value: + Promise: + node_id: n0 + var: train_features + var: X_train + - binding: + Value: + Promise: + node_id: n0 + var: test_labels + var: y_test + - binding: + Value: + Promise: + node_id: n0 + var: train_labels + var: y_train + metadata: + InterruptibleValue: null + name: svm_predictor + retries: {} + upstream_node_ids: + - n0 + outputs: + - binding: + Value: + Promise: + node_id: n1 + var: o0 + var: o0 diff --git a/pkg/compiler/test/testdata/snacks-core/003_core.containerization.multi_images.my_workflow_3.pb b/pkg/compiler/test/testdata/snacks-core/003_core.containerization.multi_images.my_workflow_3.pb new file mode 100644 index 0000000000..072c889a20 Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/003_core.containerization.multi_images.my_workflow_3.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/004_ellipse-area-metadata-shell_1.pb b/pkg/compiler/test/testdata/snacks-core/004_ellipse-area-metadata-shell_1.pb new file mode 100644 index 0000000000..f76b871de8 Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/004_ellipse-area-metadata-shell_1.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/004_ellipse-area-metadata-shell_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/004_ellipse-area-metadata-shell_1_task.yaml new file mode 100755 index 0000000000..3d6250a030 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/004_ellipse-area-metadata-shell_1_task.yaml @@ -0,0 +1,49 @@ +template: + Target: + Container: + command: + - ./calculate-ellipse-area.sh + - /var/inputs + - /var/outputs + data_config: + enabled: true + input_path: /var/inputs + output_path: /var/outputs + image: ghcr.io/flyteorg/rawcontainers-shell:v1 + resources: {} + id: + name: ellipse-area-metadata-shell + resource_type: 1 + interface: + inputs: + variables: + a: + description: a + type: + Type: + Simple: 2 + b: + description: b + type: + Type: + Simple: 2 + outputs: + variables: + area: + description: area + type: + Type: + Simple: 2 + metadata: + description: metadata + type: + Type: + Simple: 3 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: raw-container diff --git a/pkg/compiler/test/testdata/snacks-core/005_ellipse-area-metadata-python_1.pb b/pkg/compiler/test/testdata/snacks-core/005_ellipse-area-metadata-python_1.pb new file mode 100644 index 0000000000..5f8c796413 Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/005_ellipse-area-metadata-python_1.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/005_ellipse-area-metadata-python_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/005_ellipse-area-metadata-python_1_task.yaml new file mode 100755 index 0000000000..f395e1f6d7 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/005_ellipse-area-metadata-python_1_task.yaml @@ -0,0 +1,50 @@ +template: + Target: + Container: + command: + - python + - calculate-ellipse-area.py + - /var/inputs + - /var/outputs + data_config: + enabled: true + input_path: /var/inputs + output_path: /var/outputs + image: ghcr.io/flyteorg/rawcontainers-python:v1 + resources: {} + id: + name: ellipse-area-metadata-python + resource_type: 1 + interface: + inputs: + variables: + a: + description: a + type: + Type: + Simple: 2 + b: + description: b + type: + Type: + Simple: 2 + outputs: + variables: + area: + description: area + type: + Type: + Simple: 2 + metadata: + description: metadata + type: + Type: + Simple: 3 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: raw-container diff --git a/pkg/compiler/test/testdata/snacks-core/006_ellipse-area-metadata-r_1.pb b/pkg/compiler/test/testdata/snacks-core/006_ellipse-area-metadata-r_1.pb new file mode 100644 index 0000000000..279461ad1e Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/006_ellipse-area-metadata-r_1.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/006_ellipse-area-metadata-r_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/006_ellipse-area-metadata-r_1_task.yaml new file mode 100755 index 0000000000..b491ef15e3 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/006_ellipse-area-metadata-r_1_task.yaml @@ -0,0 +1,51 @@ +template: + Target: + Container: + command: + - Rscript + - --vanilla + - calculate-ellipse-area.R + - /var/inputs + - /var/outputs + data_config: + enabled: true + input_path: /var/inputs + output_path: /var/outputs + image: ghcr.io/flyteorg/rawcontainers-r:v1 + resources: {} + id: + name: ellipse-area-metadata-r + resource_type: 1 + interface: + inputs: + variables: + a: + description: a + type: + Type: + Simple: 2 + b: + description: b + type: + Type: + Simple: 2 + outputs: + variables: + area: + description: area + type: + Type: + Simple: 2 + metadata: + description: metadata + type: + Type: + Simple: 3 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: raw-container diff --git a/pkg/compiler/test/testdata/snacks-core/007_ellipse-area-metadata-haskell_1.pb b/pkg/compiler/test/testdata/snacks-core/007_ellipse-area-metadata-haskell_1.pb new file mode 100644 index 0000000000..b34c493b66 Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/007_ellipse-area-metadata-haskell_1.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/007_ellipse-area-metadata-haskell_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/007_ellipse-area-metadata-haskell_1_task.yaml new file mode 100755 index 0000000000..cf515ee089 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/007_ellipse-area-metadata-haskell_1_task.yaml @@ -0,0 +1,49 @@ +template: + Target: + Container: + command: + - ./calculate-ellipse-area + - /var/inputs + - /var/outputs + data_config: + enabled: true + input_path: /var/inputs + output_path: /var/outputs + image: ghcr.io/flyteorg/rawcontainers-haskell:v1 + resources: {} + id: + name: ellipse-area-metadata-haskell + resource_type: 1 + interface: + inputs: + variables: + a: + description: a + type: + Type: + Simple: 2 + b: + description: b + type: + Type: + Simple: 2 + outputs: + variables: + area: + description: area + type: + Type: + Simple: 2 + metadata: + description: metadata + type: + Type: + Simple: 3 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: raw-container diff --git a/pkg/compiler/test/testdata/snacks-core/008_ellipse-area-metadata-julia_1.pb b/pkg/compiler/test/testdata/snacks-core/008_ellipse-area-metadata-julia_1.pb new file mode 100644 index 0000000000..a3b335928a Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/008_ellipse-area-metadata-julia_1.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/008_ellipse-area-metadata-julia_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/008_ellipse-area-metadata-julia_1_task.yaml new file mode 100755 index 0000000000..32c5b68826 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/008_ellipse-area-metadata-julia_1_task.yaml @@ -0,0 +1,50 @@ +template: + Target: + Container: + command: + - julia + - calculate-ellipse-area.jl + - /var/inputs + - /var/outputs + data_config: + enabled: true + input_path: /var/inputs + output_path: /var/outputs + image: ghcr.io/flyteorg/rawcontainers-julia:v1 + resources: {} + id: + name: ellipse-area-metadata-julia + resource_type: 1 + interface: + inputs: + variables: + a: + description: a + type: + Type: + Simple: 2 + b: + description: b + type: + Type: + Simple: 2 + outputs: + variables: + area: + description: area + type: + Type: + Simple: 2 + metadata: + description: metadata + type: + Type: + Simple: 3 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: raw-container diff --git a/pkg/compiler/test/testdata/snacks-core/009_core.containerization.raw_container.report_all_calculated_areas_1.pb b/pkg/compiler/test/testdata/snacks-core/009_core.containerization.raw_container.report_all_calculated_areas_1.pb new file mode 100644 index 0000000000..0e38b3ab0c Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/009_core.containerization.raw_container.report_all_calculated_areas_1.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/009_core.containerization.raw_container.report_all_calculated_areas_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/009_core.containerization.raw_container.report_all_calculated_areas_1_task.yaml new file mode 100755 index 0000000000..6fa4a9f509 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/009_core.containerization.raw_container.report_all_calculated_areas_1_task.yaml @@ -0,0 +1,89 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.containerization.raw_container + - task-name + - report_all_calculated_areas + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.containerization.raw_container.report_all_calculated_areas + resource_type: 1 + interface: + inputs: + variables: + area_haskell: + description: area_haskell + type: + Type: + Simple: 2 + area_julia: + description: area_julia + type: + Type: + Simple: 2 + area_python: + description: area_python + type: + Type: + Simple: 2 + area_r: + description: area_r + type: + Type: + Simple: 2 + area_shell: + description: area_shell + type: + Type: + Simple: 2 + metadata_haskell: + description: metadata_haskell + type: + Type: + Simple: 3 + metadata_julia: + description: metadata_julia + type: + Type: + Simple: 3 + metadata_python: + description: metadata_python + type: + Type: + Simple: 3 + metadata_r: + description: metadata_r + type: + Type: + Simple: 3 + metadata_shell: + description: metadata_shell + type: + Type: + Simple: 3 + outputs: {} + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/010_core.containerization.raw_container.wf_2.pb b/pkg/compiler/test/testdata/snacks-core/010_core.containerization.raw_container.wf_2.pb new file mode 100644 index 0000000000..7535ee26ca Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/010_core.containerization.raw_container.wf_2.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/010_core.containerization.raw_container.wf_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/010_core.containerization.raw_container.wf_2_wf.yaml new file mode 100755 index 0000000000..1afa966d41 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/010_core.containerization.raw_container.wf_2_wf.yaml @@ -0,0 +1,215 @@ +workflow: + id: + name: core.containerization.raw_container.wf + resource_type: 2 + interface: + inputs: + variables: + a: + description: a + type: + Type: + Simple: 2 + b: + description: b + type: + Type: + Simple: 2 + outputs: {} + metadata: {} + metadata_defaults: {} + nodes: + - Target: + TaskNode: + Reference: + ReferenceId: + name: ellipse-area-metadata-shell + resource_type: 1 + overrides: {} + id: n0 + inputs: + - binding: + Value: + Promise: + var: a + var: a + - binding: + Value: + Promise: + var: b + var: b + metadata: + InterruptibleValue: null + name: ellipse-area-metadata-shell + retries: {} + - Target: + TaskNode: + Reference: + ReferenceId: + name: ellipse-area-metadata-python + resource_type: 1 + overrides: {} + id: n1 + inputs: + - binding: + Value: + Promise: + var: a + var: a + - binding: + Value: + Promise: + var: b + var: b + metadata: + InterruptibleValue: null + name: ellipse-area-metadata-python + retries: {} + - Target: + TaskNode: + Reference: + ReferenceId: + name: ellipse-area-metadata-r + resource_type: 1 + overrides: {} + id: n2 + inputs: + - binding: + Value: + Promise: + var: a + var: a + - binding: + Value: + Promise: + var: b + var: b + metadata: + InterruptibleValue: null + name: ellipse-area-metadata-r + retries: {} + - Target: + TaskNode: + Reference: + ReferenceId: + name: ellipse-area-metadata-haskell + resource_type: 1 + overrides: {} + id: n3 + inputs: + - binding: + Value: + Promise: + var: a + var: a + - binding: + Value: + Promise: + var: b + var: b + metadata: + InterruptibleValue: null + name: ellipse-area-metadata-haskell + retries: {} + - Target: + TaskNode: + Reference: + ReferenceId: + name: ellipse-area-metadata-julia + resource_type: 1 + overrides: {} + id: n4 + inputs: + - binding: + Value: + Promise: + var: a + var: a + - binding: + Value: + Promise: + var: b + var: b + metadata: + InterruptibleValue: null + name: ellipse-area-metadata-julia + retries: {} + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.containerization.raw_container.report_all_calculated_areas + resource_type: 1 + overrides: {} + id: n5 + inputs: + - binding: + Value: + Promise: + node_id: n3 + var: area + var: area_haskell + - binding: + Value: + Promise: + node_id: n4 + var: area + var: area_julia + - binding: + Value: + Promise: + node_id: n1 + var: area + var: area_python + - binding: + Value: + Promise: + node_id: n2 + var: area + var: area_r + - binding: + Value: + Promise: + node_id: n0 + var: area + var: area_shell + - binding: + Value: + Promise: + node_id: n3 + var: metadata + var: metadata_haskell + - binding: + Value: + Promise: + node_id: n4 + var: metadata + var: metadata_julia + - binding: + Value: + Promise: + node_id: n1 + var: metadata + var: metadata_python + - binding: + Value: + Promise: + node_id: n2 + var: metadata + var: metadata_r + - binding: + Value: + Promise: + node_id: n0 + var: metadata + var: metadata_shell + metadata: + InterruptibleValue: null + name: report_all_calculated_areas + retries: {} + upstream_node_ids: + - n0 + - n1 + - n2 + - n4 + - n3 diff --git a/pkg/compiler/test/testdata/snacks-core/011_core.containerization.raw_container.wf_3.pb b/pkg/compiler/test/testdata/snacks-core/011_core.containerization.raw_container.wf_3.pb new file mode 100644 index 0000000000..55fc80a0e8 Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/011_core.containerization.raw_container.wf_3.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/012_core.containerization.use_secrets.secret_task_1.pb b/pkg/compiler/test/testdata/snacks-core/012_core.containerization.use_secrets.secret_task_1.pb new file mode 100644 index 0000000000..aae8aaac53 Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/012_core.containerization.use_secrets.secret_task_1.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/012_core.containerization.use_secrets.secret_task_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/012_core.containerization.use_secrets.secret_task_1_task.yaml new file mode 100755 index 0000000000..370bebdc1c --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/012_core.containerization.use_secrets.secret_task_1_task.yaml @@ -0,0 +1,48 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.containerization.use_secrets + - task-name + - secret_task + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.containerization.use_secrets.secret_task + resource_type: 1 + interface: + inputs: {} + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 3 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + security_context: + secrets: + - group: user-info + key: user_secret + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/013_core.containerization.use_secrets.user_info_task_1.pb b/pkg/compiler/test/testdata/snacks-core/013_core.containerization.use_secrets.user_info_task_1.pb new file mode 100644 index 0000000000..e1b2501b7b Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/013_core.containerization.use_secrets.user_info_task_1.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/013_core.containerization.use_secrets.user_info_task_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/013_core.containerization.use_secrets.user_info_task_1_task.yaml new file mode 100755 index 0000000000..eb509172f6 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/013_core.containerization.use_secrets.user_info_task_1_task.yaml @@ -0,0 +1,55 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.containerization.use_secrets + - task-name + - user_info_task + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.containerization.use_secrets.user_info_task + resource_type: 1 + interface: + inputs: {} + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 3 + o1: + description: o1 + type: + Type: + Simple: 3 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + security_context: + secrets: + - group: user-info + key: username + - group: user-info + key: password + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/014_core.containerization.use_secrets.secret_file_task_1.pb b/pkg/compiler/test/testdata/snacks-core/014_core.containerization.use_secrets.secret_file_task_1.pb new file mode 100644 index 0000000000..de310721b9 Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/014_core.containerization.use_secrets.secret_file_task_1.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/014_core.containerization.use_secrets.secret_file_task_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/014_core.containerization.use_secrets.secret_file_task_1_task.yaml new file mode 100755 index 0000000000..417db47255 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/014_core.containerization.use_secrets.secret_file_task_1_task.yaml @@ -0,0 +1,54 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.containerization.use_secrets + - task-name + - secret_file_task + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.containerization.use_secrets.secret_file_task + resource_type: 1 + interface: + inputs: {} + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 3 + o1: + description: o1 + type: + Type: + Simple: 3 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + security_context: + secrets: + - group: user-info + key: user_secret + mount_requirement: 1 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/015_core.containerization.use_secrets.my_secret_workflow_2.pb b/pkg/compiler/test/testdata/snacks-core/015_core.containerization.use_secrets.my_secret_workflow_2.pb new file mode 100644 index 0000000000..138c8e04f2 Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/015_core.containerization.use_secrets.my_secret_workflow_2.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/015_core.containerization.use_secrets.my_secret_workflow_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/015_core.containerization.use_secrets.my_secret_workflow_2_wf.yaml new file mode 100755 index 0000000000..4054a0494d --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/015_core.containerization.use_secrets.my_secret_workflow_2_wf.yaml @@ -0,0 +1,103 @@ +workflow: + id: + name: core.containerization.use_secrets.my_secret_workflow + resource_type: 2 + interface: + inputs: {} + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 3 + o1: + description: o1 + type: + Type: + Simple: 3 + o2: + description: o2 + type: + Type: + Simple: 3 + o3: + description: o3 + type: + Type: + Simple: 3 + o4: + description: o4 + type: + Type: + Simple: 3 + metadata: {} + metadata_defaults: {} + nodes: + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.containerization.use_secrets.secret_task + resource_type: 1 + overrides: {} + id: n0 + metadata: + InterruptibleValue: null + name: secret_task + retries: {} + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.containerization.use_secrets.user_info_task + resource_type: 1 + overrides: {} + id: n1 + metadata: + InterruptibleValue: null + name: user_info_task + retries: {} + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.containerization.use_secrets.secret_file_task + resource_type: 1 + overrides: {} + id: n2 + metadata: + InterruptibleValue: null + name: secret_file_task + retries: {} + outputs: + - binding: + Value: + Promise: + node_id: n0 + var: o0 + var: o0 + - binding: + Value: + Promise: + node_id: n1 + var: o0 + var: o1 + - binding: + Value: + Promise: + node_id: n1 + var: o1 + var: o2 + - binding: + Value: + Promise: + node_id: n2 + var: o0 + var: o3 + - binding: + Value: + Promise: + node_id: n2 + var: o1 + var: o4 diff --git a/pkg/compiler/test/testdata/snacks-core/016_core.containerization.use_secrets.my_secret_workflow_3.pb b/pkg/compiler/test/testdata/snacks-core/016_core.containerization.use_secrets.my_secret_workflow_3.pb new file mode 100644 index 0000000000..127fd71302 Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/016_core.containerization.use_secrets.my_secret_workflow_3.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/017_core.control_flow.chain_tasks.read_1.pb b/pkg/compiler/test/testdata/snacks-core/017_core.control_flow.chain_tasks.read_1.pb new file mode 100644 index 0000000000..8bff3316ac Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/017_core.control_flow.chain_tasks.read_1.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/017_core.control_flow.chain_tasks.read_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/017_core.control_flow.chain_tasks.read_1_task.yaml new file mode 100755 index 0000000000..2923c59e00 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/017_core.control_flow.chain_tasks.read_1_task.yaml @@ -0,0 +1,45 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.control_flow.chain_tasks + - task-name + - read + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.control_flow.chain_tasks.read + resource_type: 1 + interface: + inputs: {} + outputs: + variables: + o0: + description: o0 + type: + Type: + StructuredDatasetType: + format: parquet + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/018_core.control_flow.chain_tasks.write_1.pb b/pkg/compiler/test/testdata/snacks-core/018_core.control_flow.chain_tasks.write_1.pb new file mode 100644 index 0000000000..5964170968 Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/018_core.control_flow.chain_tasks.write_1.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/018_core.control_flow.chain_tasks.write_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/018_core.control_flow.chain_tasks.write_1_task.yaml new file mode 100755 index 0000000000..fa6dc89687 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/018_core.control_flow.chain_tasks.write_1_task.yaml @@ -0,0 +1,38 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.control_flow.chain_tasks + - task-name + - write + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.control_flow.chain_tasks.write + resource_type: 1 + interface: + inputs: {} + outputs: {} + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/019_core.control_flow.chain_tasks.chain_tasks_wf_2.pb b/pkg/compiler/test/testdata/snacks-core/019_core.control_flow.chain_tasks.chain_tasks_wf_2.pb new file mode 100644 index 0000000000..cd2cb0ab2a Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/019_core.control_flow.chain_tasks.chain_tasks_wf_2.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/019_core.control_flow.chain_tasks.chain_tasks_wf_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/019_core.control_flow.chain_tasks.chain_tasks_wf_2_wf.yaml new file mode 100755 index 0000000000..29e803bed3 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/019_core.control_flow.chain_tasks.chain_tasks_wf_2_wf.yaml @@ -0,0 +1,50 @@ +workflow: + id: + name: core.control_flow.chain_tasks.chain_tasks_wf + resource_type: 2 + interface: + inputs: {} + outputs: + variables: + o0: + description: o0 + type: + Type: + StructuredDatasetType: + format: parquet + metadata: {} + metadata_defaults: {} + nodes: + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.control_flow.chain_tasks.write + resource_type: 1 + overrides: {} + id: n0 + metadata: + InterruptibleValue: null + name: write + retries: {} + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.control_flow.chain_tasks.read + resource_type: 1 + overrides: {} + id: n1 + metadata: + InterruptibleValue: null + name: read + retries: {} + upstream_node_ids: + - n0 + outputs: + - binding: + Value: + Promise: + node_id: n1 + var: o0 + var: o0 diff --git a/pkg/compiler/test/testdata/snacks-core/020_core.control_flow.chain_tasks.chain_tasks_wf_3.pb b/pkg/compiler/test/testdata/snacks-core/020_core.control_flow.chain_tasks.chain_tasks_wf_3.pb new file mode 100644 index 0000000000..7de3c3c568 Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/020_core.control_flow.chain_tasks.chain_tasks_wf_3.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/021_core.control_flow.checkpoint.use_checkpoint_1.pb b/pkg/compiler/test/testdata/snacks-core/021_core.control_flow.checkpoint.use_checkpoint_1.pb new file mode 100644 index 0000000000..2d3fea58b5 Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/021_core.control_flow.checkpoint.use_checkpoint_1.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/021_core.control_flow.checkpoint.use_checkpoint_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/021_core.control_flow.checkpoint.use_checkpoint_1_task.yaml new file mode 100755 index 0000000000..97d873f7d3 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/021_core.control_flow.checkpoint.use_checkpoint_1_task.yaml @@ -0,0 +1,51 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.control_flow.checkpoint + - task-name + - use_checkpoint + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.control_flow.checkpoint.use_checkpoint + resource_type: 1 + interface: + inputs: + variables: + n_iterations: + description: n_iterations + type: + Type: + Simple: 1 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 1 + metadata: + InterruptibleValue: null + retries: + retries: 3 + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/022_core.control_flow.checkpoint.example_2.pb b/pkg/compiler/test/testdata/snacks-core/022_core.control_flow.checkpoint.example_2.pb new file mode 100644 index 0000000000..560fb1eff5 Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/022_core.control_flow.checkpoint.example_2.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/022_core.control_flow.checkpoint.example_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/022_core.control_flow.checkpoint.example_2_wf.yaml new file mode 100755 index 0000000000..22173f71c6 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/022_core.control_flow.checkpoint.example_2_wf.yaml @@ -0,0 +1,48 @@ +workflow: + id: + name: core.control_flow.checkpoint.example + resource_type: 2 + interface: + inputs: + variables: + n_iterations: + description: n_iterations + type: + Type: + Simple: 1 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 1 + metadata: {} + metadata_defaults: {} + nodes: + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.control_flow.checkpoint.use_checkpoint + resource_type: 1 + overrides: {} + id: n0 + inputs: + - binding: + Value: + Promise: + var: n_iterations + var: n_iterations + metadata: + InterruptibleValue: null + name: use_checkpoint + retries: + retries: 3 + outputs: + - binding: + Value: + Promise: + node_id: n0 + var: o0 + var: o0 diff --git a/pkg/compiler/test/testdata/snacks-core/023_core.control_flow.checkpoint.example_3.pb b/pkg/compiler/test/testdata/snacks-core/023_core.control_flow.checkpoint.example_3.pb new file mode 100644 index 0000000000..2cfa5c6f2e Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/023_core.control_flow.checkpoint.example_3.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/024_core.control_flow.conditions.square_1.pb b/pkg/compiler/test/testdata/snacks-core/024_core.control_flow.conditions.square_1.pb new file mode 100644 index 0000000000..4b738705b4 Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/024_core.control_flow.conditions.square_1.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/024_core.control_flow.conditions.square_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/024_core.control_flow.conditions.square_1_task.yaml new file mode 100755 index 0000000000..740868a86a --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/024_core.control_flow.conditions.square_1_task.yaml @@ -0,0 +1,52 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.control_flow.conditions + - task-name + - square + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.control_flow.conditions.square + resource_type: 1 + interface: + inputs: + variables: + "n": + description: |- + name of the parameter for the task is derived from the name of the input variable, and + the type is automatically mapped to Types.Integer + type: + Type: + Simple: 2 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 2 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/025_core.control_flow.conditions.double_1.pb b/pkg/compiler/test/testdata/snacks-core/025_core.control_flow.conditions.double_1.pb new file mode 100644 index 0000000000..40c95bc825 Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/025_core.control_flow.conditions.double_1.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/025_core.control_flow.conditions.double_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/025_core.control_flow.conditions.double_1_task.yaml new file mode 100755 index 0000000000..6fe4ef85ba --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/025_core.control_flow.conditions.double_1_task.yaml @@ -0,0 +1,52 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.control_flow.conditions + - task-name + - double + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.control_flow.conditions.double + resource_type: 1 + interface: + inputs: + variables: + "n": + description: |- + name of the parameter for the task is derived from the name of the input variable + and the type is mapped to ``Types.Integer`` + type: + Type: + Simple: 2 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 2 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/026_core.control_flow.conditions.multiplier_2.pb b/pkg/compiler/test/testdata/snacks-core/026_core.control_flow.conditions.multiplier_2.pb new file mode 100644 index 0000000000..e09def6a8f Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/026_core.control_flow.conditions.multiplier_2.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/026_core.control_flow.conditions.multiplier_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/026_core.control_flow.conditions.multiplier_2_wf.yaml new file mode 100755 index 0000000000..5f1aeaa2f7 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/026_core.control_flow.conditions.multiplier_2_wf.yaml @@ -0,0 +1,110 @@ +workflow: + id: + name: core.control_flow.conditions.multiplier + resource_type: 2 + interface: + inputs: + variables: + my_input: + description: my_input + type: + Type: + Simple: 2 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 2 + metadata: {} + metadata_defaults: {} + nodes: + - Target: + BranchNode: + if_else: + Default: + ElseNode: + Target: + TaskNode: + Reference: + ReferenceId: + name: core.control_flow.conditions.square + resource_type: 1 + overrides: {} + id: n1 + inputs: + - binding: + Value: + Promise: + var: my_input + var: "n" + metadata: + InterruptibleValue: null + name: square + retries: {} + case: + condition: + Expr: + Conjunction: + left_expression: + Expr: + Comparison: + left_value: + Val: + Var: .my_input + operator: 3 + right_value: + Val: + Primitive: + Value: + FloatValue: 0.1 + right_expression: + Expr: + Comparison: + left_value: + Val: + Var: .my_input + operator: 5 + right_value: + Val: + Primitive: + Value: + FloatValue: 1 + then_node: + Target: + TaskNode: + Reference: + ReferenceId: + name: core.control_flow.conditions.double + resource_type: 1 + overrides: {} + id: n0 + inputs: + - binding: + Value: + Promise: + var: my_input + var: "n" + metadata: + InterruptibleValue: null + name: double + retries: {} + id: n0 + inputs: + - binding: + Value: + Promise: + var: my_input + var: .my_input + metadata: + InterruptibleValue: null + name: fractions + retries: {} + outputs: + - binding: + Value: + Promise: + node_id: n0 + var: o0 + var: o0 diff --git a/pkg/compiler/test/testdata/snacks-core/027_core.control_flow.conditions.multiplier_3.pb b/pkg/compiler/test/testdata/snacks-core/027_core.control_flow.conditions.multiplier_3.pb new file mode 100644 index 0000000000..ff83e8b6b9 Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/027_core.control_flow.conditions.multiplier_3.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/028_core.control_flow.conditions.multiplier_2_2.pb b/pkg/compiler/test/testdata/snacks-core/028_core.control_flow.conditions.multiplier_2_2.pb new file mode 100644 index 0000000000..1b9d53d415 Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/028_core.control_flow.conditions.multiplier_2_2.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/028_core.control_flow.conditions.multiplier_2_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/028_core.control_flow.conditions.multiplier_2_2_wf.yaml new file mode 100755 index 0000000000..05af837afb --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/028_core.control_flow.conditions.multiplier_2_2_wf.yaml @@ -0,0 +1,141 @@ +workflow: + id: + name: core.control_flow.conditions.multiplier_2 + resource_type: 2 + interface: + inputs: + variables: + my_input: + description: my_input + type: + Type: + Simple: 2 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 2 + metadata: {} + metadata_defaults: {} + nodes: + - Target: + BranchNode: + if_else: + Default: + Error: + failed_node_id: fractions + message: The input must be between 0 and 10 + case: + condition: + Expr: + Conjunction: + left_expression: + Expr: + Comparison: + left_value: + Val: + Var: .my_input + operator: 2 + right_value: + Val: + Primitive: + Value: + FloatValue: 0.1 + right_expression: + Expr: + Comparison: + left_value: + Val: + Var: .my_input + operator: 4 + right_value: + Val: + Primitive: + Value: + FloatValue: 1 + then_node: + Target: + TaskNode: + Reference: + ReferenceId: + name: core.control_flow.conditions.double + resource_type: 1 + overrides: {} + id: n0 + inputs: + - binding: + Value: + Promise: + var: my_input + var: "n" + metadata: + InterruptibleValue: null + name: double + retries: {} + other: + - condition: + Expr: + Conjunction: + left_expression: + Expr: + Comparison: + left_value: + Val: + Var: .my_input + operator: 2 + right_value: + Val: + Primitive: + Value: + FloatValue: 1 + right_expression: + Expr: + Comparison: + left_value: + Val: + Var: .my_input + operator: 5 + right_value: + Val: + Primitive: + Value: + FloatValue: 10 + then_node: + Target: + TaskNode: + Reference: + ReferenceId: + name: core.control_flow.conditions.square + resource_type: 1 + overrides: {} + id: n1 + inputs: + - binding: + Value: + Promise: + var: my_input + var: "n" + metadata: + InterruptibleValue: null + name: square + retries: {} + id: n0 + inputs: + - binding: + Value: + Promise: + var: my_input + var: .my_input + metadata: + InterruptibleValue: null + name: fractions + retries: {} + outputs: + - binding: + Value: + Promise: + node_id: n0 + var: o0 + var: o0 diff --git a/pkg/compiler/test/testdata/snacks-core/029_core.control_flow.conditions.multiplier_2_3.pb b/pkg/compiler/test/testdata/snacks-core/029_core.control_flow.conditions.multiplier_2_3.pb new file mode 100644 index 0000000000..ee9bda460e Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/029_core.control_flow.conditions.multiplier_2_3.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/030_core.control_flow.conditions.multiplier_3_2.pb b/pkg/compiler/test/testdata/snacks-core/030_core.control_flow.conditions.multiplier_3_2.pb new file mode 100644 index 0000000000..d8ef87314b Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/030_core.control_flow.conditions.multiplier_3_2.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/030_core.control_flow.conditions.multiplier_3_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/030_core.control_flow.conditions.multiplier_3_2_wf.yaml new file mode 100755 index 0000000000..37e1e41eea --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/030_core.control_flow.conditions.multiplier_3_2_wf.yaml @@ -0,0 +1,162 @@ +workflow: + id: + name: core.control_flow.conditions.multiplier_3 + resource_type: 2 + interface: + inputs: + variables: + my_input: + description: my_input + type: + Type: + Simple: 2 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 2 + metadata: {} + metadata_defaults: {} + nodes: + - Target: + BranchNode: + if_else: + Default: + Error: + failed_node_id: fractions + message: The input must be between 0 and 10 + case: + condition: + Expr: + Conjunction: + left_expression: + Expr: + Comparison: + left_value: + Val: + Var: .my_input + operator: 2 + right_value: + Val: + Primitive: + Value: + FloatValue: 0.1 + right_expression: + Expr: + Comparison: + left_value: + Val: + Var: .my_input + operator: 4 + right_value: + Val: + Primitive: + Value: + FloatValue: 1 + then_node: + Target: + TaskNode: + Reference: + ReferenceId: + name: core.control_flow.conditions.double + resource_type: 1 + overrides: {} + id: n0 + inputs: + - binding: + Value: + Promise: + var: my_input + var: "n" + metadata: + InterruptibleValue: null + name: double + retries: {} + other: + - condition: + Expr: + Conjunction: + left_expression: + Expr: + Comparison: + left_value: + Val: + Var: .my_input + operator: 2 + right_value: + Val: + Primitive: + Value: + FloatValue: 1 + right_expression: + Expr: + Comparison: + left_value: + Val: + Var: .my_input + operator: 4 + right_value: + Val: + Primitive: + Value: + FloatValue: 10 + then_node: + Target: + TaskNode: + Reference: + ReferenceId: + name: core.control_flow.conditions.square + resource_type: 1 + overrides: {} + id: n1 + inputs: + - binding: + Value: + Promise: + var: my_input + var: "n" + metadata: + InterruptibleValue: null + name: square + retries: {} + id: n0 + inputs: + - binding: + Value: + Promise: + var: my_input + var: .my_input + metadata: + InterruptibleValue: null + name: fractions + retries: {} + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.control_flow.conditions.double + resource_type: 1 + overrides: {} + id: n1 + inputs: + - binding: + Value: + Promise: + node_id: n0 + var: o0 + var: "n" + metadata: + InterruptibleValue: null + name: double + retries: {} + upstream_node_ids: + - n0 + outputs: + - binding: + Value: + Promise: + node_id: n1 + var: o0 + var: o0 diff --git a/pkg/compiler/test/testdata/snacks-core/031_core.control_flow.conditions.multiplier_3_3.pb b/pkg/compiler/test/testdata/snacks-core/031_core.control_flow.conditions.multiplier_3_3.pb new file mode 100644 index 0000000000..bbf725801b Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/031_core.control_flow.conditions.multiplier_3_3.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/032_core.control_flow.conditions.coin_toss_1.pb b/pkg/compiler/test/testdata/snacks-core/032_core.control_flow.conditions.coin_toss_1.pb new file mode 100644 index 0000000000..2e811f341b Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/032_core.control_flow.conditions.coin_toss_1.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/032_core.control_flow.conditions.coin_toss_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/032_core.control_flow.conditions.coin_toss_1_task.yaml new file mode 100755 index 0000000000..c3702e11ee --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/032_core.control_flow.conditions.coin_toss_1_task.yaml @@ -0,0 +1,50 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.control_flow.conditions + - task-name + - coin_toss + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.control_flow.conditions.coin_toss + resource_type: 1 + interface: + inputs: + variables: + seed: + description: seed + type: + Type: + Simple: 1 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 4 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/033_core.control_flow.conditions.failed_1.pb b/pkg/compiler/test/testdata/snacks-core/033_core.control_flow.conditions.failed_1.pb new file mode 100644 index 0000000000..043bf2269b Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/033_core.control_flow.conditions.failed_1.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/033_core.control_flow.conditions.failed_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/033_core.control_flow.conditions.failed_1_task.yaml new file mode 100755 index 0000000000..d0d38e1a5b --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/033_core.control_flow.conditions.failed_1_task.yaml @@ -0,0 +1,44 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.control_flow.conditions + - task-name + - failed + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.control_flow.conditions.failed + resource_type: 1 + interface: + inputs: {} + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 1 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/034_core.control_flow.conditions.success_1.pb b/pkg/compiler/test/testdata/snacks-core/034_core.control_flow.conditions.success_1.pb new file mode 100644 index 0000000000..70abfdfa48 Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/034_core.control_flow.conditions.success_1.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/034_core.control_flow.conditions.success_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/034_core.control_flow.conditions.success_1_task.yaml new file mode 100755 index 0000000000..6ca896e2dc --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/034_core.control_flow.conditions.success_1_task.yaml @@ -0,0 +1,44 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.control_flow.conditions + - task-name + - success + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.control_flow.conditions.success + resource_type: 1 + interface: + inputs: {} + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 1 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/035_core.control_flow.conditions.basic_boolean_wf_2.pb b/pkg/compiler/test/testdata/snacks-core/035_core.control_flow.conditions.basic_boolean_wf_2.pb new file mode 100644 index 0000000000..67ff304f75 Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/035_core.control_flow.conditions.basic_boolean_wf_2.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/035_core.control_flow.conditions.basic_boolean_wf_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/035_core.control_flow.conditions.basic_boolean_wf_2_wf.yaml new file mode 100755 index 0000000000..e9274df154 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/035_core.control_flow.conditions.basic_boolean_wf_2_wf.yaml @@ -0,0 +1,103 @@ +workflow: + id: + name: core.control_flow.conditions.basic_boolean_wf + resource_type: 2 + interface: + inputs: + variables: + seed: + description: seed + type: + Type: + Simple: 1 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 1 + metadata: {} + metadata_defaults: {} + nodes: + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.control_flow.conditions.coin_toss + resource_type: 1 + overrides: {} + id: n0 + inputs: + - binding: + Value: + Promise: + var: seed + var: seed + metadata: + InterruptibleValue: null + name: coin_toss + retries: {} + - Target: + BranchNode: + if_else: + Default: + ElseNode: + Target: + TaskNode: + Reference: + ReferenceId: + name: core.control_flow.conditions.failed + resource_type: 1 + overrides: {} + id: n1 + metadata: + InterruptibleValue: null + name: failed + retries: {} + case: + condition: + Expr: + Comparison: + left_value: + Val: + Var: n0.o0 + right_value: + Val: + Primitive: + Value: + Boolean: true + then_node: + Target: + TaskNode: + Reference: + ReferenceId: + name: core.control_flow.conditions.success + resource_type: 1 + overrides: {} + id: n0 + metadata: + InterruptibleValue: null + name: success + retries: {} + id: n1 + inputs: + - binding: + Value: + Promise: + node_id: n0 + var: o0 + var: n0.o0 + metadata: + InterruptibleValue: null + name: test + retries: {} + upstream_node_ids: + - n0 + outputs: + - binding: + Value: + Promise: + node_id: n1 + var: o0 + var: o0 diff --git a/pkg/compiler/test/testdata/snacks-core/036_core.control_flow.conditions.basic_boolean_wf_3.pb b/pkg/compiler/test/testdata/snacks-core/036_core.control_flow.conditions.basic_boolean_wf_3.pb new file mode 100644 index 0000000000..b1719da430 Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/036_core.control_flow.conditions.basic_boolean_wf_3.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/037_core.control_flow.conditions.bool_input_wf_2.pb b/pkg/compiler/test/testdata/snacks-core/037_core.control_flow.conditions.bool_input_wf_2.pb new file mode 100644 index 0000000000..464761476a Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/037_core.control_flow.conditions.bool_input_wf_2.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/037_core.control_flow.conditions.bool_input_wf_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/037_core.control_flow.conditions.bool_input_wf_2_wf.yaml new file mode 100755 index 0000000000..9afb0ad756 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/037_core.control_flow.conditions.bool_input_wf_2_wf.yaml @@ -0,0 +1,82 @@ +workflow: + id: + name: core.control_flow.conditions.bool_input_wf + resource_type: 2 + interface: + inputs: + variables: + b: + description: b + type: + Type: + Simple: 4 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 1 + metadata: {} + metadata_defaults: {} + nodes: + - Target: + BranchNode: + if_else: + Default: + ElseNode: + Target: + TaskNode: + Reference: + ReferenceId: + name: core.control_flow.conditions.failed + resource_type: 1 + overrides: {} + id: n1 + metadata: + InterruptibleValue: null + name: failed + retries: {} + case: + condition: + Expr: + Comparison: + left_value: + Val: + Var: .b + right_value: + Val: + Primitive: + Value: + Boolean: true + then_node: + Target: + TaskNode: + Reference: + ReferenceId: + name: core.control_flow.conditions.success + resource_type: 1 + overrides: {} + id: n0 + metadata: + InterruptibleValue: null + name: success + retries: {} + id: n0 + inputs: + - binding: + Value: + Promise: + var: b + var: .b + metadata: + InterruptibleValue: null + name: test + retries: {} + outputs: + - binding: + Value: + Promise: + node_id: n0 + var: o0 + var: o0 diff --git a/pkg/compiler/test/testdata/snacks-core/038_core.control_flow.conditions.bool_input_wf_3.pb b/pkg/compiler/test/testdata/snacks-core/038_core.control_flow.conditions.bool_input_wf_3.pb new file mode 100644 index 0000000000..b2e66930bd Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/038_core.control_flow.conditions.bool_input_wf_3.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/039_core.control_flow.conditions.nested_conditions_2.pb b/pkg/compiler/test/testdata/snacks-core/039_core.control_flow.conditions.nested_conditions_2.pb new file mode 100644 index 0000000000..45a5e41672 Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/039_core.control_flow.conditions.nested_conditions_2.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/039_core.control_flow.conditions.nested_conditions_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/039_core.control_flow.conditions.nested_conditions_2_wf.yaml new file mode 100755 index 0000000000..5d7d6761b1 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/039_core.control_flow.conditions.nested_conditions_2_wf.yaml @@ -0,0 +1,236 @@ +workflow: + id: + name: core.control_flow.conditions.nested_conditions + resource_type: 2 + interface: + inputs: + variables: + my_input: + description: my_input + type: + Type: + Simple: 2 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 2 + metadata: {} + metadata_defaults: {} + nodes: + - Target: + BranchNode: + if_else: + Default: + ElseNode: + Target: + TaskNode: + Reference: + ReferenceId: + name: core.control_flow.conditions.double + resource_type: 1 + overrides: {} + id: n2 + inputs: + - binding: + Value: + Promise: + var: my_input + var: "n" + metadata: + InterruptibleValue: null + name: double + retries: {} + case: + condition: + Expr: + Conjunction: + left_expression: + Expr: + Comparison: + left_value: + Val: + Var: .my_input + operator: 2 + right_value: + Val: + Primitive: + Value: + FloatValue: 0.1 + right_expression: + Expr: + Comparison: + left_value: + Val: + Var: .my_input + operator: 4 + right_value: + Val: + Primitive: + Value: + FloatValue: 1 + then_node: + Target: + BranchNode: + if_else: + Default: + Error: + failed_node_id: inner_fractions + message: Only <0.7 allowed + case: + condition: + Expr: + Comparison: + left_value: + Val: + Var: .my_input + operator: 4 + right_value: + Val: + Primitive: + Value: + FloatValue: 0.5 + then_node: + Target: + TaskNode: + Reference: + ReferenceId: + name: core.control_flow.conditions.double + resource_type: 1 + overrides: {} + id: n0 + inputs: + - binding: + Value: + Promise: + var: my_input + var: "n" + metadata: + InterruptibleValue: null + name: double + retries: {} + other: + - condition: + Expr: + Conjunction: + left_expression: + Expr: + Comparison: + left_value: + Val: + Var: .my_input + operator: 2 + right_value: + Val: + Primitive: + Value: + FloatValue: 0.5 + right_expression: + Expr: + Comparison: + left_value: + Val: + Var: .my_input + operator: 4 + right_value: + Val: + Primitive: + Value: + FloatValue: 0.7 + then_node: + Target: + TaskNode: + Reference: + ReferenceId: + name: core.control_flow.conditions.square + resource_type: 1 + overrides: {} + id: n1 + inputs: + - binding: + Value: + Promise: + var: my_input + var: "n" + metadata: + InterruptibleValue: null + name: square + retries: {} + id: n0 + inputs: + - binding: + Value: + Promise: + var: my_input + var: .my_input + metadata: + InterruptibleValue: null + name: inner_fractions + retries: {} + other: + - condition: + Expr: + Conjunction: + left_expression: + Expr: + Comparison: + left_value: + Val: + Var: .my_input + operator: 2 + right_value: + Val: + Primitive: + Value: + FloatValue: 1 + right_expression: + Expr: + Comparison: + left_value: + Val: + Var: .my_input + operator: 4 + right_value: + Val: + Primitive: + Value: + FloatValue: 10 + then_node: + Target: + TaskNode: + Reference: + ReferenceId: + name: core.control_flow.conditions.square + resource_type: 1 + overrides: {} + id: n1 + inputs: + - binding: + Value: + Promise: + var: my_input + var: "n" + metadata: + InterruptibleValue: null + name: square + retries: {} + id: n0 + inputs: + - binding: + Value: + Promise: + var: my_input + var: .my_input + metadata: + InterruptibleValue: null + name: fractions + retries: {} + outputs: + - binding: + Value: + Promise: + node_id: n0 + var: o0 + var: o0 diff --git a/pkg/compiler/test/testdata/snacks-core/040_core.control_flow.conditions.nested_conditions_3.pb b/pkg/compiler/test/testdata/snacks-core/040_core.control_flow.conditions.nested_conditions_3.pb new file mode 100644 index 0000000000..652dfa2f93 Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/040_core.control_flow.conditions.nested_conditions_3.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/041_core.control_flow.conditions.calc_sum_1.pb b/pkg/compiler/test/testdata/snacks-core/041_core.control_flow.conditions.calc_sum_1.pb new file mode 100644 index 0000000000..0f491e4658 Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/041_core.control_flow.conditions.calc_sum_1.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/041_core.control_flow.conditions.calc_sum_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/041_core.control_flow.conditions.calc_sum_1_task.yaml new file mode 100755 index 0000000000..222033d87f --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/041_core.control_flow.conditions.calc_sum_1_task.yaml @@ -0,0 +1,55 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.control_flow.conditions + - task-name + - calc_sum + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.control_flow.conditions.calc_sum + resource_type: 1 + interface: + inputs: + variables: + a: + description: a + type: + Type: + Simple: 2 + b: + description: b + type: + Type: + Simple: 2 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 2 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/042_core.control_flow.conditions.consume_outputs_2.pb b/pkg/compiler/test/testdata/snacks-core/042_core.control_flow.conditions.consume_outputs_2.pb new file mode 100644 index 0000000000..625b327cd4 Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/042_core.control_flow.conditions.consume_outputs_2.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/042_core.control_flow.conditions.consume_outputs_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/042_core.control_flow.conditions.consume_outputs_2_wf.yaml new file mode 100755 index 0000000000..ae0b320bf9 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/042_core.control_flow.conditions.consume_outputs_2_wf.yaml @@ -0,0 +1,146 @@ +workflow: + id: + name: core.control_flow.conditions.consume_outputs + resource_type: 2 + interface: + inputs: + variables: + my_input: + description: my_input + type: + Type: + Simple: 2 + seed: + description: seed + type: + Type: + Simple: 1 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 2 + metadata: {} + metadata_defaults: {} + nodes: + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.control_flow.conditions.coin_toss + resource_type: 1 + overrides: {} + id: n0 + inputs: + - binding: + Value: + Promise: + var: seed + var: seed + metadata: + InterruptibleValue: null + name: coin_toss + retries: {} + - Target: + BranchNode: + if_else: + Default: + ElseNode: + Target: + TaskNode: + Reference: + ReferenceId: + name: core.control_flow.conditions.calc_sum + resource_type: 1 + overrides: {} + id: n1 + inputs: + - binding: + Value: + Promise: + var: my_input + var: a + - binding: + Value: + Promise: + var: my_input + var: b + metadata: + InterruptibleValue: null + name: calc_sum + retries: {} + case: + condition: + Expr: + Comparison: + left_value: + Val: + Var: n0.o0 + right_value: + Val: + Primitive: + Value: + Boolean: true + then_node: + Target: + TaskNode: + Reference: + ReferenceId: + name: core.control_flow.conditions.square + resource_type: 1 + overrides: {} + id: n0 + inputs: + - binding: + Value: + Promise: + var: my_input + var: "n" + metadata: + InterruptibleValue: null + name: square + retries: {} + id: n1 + inputs: + - binding: + Value: + Promise: + node_id: n0 + var: o0 + var: n0.o0 + metadata: + InterruptibleValue: null + name: double_or_square + retries: {} + upstream_node_ids: + - n0 + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.control_flow.conditions.double + resource_type: 1 + overrides: {} + id: n2 + inputs: + - binding: + Value: + Promise: + node_id: n1 + var: o0 + var: "n" + metadata: + InterruptibleValue: null + name: double + retries: {} + upstream_node_ids: + - n1 + outputs: + - binding: + Value: + Promise: + node_id: n2 + var: o0 + var: o0 diff --git a/pkg/compiler/test/testdata/snacks-core/043_core.control_flow.conditions.consume_outputs_3.pb b/pkg/compiler/test/testdata/snacks-core/043_core.control_flow.conditions.consume_outputs_3.pb new file mode 100644 index 0000000000..3e045ceb40 Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/043_core.control_flow.conditions.consume_outputs_3.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/044_core.control_flow.dynamics.return_index_1.pb b/pkg/compiler/test/testdata/snacks-core/044_core.control_flow.dynamics.return_index_1.pb new file mode 100644 index 0000000000..34816bca2d Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/044_core.control_flow.dynamics.return_index_1.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/044_core.control_flow.dynamics.return_index_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/044_core.control_flow.dynamics.return_index_1_task.yaml new file mode 100755 index 0000000000..5d777782fa --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/044_core.control_flow.dynamics.return_index_1_task.yaml @@ -0,0 +1,50 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.control_flow.dynamics + - task-name + - return_index + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.control_flow.dynamics.return_index + resource_type: 1 + interface: + inputs: + variables: + character: + description: character + type: + Type: + Simple: 3 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 1 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/045_core.control_flow.dynamics.update_list_1.pb b/pkg/compiler/test/testdata/snacks-core/045_core.control_flow.dynamics.update_list_1.pb new file mode 100644 index 0000000000..ec27f0f97b Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/045_core.control_flow.dynamics.update_list_1.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/045_core.control_flow.dynamics.update_list_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/045_core.control_flow.dynamics.update_list_1_task.yaml new file mode 100755 index 0000000000..b9e7fb77b4 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/045_core.control_flow.dynamics.update_list_1_task.yaml @@ -0,0 +1,59 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.control_flow.dynamics + - task-name + - update_list + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.control_flow.dynamics.update_list + resource_type: 1 + interface: + inputs: + variables: + freq_list: + description: freq_list + type: + Type: + CollectionType: + Type: + Simple: 1 + list_index: + description: list_index + type: + Type: + Simple: 1 + outputs: + variables: + o0: + description: o0 + type: + Type: + CollectionType: + Type: + Simple: 1 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/046_core.control_flow.dynamics.derive_count_1.pb b/pkg/compiler/test/testdata/snacks-core/046_core.control_flow.dynamics.derive_count_1.pb new file mode 100644 index 0000000000..753f96b79c Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/046_core.control_flow.dynamics.derive_count_1.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/046_core.control_flow.dynamics.derive_count_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/046_core.control_flow.dynamics.derive_count_1_task.yaml new file mode 100755 index 0000000000..4e62f3ed8a --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/046_core.control_flow.dynamics.derive_count_1_task.yaml @@ -0,0 +1,59 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.control_flow.dynamics + - task-name + - derive_count + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.control_flow.dynamics.derive_count + resource_type: 1 + interface: + inputs: + variables: + freq1: + description: freq1 + type: + Type: + CollectionType: + Type: + Simple: 1 + freq2: + description: freq2 + type: + Type: + CollectionType: + Type: + Simple: 1 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 1 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/047_core.control_flow.dynamics.count_characters_1.pb b/pkg/compiler/test/testdata/snacks-core/047_core.control_flow.dynamics.count_characters_1.pb new file mode 100644 index 0000000000..b5152116d6 Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/047_core.control_flow.dynamics.count_characters_1.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/047_core.control_flow.dynamics.count_characters_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/047_core.control_flow.dynamics.count_characters_1_task.yaml new file mode 100755 index 0000000000..1d7655190a --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/047_core.control_flow.dynamics.count_characters_1_task.yaml @@ -0,0 +1,58 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.control_flow.dynamics + - task-name + - count_characters + env: + - key: _F_SS_C + value: H4sIAAAAAAAC/8VRXW7DIAy+ysTzWtKtUuiuMk2RAZN6pZCBE6mrevcBXdXuBHsIsr8f85mcBR1hxMHE4GgUb09nYdHB7HloREMCHGtxY8Tzk3BfoSLj3qQ1Ren8iTGm8VqYGA+6fFXIUKcKExOulFa4AbXdmR1YZ0CpbruDrt/qTWfBvvTK9V3X9+JSjO36XLzv/xHgo0aYUvxEw8UeZu8LYOMRKNz7BVOm+ABgWO7NdOJ9DAMFxjQlLGcNIuPEcilCqSnIq+a1bVSTH4iHhRLP4ItkSDHyX5MnfTOtlczEuJrAHOpbyduENg0yDxkTgadv4JKydMwUxtz+KQbQHm2pHfiMdTnMhb5KLaWHrSlzIj03xkfTJL/05fIDQ+X0kUICAAA= + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.control_flow.dynamics.count_characters + resource_type: 1 + interface: + inputs: + variables: + s1: + description: s1 + type: + Type: + Simple: 3 + s2: + description: s2 + type: + Type: + Simple: 3 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 1 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/048_core.control_flow.dynamics.wf_2.pb b/pkg/compiler/test/testdata/snacks-core/048_core.control_flow.dynamics.wf_2.pb new file mode 100644 index 0000000000..1ffde2f876 Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/048_core.control_flow.dynamics.wf_2.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/048_core.control_flow.dynamics.wf_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/048_core.control_flow.dynamics.wf_2_wf.yaml new file mode 100755 index 0000000000..f551fd150d --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/048_core.control_flow.dynamics.wf_2_wf.yaml @@ -0,0 +1,57 @@ +workflow: + id: + name: core.control_flow.dynamics.wf + resource_type: 2 + interface: + inputs: + variables: + s1: + description: s1 + type: + Type: + Simple: 3 + s2: + description: s2 + type: + Type: + Simple: 3 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 1 + metadata: {} + metadata_defaults: {} + nodes: + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.control_flow.dynamics.count_characters + resource_type: 1 + overrides: {} + id: n0 + inputs: + - binding: + Value: + Promise: + var: s1 + var: s1 + - binding: + Value: + Promise: + var: s2 + var: s2 + metadata: + InterruptibleValue: null + name: count_characters + retries: {} + outputs: + - binding: + Value: + Promise: + node_id: n0 + var: o0 + var: o0 diff --git a/pkg/compiler/test/testdata/snacks-core/049_core.control_flow.dynamics.wf_3.pb b/pkg/compiler/test/testdata/snacks-core/049_core.control_flow.dynamics.wf_3.pb new file mode 100644 index 0000000000..4304a3ad80 Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/049_core.control_flow.dynamics.wf_3.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/050_core.control_flow.map_task.a_mappable_task_1.pb b/pkg/compiler/test/testdata/snacks-core/050_core.control_flow.map_task.a_mappable_task_1.pb new file mode 100644 index 0000000000..391321b2fb Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/050_core.control_flow.map_task.a_mappable_task_1.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/050_core.control_flow.map_task.a_mappable_task_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/050_core.control_flow.map_task.a_mappable_task_1_task.yaml new file mode 100755 index 0000000000..a56819bc5f --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/050_core.control_flow.map_task.a_mappable_task_1_task.yaml @@ -0,0 +1,50 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.control_flow.map_task + - task-name + - a_mappable_task + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.control_flow.map_task.a_mappable_task + resource_type: 1 + interface: + inputs: + variables: + a: + description: a + type: + Type: + Simple: 1 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 3 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/051_core.control_flow.map_task.coalesce_1.pb b/pkg/compiler/test/testdata/snacks-core/051_core.control_flow.map_task.coalesce_1.pb new file mode 100644 index 0000000000..2edc486787 Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/051_core.control_flow.map_task.coalesce_1.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/051_core.control_flow.map_task.coalesce_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/051_core.control_flow.map_task.coalesce_1_task.yaml new file mode 100755 index 0000000000..eceff4ce1a --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/051_core.control_flow.map_task.coalesce_1_task.yaml @@ -0,0 +1,52 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.control_flow.map_task + - task-name + - coalesce + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.control_flow.map_task.coalesce + resource_type: 1 + interface: + inputs: + variables: + b: + description: b + type: + Type: + CollectionType: + Type: + Simple: 3 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 3 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/052_core.control_flow.map_task.mapper_a_mappable_task_0_1.pb b/pkg/compiler/test/testdata/snacks-core/052_core.control_flow.map_task.mapper_a_mappable_task_0_1.pb new file mode 100644 index 0000000000..b99c1fba2d Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/052_core.control_flow.map_task.mapper_a_mappable_task_0_1.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/052_core.control_flow.map_task.mapper_a_mappable_task_0_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/052_core.control_flow.map_task.mapper_a_mappable_task_0_1_task.yaml new file mode 100755 index 0000000000..559f46b5c5 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/052_core.control_flow.map_task.mapper_a_mappable_task_0_1_task.yaml @@ -0,0 +1,57 @@ +template: + Target: + Container: + args: + - pyflyte-map-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.control_flow.map_task + - task-name + - a_mappable_task + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + custom: + minSuccessRatio: 1 + id: + name: core.control_flow.map_task.mapper_a_mappable_task_0 + resource_type: 1 + interface: + inputs: + variables: + a: + description: a + type: + Type: + CollectionType: + Type: + Simple: 1 + outputs: + variables: + o0: + description: o0 + type: + Type: + CollectionType: + Type: + Simple: 3 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + task_type_version: 1 + type: container_array diff --git a/pkg/compiler/test/testdata/snacks-core/053_core.control_flow.map_task.my_map_workflow_2.pb b/pkg/compiler/test/testdata/snacks-core/053_core.control_flow.map_task.my_map_workflow_2.pb new file mode 100644 index 0000000000..b815ad0c77 Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/053_core.control_flow.map_task.my_map_workflow_2.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/053_core.control_flow.map_task.my_map_workflow_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/053_core.control_flow.map_task.my_map_workflow_2_wf.yaml new file mode 100755 index 0000000000..91a7a3dec2 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/053_core.control_flow.map_task.my_map_workflow_2_wf.yaml @@ -0,0 +1,78 @@ +workflow: + id: + name: core.control_flow.map_task.my_map_workflow + resource_type: 2 + interface: + inputs: + variables: + a: + description: a + type: + Type: + CollectionType: + Type: + Simple: 1 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 3 + metadata: {} + metadata_defaults: {} + nodes: + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.control_flow.map_task.mapper_a_mappable_task_0 + resource_type: 1 + overrides: + resources: + limits: + - name: 3 + value: 500Mi + requests: + - name: 3 + value: 300Mi + id: n0 + inputs: + - binding: + Value: + Promise: + var: a + var: a + metadata: + InterruptibleValue: null + name: mapper_a_mappable_task_0 + retries: + retries: 1 + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.control_flow.map_task.coalesce + resource_type: 1 + overrides: {} + id: n1 + inputs: + - binding: + Value: + Promise: + node_id: n0 + var: o0 + var: b + metadata: + InterruptibleValue: null + name: coalesce + retries: {} + upstream_node_ids: + - n0 + outputs: + - binding: + Value: + Promise: + node_id: n1 + var: o0 + var: o0 diff --git a/pkg/compiler/test/testdata/snacks-core/054_core.control_flow.map_task.my_map_workflow_3.pb b/pkg/compiler/test/testdata/snacks-core/054_core.control_flow.map_task.my_map_workflow_3.pb new file mode 100644 index 0000000000..59b05ceee6 Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/054_core.control_flow.map_task.my_map_workflow_3.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/055_core.control_flow.merge_sort.split_1.pb b/pkg/compiler/test/testdata/snacks-core/055_core.control_flow.merge_sort.split_1.pb new file mode 100644 index 0000000000..d9628e6dfe Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/055_core.control_flow.merge_sort.split_1.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/055_core.control_flow.merge_sort.split_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/055_core.control_flow.merge_sort.split_1_task.yaml new file mode 100755 index 0000000000..a2297e3685 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/055_core.control_flow.merge_sort.split_1_task.yaml @@ -0,0 +1,66 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.control_flow.merge_sort + - task-name + - split + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.control_flow.merge_sort.split + resource_type: 1 + interface: + inputs: + variables: + numbers: + description: numbers + type: + Type: + CollectionType: + Type: + Simple: 1 + outputs: + variables: + o0: + description: o0 + type: + Type: + CollectionType: + Type: + Simple: 1 + o1: + description: o1 + type: + Type: + CollectionType: + Type: + Simple: 1 + o2: + description: o2 + type: + Type: + Simple: 1 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/056_core.control_flow.merge_sort.merge_1.pb b/pkg/compiler/test/testdata/snacks-core/056_core.control_flow.merge_sort.merge_1.pb new file mode 100644 index 0000000000..096d3a0dad Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/056_core.control_flow.merge_sort.merge_1.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/056_core.control_flow.merge_sort.merge_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/056_core.control_flow.merge_sort.merge_1_task.yaml new file mode 100755 index 0000000000..194f004bd4 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/056_core.control_flow.merge_sort.merge_1_task.yaml @@ -0,0 +1,61 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.control_flow.merge_sort + - task-name + - merge + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.control_flow.merge_sort.merge + resource_type: 1 + interface: + inputs: + variables: + sorted_list1: + description: sorted_list1 + type: + Type: + CollectionType: + Type: + Simple: 1 + sorted_list2: + description: sorted_list2 + type: + Type: + CollectionType: + Type: + Simple: 1 + outputs: + variables: + o0: + description: o0 + type: + Type: + CollectionType: + Type: + Simple: 1 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/057_core.control_flow.merge_sort.sort_locally_1.pb b/pkg/compiler/test/testdata/snacks-core/057_core.control_flow.merge_sort.sort_locally_1.pb new file mode 100644 index 0000000000..93ca39cace Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/057_core.control_flow.merge_sort.sort_locally_1.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/057_core.control_flow.merge_sort.sort_locally_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/057_core.control_flow.merge_sort.sort_locally_1_task.yaml new file mode 100755 index 0000000000..d93f896f95 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/057_core.control_flow.merge_sort.sort_locally_1_task.yaml @@ -0,0 +1,54 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.control_flow.merge_sort + - task-name + - sort_locally + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.control_flow.merge_sort.sort_locally + resource_type: 1 + interface: + inputs: + variables: + numbers: + description: numbers + type: + Type: + CollectionType: + Type: + Simple: 1 + outputs: + variables: + o0: + description: o0 + type: + Type: + CollectionType: + Type: + Simple: 1 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/058_core.control_flow.merge_sort.merge_sort_remotely_1.pb b/pkg/compiler/test/testdata/snacks-core/058_core.control_flow.merge_sort.merge_sort_remotely_1.pb new file mode 100644 index 0000000000..ff71e550b9 Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/058_core.control_flow.merge_sort.merge_sort_remotely_1.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/058_core.control_flow.merge_sort.merge_sort_remotely_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/058_core.control_flow.merge_sort.merge_sort_remotely_1_task.yaml new file mode 100755 index 0000000000..60acfeb941 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/058_core.control_flow.merge_sort.merge_sort_remotely_1_task.yaml @@ -0,0 +1,62 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.control_flow.merge_sort + - task-name + - merge_sort_remotely + env: + - key: _F_SS_C + value: H4sIAAAAAAAC/8VRXW7DIAy+ysTzWtKtUuiuMk2RAZN6pZCBE6mrevcBXdXuBHsIsr8f85mcBR1hxMHE4GgUb09nYdHB7HloREMCHGtxY8Tzk3BfoSLj3qQ1Ren8iTGm8VqYGA+6fFXIUKcKExOulFa4AbXdmR1YZ0CpbruDrt/qTWfBvvTK9V3X9+JSjO36XLzv/xHgo0aYUvxEw8UeZu8LYOMRKNz7BVOm+ABgWO7NdOJ9DAMFxjQlLGcNIuPEcilCqSnIq+a1bVSTH4iHhRLP4ItkSDHyX5MnfTOtlczEuJrAHOpbyduENg0yDxkTgadv4JKydMwUxtz+KQbQHm2pHfiMdTnMhb5KLaWHrSlzIj03xkfTJL/05fIDQ+X0kUICAAA= + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.control_flow.merge_sort.merge_sort_remotely + resource_type: 1 + interface: + inputs: + variables: + numbers: + description: numbers + type: + Type: + CollectionType: + Type: + Simple: 1 + run_local_at_count: + description: run_local_at_count + type: + Type: + Simple: 1 + outputs: + variables: + o0: + description: o0 + type: + Type: + CollectionType: + Type: + Simple: 1 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/059_core.control_flow.merge_sort.merge_sort_2.pb b/pkg/compiler/test/testdata/snacks-core/059_core.control_flow.merge_sort.merge_sort_2.pb new file mode 100644 index 0000000000..da9702df2a Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/059_core.control_flow.merge_sort.merge_sort_2.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/059_core.control_flow.merge_sort.merge_sort_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/059_core.control_flow.merge_sort.merge_sort_2_wf.yaml new file mode 100755 index 0000000000..f97e6aae47 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/059_core.control_flow.merge_sort.merge_sort_2_wf.yaml @@ -0,0 +1,117 @@ +workflow: + id: + name: core.control_flow.merge_sort.merge_sort + resource_type: 2 + interface: + inputs: + variables: + numbers: + description: numbers + type: + Type: + CollectionType: + Type: + Simple: 1 + numbers_count: + description: numbers_count + type: + Type: + Simple: 1 + run_local_at_count: + description: run_local_at_count + type: + Type: + Simple: 1 + outputs: + variables: + o0: + description: o0 + type: + Type: + CollectionType: + Type: + Simple: 1 + metadata: {} + metadata_defaults: {} + nodes: + - Target: + BranchNode: + if_else: + Default: + ElseNode: + Target: + TaskNode: + Reference: + ReferenceId: + name: core.control_flow.merge_sort.merge_sort_remotely + resource_type: 1 + overrides: {} + id: n1 + inputs: + - binding: + Value: + Promise: + var: numbers + var: numbers + - binding: + Value: + Promise: + var: run_local_at_count + var: run_local_at_count + metadata: + InterruptibleValue: null + name: merge_sort_remotely + retries: {} + case: + condition: + Expr: + Comparison: + left_value: + Val: + Var: .numbers_count + operator: 5 + right_value: + Val: + Var: .run_local_at_count + then_node: + Target: + TaskNode: + Reference: + ReferenceId: + name: core.control_flow.merge_sort.sort_locally + resource_type: 1 + overrides: {} + id: n0 + inputs: + - binding: + Value: + Promise: + var: numbers + var: numbers + metadata: + InterruptibleValue: null + name: sort_locally + retries: {} + id: n0 + inputs: + - binding: + Value: + Promise: + var: numbers_count + var: .numbers_count + - binding: + Value: + Promise: + var: run_local_at_count + var: .run_local_at_count + metadata: + InterruptibleValue: null + name: terminal_case + retries: {} + outputs: + - binding: + Value: + Promise: + node_id: n0 + var: o0 + var: o0 diff --git a/pkg/compiler/test/testdata/snacks-core/060_core.control_flow.merge_sort.merge_sort_3.pb b/pkg/compiler/test/testdata/snacks-core/060_core.control_flow.merge_sort.merge_sort_3.pb new file mode 100644 index 0000000000..dcb4bcbc00 Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/060_core.control_flow.merge_sort.merge_sort_3.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/061_core.control_flow.subworkflows.t1_1.pb b/pkg/compiler/test/testdata/snacks-core/061_core.control_flow.subworkflows.t1_1.pb new file mode 100644 index 0000000000..f558142858 Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/061_core.control_flow.subworkflows.t1_1.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/061_core.control_flow.subworkflows.t1_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/061_core.control_flow.subworkflows.t1_1_task.yaml new file mode 100755 index 0000000000..668bd3b506 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/061_core.control_flow.subworkflows.t1_1_task.yaml @@ -0,0 +1,55 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.control_flow.subworkflows + - task-name + - t1 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.control_flow.subworkflows.t1 + resource_type: 1 + interface: + inputs: + variables: + a: + description: a + type: + Type: + Simple: 1 + outputs: + variables: + c: + description: c + type: + Type: + Simple: 3 + t1_int_output: + description: t1_int_output + type: + Type: + Simple: 1 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/062_core.control_flow.subworkflows.my_subwf_2.pb b/pkg/compiler/test/testdata/snacks-core/062_core.control_flow.subworkflows.my_subwf_2.pb new file mode 100644 index 0000000000..f3396c064f Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/062_core.control_flow.subworkflows.my_subwf_2.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/062_core.control_flow.subworkflows.my_subwf_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/062_core.control_flow.subworkflows.my_subwf_2_wf.yaml new file mode 100755 index 0000000000..3d30e1ed40 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/062_core.control_flow.subworkflows.my_subwf_2_wf.yaml @@ -0,0 +1,79 @@ +workflow: + id: + name: core.control_flow.subworkflows.my_subwf + resource_type: 2 + interface: + inputs: + variables: + a: + description: a + type: + Type: + Simple: 1 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 3 + o1: + description: o1 + type: + Type: + Simple: 3 + metadata: {} + metadata_defaults: {} + nodes: + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.control_flow.subworkflows.t1 + resource_type: 1 + overrides: {} + id: n0 + inputs: + - binding: + Value: + Promise: + var: a + var: a + metadata: + InterruptibleValue: null + name: t1 + retries: {} + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.control_flow.subworkflows.t1 + resource_type: 1 + overrides: {} + id: n1 + inputs: + - binding: + Value: + Promise: + node_id: n0 + var: t1_int_output + var: a + metadata: + InterruptibleValue: null + name: t1 + retries: {} + upstream_node_ids: + - n0 + outputs: + - binding: + Value: + Promise: + node_id: n0 + var: c + var: o0 + - binding: + Value: + Promise: + node_id: n1 + var: c + var: o1 diff --git a/pkg/compiler/test/testdata/snacks-core/063_core.control_flow.subworkflows.my_subwf_3.pb b/pkg/compiler/test/testdata/snacks-core/063_core.control_flow.subworkflows.my_subwf_3.pb new file mode 100644 index 0000000000..fdec6ecffc Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/063_core.control_flow.subworkflows.my_subwf_3.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/064_core.control_flow.subworkflows.parent_wf_2.pb b/pkg/compiler/test/testdata/snacks-core/064_core.control_flow.subworkflows.parent_wf_2.pb new file mode 100644 index 0000000000..f78d74c653 Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/064_core.control_flow.subworkflows.parent_wf_2.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/065_core.control_flow.subworkflows.parent_wf_3.pb b/pkg/compiler/test/testdata/snacks-core/065_core.control_flow.subworkflows.parent_wf_3.pb new file mode 100644 index 0000000000..1eb259995b Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/065_core.control_flow.subworkflows.parent_wf_3.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/066_core.control_flow.subworkflows.nested_parent_wf_2.pb b/pkg/compiler/test/testdata/snacks-core/066_core.control_flow.subworkflows.nested_parent_wf_2.pb new file mode 100644 index 0000000000..c9a27f0ea0 Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/066_core.control_flow.subworkflows.nested_parent_wf_2.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/067_core.control_flow.subworkflows.nested_parent_wf_3.pb b/pkg/compiler/test/testdata/snacks-core/067_core.control_flow.subworkflows.nested_parent_wf_3.pb new file mode 100644 index 0000000000..1cf3c7049b Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/067_core.control_flow.subworkflows.nested_parent_wf_3.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/068_core.control_flow.subworkflows.count_freq_words_1.pb b/pkg/compiler/test/testdata/snacks-core/068_core.control_flow.subworkflows.count_freq_words_1.pb new file mode 100644 index 0000000000..11f1bd2284 Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/068_core.control_flow.subworkflows.count_freq_words_1.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/068_core.control_flow.subworkflows.count_freq_words_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/068_core.control_flow.subworkflows.count_freq_words_1_task.yaml new file mode 100755 index 0000000000..ccd68f4c08 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/068_core.control_flow.subworkflows.count_freq_words_1_task.yaml @@ -0,0 +1,50 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.control_flow.subworkflows + - task-name + - count_freq_words + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.control_flow.subworkflows.count_freq_words + resource_type: 1 + interface: + inputs: + variables: + input_string1: + description: input_string1 + type: + Type: + Simple: 3 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 9 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/069_core.control_flow.subworkflows.ext_workflow_2.pb b/pkg/compiler/test/testdata/snacks-core/069_core.control_flow.subworkflows.ext_workflow_2.pb new file mode 100644 index 0000000000..824b349b06 Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/069_core.control_flow.subworkflows.ext_workflow_2.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/069_core.control_flow.subworkflows.ext_workflow_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/069_core.control_flow.subworkflows.ext_workflow_2_wf.yaml new file mode 100755 index 0000000000..770e0e51ee --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/069_core.control_flow.subworkflows.ext_workflow_2_wf.yaml @@ -0,0 +1,47 @@ +workflow: + id: + name: core.control_flow.subworkflows.ext_workflow + resource_type: 2 + interface: + inputs: + variables: + my_input: + description: my_input + type: + Type: + Simple: 3 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 9 + metadata: {} + metadata_defaults: {} + nodes: + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.control_flow.subworkflows.count_freq_words + resource_type: 1 + overrides: {} + id: n0 + inputs: + - binding: + Value: + Promise: + var: my_input + var: input_string1 + metadata: + InterruptibleValue: null + name: count_freq_words + retries: {} + outputs: + - binding: + Value: + Promise: + node_id: n0 + var: o0 + var: o0 diff --git a/pkg/compiler/test/testdata/snacks-core/070_core.control_flow.subworkflows.ext_workflow_3.pb b/pkg/compiler/test/testdata/snacks-core/070_core.control_flow.subworkflows.ext_workflow_3.pb new file mode 100644 index 0000000000..047f3fdd7d Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/070_core.control_flow.subworkflows.ext_workflow_3.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/071_parent_workflow_execution_3.pb b/pkg/compiler/test/testdata/snacks-core/071_parent_workflow_execution_3.pb new file mode 100644 index 0000000000..a59fe895ab Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/071_parent_workflow_execution_3.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/072_core.control_flow.subworkflows.count_repetitive_words_1.pb b/pkg/compiler/test/testdata/snacks-core/072_core.control_flow.subworkflows.count_repetitive_words_1.pb new file mode 100644 index 0000000000..2b2edf718e Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/072_core.control_flow.subworkflows.count_repetitive_words_1.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/072_core.control_flow.subworkflows.count_repetitive_words_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/072_core.control_flow.subworkflows.count_repetitive_words_1_task.yaml new file mode 100755 index 0000000000..007824cbce --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/072_core.control_flow.subworkflows.count_repetitive_words_1_task.yaml @@ -0,0 +1,52 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.control_flow.subworkflows + - task-name + - count_repetitive_words + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.control_flow.subworkflows.count_repetitive_words + resource_type: 1 + interface: + inputs: + variables: + word_counter: + description: word_counter + type: + Type: + Simple: 9 + outputs: + variables: + o0: + description: o0 + type: + Type: + CollectionType: + Type: + Simple: 3 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/075_my-objectstore-sensor_1.pb b/pkg/compiler/test/testdata/snacks-core/075_my-objectstore-sensor_1.pb new file mode 100644 index 0000000000..25714e0838 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/075_my-objectstore-sensor_1.pb @@ -0,0 +1,13 @@ + +} +"my-objectstore-sensorobject-store-sensor0.32.6python"° *( +", + + +path + +path + +path + +path \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/075_my-objectstore-sensor_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/075_my-objectstore-sensor_1_task.yaml new file mode 100755 index 0000000000..02c6d654ba --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/075_my-objectstore-sensor_1_task.yaml @@ -0,0 +1,31 @@ +template: + Target: null + id: + name: my-objectstore-sensor + resource_type: 1 + interface: + inputs: + variables: + path: + description: path + type: + Type: + Simple: 3 + outputs: + variables: + path: + description: path + type: + Type: + Simple: 3 + metadata: + InterruptibleValue: null + retries: + retries: 10 + runtime: + flavor: python + type: 1 + version: 0.32.6 + timeout: + seconds: 1200 + type: object-store-sensor diff --git a/pkg/compiler/test/testdata/snacks-core/076_core.extend_flyte.custom_task_plugin.print_file_1.pb b/pkg/compiler/test/testdata/snacks-core/076_core.extend_flyte.custom_task_plugin.print_file_1.pb new file mode 100644 index 0000000000..82aabf8da7 Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/076_core.extend_flyte.custom_task_plugin.print_file_1.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/076_core.extend_flyte.custom_task_plugin.print_file_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/076_core.extend_flyte.custom_task_plugin.print_file_1_task.yaml new file mode 100755 index 0000000000..c0da8ea5ba --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/076_core.extend_flyte.custom_task_plugin.print_file_1_task.yaml @@ -0,0 +1,50 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.extend_flyte.custom_task_plugin + - task-name + - print_file + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.extend_flyte.custom_task_plugin.print_file + resource_type: 1 + interface: + inputs: + variables: + path: + description: path + type: + Type: + Simple: 3 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 3 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/077_core.extend_flyte.custom_task_plugin.my_workflow_2.pb b/pkg/compiler/test/testdata/snacks-core/077_core.extend_flyte.custom_task_plugin.my_workflow_2.pb new file mode 100644 index 0000000000..1079c82193 Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/077_core.extend_flyte.custom_task_plugin.my_workflow_2.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/077_core.extend_flyte.custom_task_plugin.my_workflow_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/077_core.extend_flyte.custom_task_plugin.my_workflow_2_wf.yaml new file mode 100755 index 0000000000..4081c79f35 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/077_core.extend_flyte.custom_task_plugin.my_workflow_2_wf.yaml @@ -0,0 +1,71 @@ +workflow: + id: + name: core.extend_flyte.custom_task_plugin.my_workflow + resource_type: 2 + interface: + inputs: + variables: + path: + description: path + type: + Type: + Simple: 3 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 3 + metadata: {} + metadata_defaults: {} + nodes: + - Target: + TaskNode: + Reference: + ReferenceId: + name: my-objectstore-sensor + resource_type: 1 + overrides: {} + id: n0 + inputs: + - binding: + Value: + Promise: + var: path + var: path + metadata: + InterruptibleValue: null + name: my-objectstore-sensor + retries: + retries: 10 + timeout: + seconds: 1200 + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.extend_flyte.custom_task_plugin.print_file + resource_type: 1 + overrides: {} + id: n1 + inputs: + - binding: + Value: + Promise: + node_id: n0 + var: path + var: path + metadata: + InterruptibleValue: null + name: print_file + retries: {} + upstream_node_ids: + - n0 + outputs: + - binding: + Value: + Promise: + node_id: n1 + var: o0 + var: o0 diff --git a/pkg/compiler/test/testdata/snacks-core/078_core.extend_flyte.custom_task_plugin.my_workflow_3.pb b/pkg/compiler/test/testdata/snacks-core/078_core.extend_flyte.custom_task_plugin.my_workflow_3.pb new file mode 100644 index 0000000000..b68d68cfdc Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/078_core.extend_flyte.custom_task_plugin.my_workflow_3.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/079_core.extend_flyte.custom_types.generate_1.pb b/pkg/compiler/test/testdata/snacks-core/079_core.extend_flyte.custom_types.generate_1.pb new file mode 100644 index 0000000000..1063c90684 Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/079_core.extend_flyte.custom_types.generate_1.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/079_core.extend_flyte.custom_types.generate_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/079_core.extend_flyte.custom_types.generate_1_task.yaml new file mode 100755 index 0000000000..b181c9e6a3 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/079_core.extend_flyte.custom_types.generate_1_task.yaml @@ -0,0 +1,46 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.extend_flyte.custom_types + - task-name + - generate + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.extend_flyte.custom_types.generate + resource_type: 1 + interface: + inputs: {} + outputs: + variables: + o0: + description: o0 + type: + Type: + Blob: + dimensionality: 1 + format: binary + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/080_core.extend_flyte.custom_types.consume_1.pb b/pkg/compiler/test/testdata/snacks-core/080_core.extend_flyte.custom_types.consume_1.pb new file mode 100644 index 0000000000..562df735a7 Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/080_core.extend_flyte.custom_types.consume_1.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/080_core.extend_flyte.custom_types.consume_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/080_core.extend_flyte.custom_types.consume_1_task.yaml new file mode 100755 index 0000000000..aab1c9158f --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/080_core.extend_flyte.custom_types.consume_1_task.yaml @@ -0,0 +1,52 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.extend_flyte.custom_types + - task-name + - consume + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.extend_flyte.custom_types.consume + resource_type: 1 + interface: + inputs: + variables: + d: + description: d + type: + Type: + Blob: + dimensionality: 1 + format: binary + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 3 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/081_core.extend_flyte.custom_types.wf_2.pb b/pkg/compiler/test/testdata/snacks-core/081_core.extend_flyte.custom_types.wf_2.pb new file mode 100644 index 0000000000..b11b1e9508 Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/081_core.extend_flyte.custom_types.wf_2.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/081_core.extend_flyte.custom_types.wf_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/081_core.extend_flyte.custom_types.wf_2_wf.yaml new file mode 100755 index 0000000000..49ccbf1138 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/081_core.extend_flyte.custom_types.wf_2_wf.yaml @@ -0,0 +1,56 @@ +workflow: + id: + name: core.extend_flyte.custom_types.wf + resource_type: 2 + interface: + inputs: {} + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 3 + metadata: {} + metadata_defaults: {} + nodes: + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.extend_flyte.custom_types.generate + resource_type: 1 + overrides: {} + id: n0 + metadata: + InterruptibleValue: null + name: generate + retries: {} + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.extend_flyte.custom_types.consume + resource_type: 1 + overrides: {} + id: n1 + inputs: + - binding: + Value: + Promise: + node_id: n0 + var: o0 + var: d + metadata: + InterruptibleValue: null + name: consume + retries: {} + upstream_node_ids: + - n0 + outputs: + - binding: + Value: + Promise: + node_id: n1 + var: o0 + var: o0 diff --git a/pkg/compiler/test/testdata/snacks-core/082_core.extend_flyte.custom_types.wf_3.pb b/pkg/compiler/test/testdata/snacks-core/082_core.extend_flyte.custom_types.wf_3.pb new file mode 100644 index 0000000000..d5ae631d80 Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/082_core.extend_flyte.custom_types.wf_3.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/083_core.flyte_basics.basic_workflow.t1_1.pb b/pkg/compiler/test/testdata/snacks-core/083_core.flyte_basics.basic_workflow.t1_1.pb new file mode 100644 index 0000000000..c30d7f3b14 Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/083_core.flyte_basics.basic_workflow.t1_1.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/083_core.flyte_basics.basic_workflow.t1_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/083_core.flyte_basics.basic_workflow.t1_1_task.yaml new file mode 100755 index 0000000000..019c428445 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/083_core.flyte_basics.basic_workflow.t1_1_task.yaml @@ -0,0 +1,55 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.basic_workflow + - task-name + - t1 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.flyte_basics.basic_workflow.t1 + resource_type: 1 + interface: + inputs: + variables: + a: + description: a + type: + Type: + Simple: 1 + outputs: + variables: + c: + description: c + type: + Type: + Simple: 3 + t1_int_output: + description: t1_int_output + type: + Type: + Simple: 1 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/084_core.flyte_basics.basic_workflow.t2_1.pb b/pkg/compiler/test/testdata/snacks-core/084_core.flyte_basics.basic_workflow.t2_1.pb new file mode 100644 index 0000000000..d315454c9e Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/084_core.flyte_basics.basic_workflow.t2_1.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/084_core.flyte_basics.basic_workflow.t2_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/084_core.flyte_basics.basic_workflow.t2_1_task.yaml new file mode 100755 index 0000000000..90c33f2dfb --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/084_core.flyte_basics.basic_workflow.t2_1_task.yaml @@ -0,0 +1,55 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.basic_workflow + - task-name + - t2 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.flyte_basics.basic_workflow.t2 + resource_type: 1 + interface: + inputs: + variables: + a: + description: a + type: + Type: + Simple: 3 + b: + description: b + type: + Type: + Simple: 3 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 3 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/085_core.flyte_basics.basic_workflow.my_wf_2.pb b/pkg/compiler/test/testdata/snacks-core/085_core.flyte_basics.basic_workflow.my_wf_2.pb new file mode 100644 index 0000000000..4d8dbc8f86 Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/085_core.flyte_basics.basic_workflow.my_wf_2.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/085_core.flyte_basics.basic_workflow.my_wf_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/085_core.flyte_basics.basic_workflow.my_wf_2_wf.yaml new file mode 100755 index 0000000000..2099dc484b --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/085_core.flyte_basics.basic_workflow.my_wf_2_wf.yaml @@ -0,0 +1,89 @@ +workflow: + id: + name: core.flyte_basics.basic_workflow.my_wf + resource_type: 2 + interface: + inputs: + variables: + a: + description: a + type: + Type: + Simple: 1 + b: + description: b + type: + Type: + Simple: 3 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 1 + o1: + description: o1 + type: + Type: + Simple: 3 + metadata: {} + metadata_defaults: {} + nodes: + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.flyte_basics.basic_workflow.t1 + resource_type: 1 + overrides: {} + id: n0 + inputs: + - binding: + Value: + Promise: + var: a + var: a + metadata: + InterruptibleValue: null + name: t1 + retries: {} + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.flyte_basics.basic_workflow.t2 + resource_type: 1 + overrides: {} + id: n1 + inputs: + - binding: + Value: + Promise: + node_id: n0 + var: c + var: a + - binding: + Value: + Promise: + var: b + var: b + metadata: + InterruptibleValue: null + name: t2 + retries: {} + upstream_node_ids: + - n0 + outputs: + - binding: + Value: + Promise: + node_id: n0 + var: t1_int_output + var: o0 + - binding: + Value: + Promise: + node_id: n1 + var: o0 + var: o1 diff --git a/pkg/compiler/test/testdata/snacks-core/086_core.flyte_basics.basic_workflow.my_wf_3.pb b/pkg/compiler/test/testdata/snacks-core/086_core.flyte_basics.basic_workflow.my_wf_3.pb new file mode 100644 index 0000000000..72e86a0eaf Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/086_core.flyte_basics.basic_workflow.my_wf_3.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/087_core.flyte_basics.decorating_tasks.t1_1.pb b/pkg/compiler/test/testdata/snacks-core/087_core.flyte_basics.decorating_tasks.t1_1.pb new file mode 100644 index 0000000000..11295e9fcc Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/087_core.flyte_basics.decorating_tasks.t1_1.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/087_core.flyte_basics.decorating_tasks.t1_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/087_core.flyte_basics.decorating_tasks.t1_1_task.yaml new file mode 100755 index 0000000000..ecaffc4539 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/087_core.flyte_basics.decorating_tasks.t1_1_task.yaml @@ -0,0 +1,50 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.decorating_tasks + - task-name + - t1 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.flyte_basics.decorating_tasks.t1 + resource_type: 1 + interface: + inputs: + variables: + x: + description: x + type: + Type: + Simple: 1 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 1 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/088_core.flyte_basics.decorating_tasks.t2_1.pb b/pkg/compiler/test/testdata/snacks-core/088_core.flyte_basics.decorating_tasks.t2_1.pb new file mode 100644 index 0000000000..1693412a5e Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/088_core.flyte_basics.decorating_tasks.t2_1.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/088_core.flyte_basics.decorating_tasks.t2_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/088_core.flyte_basics.decorating_tasks.t2_1_task.yaml new file mode 100755 index 0000000000..90279ea568 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/088_core.flyte_basics.decorating_tasks.t2_1_task.yaml @@ -0,0 +1,50 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.decorating_tasks + - task-name + - t2 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.flyte_basics.decorating_tasks.t2 + resource_type: 1 + interface: + inputs: + variables: + x: + description: x + type: + Type: + Simple: 1 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 1 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/089_core.flyte_basics.decorating_tasks.wf_2.pb b/pkg/compiler/test/testdata/snacks-core/089_core.flyte_basics.decorating_tasks.wf_2.pb new file mode 100644 index 0000000000..a0cdad11a3 Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/089_core.flyte_basics.decorating_tasks.wf_2.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/089_core.flyte_basics.decorating_tasks.wf_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/089_core.flyte_basics.decorating_tasks.wf_2_wf.yaml new file mode 100755 index 0000000000..8180079fc3 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/089_core.flyte_basics.decorating_tasks.wf_2_wf.yaml @@ -0,0 +1,68 @@ +workflow: + id: + name: core.flyte_basics.decorating_tasks.wf + resource_type: 2 + interface: + inputs: + variables: + x: + description: x + type: + Type: + Simple: 1 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 1 + metadata: {} + metadata_defaults: {} + nodes: + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.flyte_basics.decorating_tasks.t1 + resource_type: 1 + overrides: {} + id: n0 + inputs: + - binding: + Value: + Promise: + var: x + var: x + metadata: + InterruptibleValue: null + name: t1 + retries: {} + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.flyte_basics.decorating_tasks.t2 + resource_type: 1 + overrides: {} + id: n1 + inputs: + - binding: + Value: + Promise: + node_id: n0 + var: o0 + var: x + metadata: + InterruptibleValue: null + name: t2 + retries: {} + upstream_node_ids: + - n0 + outputs: + - binding: + Value: + Promise: + node_id: n1 + var: o0 + var: o0 diff --git a/pkg/compiler/test/testdata/snacks-core/090_core.flyte_basics.decorating_tasks.wf_3.pb b/pkg/compiler/test/testdata/snacks-core/090_core.flyte_basics.decorating_tasks.wf_3.pb new file mode 100644 index 0000000000..a2e74f4a50 Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/090_core.flyte_basics.decorating_tasks.wf_3.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/091_core.flyte_basics.decorating_workflows.setup_1.pb b/pkg/compiler/test/testdata/snacks-core/091_core.flyte_basics.decorating_workflows.setup_1.pb new file mode 100644 index 0000000000..8ec9681711 Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/091_core.flyte_basics.decorating_workflows.setup_1.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/091_core.flyte_basics.decorating_workflows.setup_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/091_core.flyte_basics.decorating_workflows.setup_1_task.yaml new file mode 100755 index 0000000000..d6993f7125 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/091_core.flyte_basics.decorating_workflows.setup_1_task.yaml @@ -0,0 +1,38 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.decorating_workflows + - task-name + - setup + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.flyte_basics.decorating_workflows.setup + resource_type: 1 + interface: + inputs: {} + outputs: {} + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/092_core.flyte_basics.decorating_workflows.teardown_1.pb b/pkg/compiler/test/testdata/snacks-core/092_core.flyte_basics.decorating_workflows.teardown_1.pb new file mode 100644 index 0000000000..deb55ff500 Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/092_core.flyte_basics.decorating_workflows.teardown_1.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/092_core.flyte_basics.decorating_workflows.teardown_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/092_core.flyte_basics.decorating_workflows.teardown_1_task.yaml new file mode 100755 index 0000000000..52043f2f6f --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/092_core.flyte_basics.decorating_workflows.teardown_1_task.yaml @@ -0,0 +1,38 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.decorating_workflows + - task-name + - teardown + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.flyte_basics.decorating_workflows.teardown + resource_type: 1 + interface: + inputs: {} + outputs: {} + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/093_core.flyte_basics.decorating_workflows.t1_1.pb b/pkg/compiler/test/testdata/snacks-core/093_core.flyte_basics.decorating_workflows.t1_1.pb new file mode 100644 index 0000000000..70d7e177c0 Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/093_core.flyte_basics.decorating_workflows.t1_1.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/093_core.flyte_basics.decorating_workflows.t1_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/093_core.flyte_basics.decorating_workflows.t1_1_task.yaml new file mode 100755 index 0000000000..6445ac6546 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/093_core.flyte_basics.decorating_workflows.t1_1_task.yaml @@ -0,0 +1,50 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.decorating_workflows + - task-name + - t1 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.flyte_basics.decorating_workflows.t1 + resource_type: 1 + interface: + inputs: + variables: + x: + description: x + type: + Type: + Simple: 2 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 2 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/094_core.flyte_basics.decorating_workflows.t2_1.pb b/pkg/compiler/test/testdata/snacks-core/094_core.flyte_basics.decorating_workflows.t2_1.pb new file mode 100644 index 0000000000..dd1333f0c0 Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/094_core.flyte_basics.decorating_workflows.t2_1.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/094_core.flyte_basics.decorating_workflows.t2_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/094_core.flyte_basics.decorating_workflows.t2_1_task.yaml new file mode 100755 index 0000000000..d114ac57bf --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/094_core.flyte_basics.decorating_workflows.t2_1_task.yaml @@ -0,0 +1,50 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.decorating_workflows + - task-name + - t2 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.flyte_basics.decorating_workflows.t2 + resource_type: 1 + interface: + inputs: + variables: + x: + description: x + type: + Type: + Simple: 2 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 2 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/095_core.flyte_basics.decorating_workflows.wf_2.pb b/pkg/compiler/test/testdata/snacks-core/095_core.flyte_basics.decorating_workflows.wf_2.pb new file mode 100644 index 0000000000..efc87e973e Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/095_core.flyte_basics.decorating_workflows.wf_2.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/095_core.flyte_basics.decorating_workflows.wf_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/095_core.flyte_basics.decorating_workflows.wf_2_wf.yaml new file mode 100755 index 0000000000..c9ee70a4d9 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/095_core.flyte_basics.decorating_workflows.wf_2_wf.yaml @@ -0,0 +1,96 @@ +workflow: + id: + name: core.flyte_basics.decorating_workflows.wf + resource_type: 2 + interface: + inputs: + variables: + x: + description: x + type: + Type: + Simple: 2 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 2 + metadata: {} + metadata_defaults: {} + nodes: + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.flyte_basics.decorating_workflows.setup + resource_type: 1 + overrides: {} + id: n0 + metadata: + InterruptibleValue: null + name: setup + retries: {} + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.flyte_basics.decorating_workflows.t1 + resource_type: 1 + overrides: {} + id: n1 + inputs: + - binding: + Value: + Promise: + var: x + var: x + metadata: + InterruptibleValue: null + name: t1 + retries: {} + upstream_node_ids: + - n0 + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.flyte_basics.decorating_workflows.t2 + resource_type: 1 + overrides: {} + id: n2 + inputs: + - binding: + Value: + Promise: + node_id: n1 + var: o0 + var: x + metadata: + InterruptibleValue: null + name: t2 + retries: {} + upstream_node_ids: + - n1 + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.flyte_basics.decorating_workflows.teardown + resource_type: 1 + overrides: {} + id: n3 + metadata: + InterruptibleValue: null + name: teardown + retries: {} + upstream_node_ids: + - n2 + outputs: + - binding: + Value: + Promise: + node_id: n2 + var: o0 + var: o0 diff --git a/pkg/compiler/test/testdata/snacks-core/096_core.flyte_basics.decorating_workflows.wf_3.pb b/pkg/compiler/test/testdata/snacks-core/096_core.flyte_basics.decorating_workflows.wf_3.pb new file mode 100644 index 0000000000..c1707a1027 Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/096_core.flyte_basics.decorating_workflows.wf_3.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/097_core.flyte_basics.documented_workflow.add_data_1.pb b/pkg/compiler/test/testdata/snacks-core/097_core.flyte_basics.documented_workflow.add_data_1.pb new file mode 100644 index 0000000000..e95fa28dde Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/097_core.flyte_basics.documented_workflow.add_data_1.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/097_core.flyte_basics.documented_workflow.add_data_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/097_core.flyte_basics.documented_workflow.add_data_1_task.yaml new file mode 100755 index 0000000000..d21207c4fa --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/097_core.flyte_basics.documented_workflow.add_data_1_task.yaml @@ -0,0 +1,73 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.documented_workflow + - task-name + - add_data + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.flyte_basics.documented_workflow.add_data + resource_type: 1 + interface: + inputs: + variables: + data: + description: data + type: + Type: + Simple: 9 + metadata: + $ref: '#/definitions/PandasdataSchema' + $schema: http://json-schema.org/draft-07/schema# + definitions: + PandasdataSchema: + additionalProperties: false + properties: + id: + default: 3 + title: id + type: integer + name: + default: Bonnie + title: name + type: string + type: object + df: + description: df + type: + Type: + StructuredDatasetType: + format: parquet + outputs: + variables: + o0: + description: o0 + type: + Type: + StructuredDatasetType: + format: parquet + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/098_core.flyte_basics.documented_workflow.sphinx_docstring_2.pb b/pkg/compiler/test/testdata/snacks-core/098_core.flyte_basics.documented_workflow.sphinx_docstring_2.pb new file mode 100644 index 0000000000..415004d373 Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/098_core.flyte_basics.documented_workflow.sphinx_docstring_2.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/098_core.flyte_basics.documented_workflow.sphinx_docstring_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/098_core.flyte_basics.documented_workflow.sphinx_docstring_2_wf.yaml new file mode 100755 index 0000000000..d55a5f2d8a --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/098_core.flyte_basics.documented_workflow.sphinx_docstring_2_wf.yaml @@ -0,0 +1,76 @@ +workflow: + id: + name: core.flyte_basics.documented_workflow.sphinx_docstring + resource_type: 2 + interface: + inputs: + variables: + data: + description: A data class pertaining to the new record to be stored in the + DataFrame + type: + Type: + Simple: 9 + metadata: + $ref: '#/definitions/PandasdataSchema' + $schema: http://json-schema.org/draft-07/schema# + definitions: + PandasdataSchema: + additionalProperties: false + properties: + id: + default: 3 + title: id + type: integer + name: + default: Bonnie + title: name + type: string + type: object + df: + description: Pandas DataFrame + type: + Type: + StructuredDatasetType: + format: parquet + outputs: + variables: + o0: + description: Pandas DataFrame + type: + Type: + StructuredDatasetType: + format: parquet + metadata: {} + metadata_defaults: {} + nodes: + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.flyte_basics.documented_workflow.add_data + resource_type: 1 + overrides: {} + id: n0 + inputs: + - binding: + Value: + Promise: + var: data + var: data + - binding: + Value: + Promise: + var: df + var: df + metadata: + InterruptibleValue: null + name: add_data + retries: {} + outputs: + - binding: + Value: + Promise: + node_id: n0 + var: o0 + var: o0 diff --git a/pkg/compiler/test/testdata/snacks-core/099_core.flyte_basics.documented_workflow.sphinx_docstring_3.pb b/pkg/compiler/test/testdata/snacks-core/099_core.flyte_basics.documented_workflow.sphinx_docstring_3.pb new file mode 100644 index 0000000000..f62b93baf3 Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/099_core.flyte_basics.documented_workflow.sphinx_docstring_3.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/100_core.flyte_basics.documented_workflow.numpy_docstring_2.pb b/pkg/compiler/test/testdata/snacks-core/100_core.flyte_basics.documented_workflow.numpy_docstring_2.pb new file mode 100644 index 0000000000..2e7a85b03a Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/100_core.flyte_basics.documented_workflow.numpy_docstring_2.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/100_core.flyte_basics.documented_workflow.numpy_docstring_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/100_core.flyte_basics.documented_workflow.numpy_docstring_2_wf.yaml new file mode 100755 index 0000000000..f631a303cb --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/100_core.flyte_basics.documented_workflow.numpy_docstring_2_wf.yaml @@ -0,0 +1,76 @@ +workflow: + id: + name: core.flyte_basics.documented_workflow.numpy_docstring + resource_type: 2 + interface: + inputs: + variables: + data: + description: A data class pertaining to the new record to be stored in the + DataFrame + type: + Type: + Simple: 9 + metadata: + $ref: '#/definitions/PandasdataSchema' + $schema: http://json-schema.org/draft-07/schema# + definitions: + PandasdataSchema: + additionalProperties: false + properties: + id: + default: 3 + title: id + type: integer + name: + default: Bonnie + title: name + type: string + type: object + df: + description: Pandas DataFrame + type: + Type: + StructuredDatasetType: + format: parquet + outputs: + variables: + o0: + description: Pandas DataFrame + type: + Type: + StructuredDatasetType: + format: parquet + metadata: {} + metadata_defaults: {} + nodes: + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.flyte_basics.documented_workflow.add_data + resource_type: 1 + overrides: {} + id: n0 + inputs: + - binding: + Value: + Promise: + var: data + var: data + - binding: + Value: + Promise: + var: df + var: df + metadata: + InterruptibleValue: null + name: add_data + retries: {} + outputs: + - binding: + Value: + Promise: + node_id: n0 + var: o0 + var: o0 diff --git a/pkg/compiler/test/testdata/snacks-core/101_core.flyte_basics.documented_workflow.numpy_docstring_3.pb b/pkg/compiler/test/testdata/snacks-core/101_core.flyte_basics.documented_workflow.numpy_docstring_3.pb new file mode 100644 index 0000000000..79054323a3 Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/101_core.flyte_basics.documented_workflow.numpy_docstring_3.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/102_core.flyte_basics.documented_workflow.google_docstring_2.pb b/pkg/compiler/test/testdata/snacks-core/102_core.flyte_basics.documented_workflow.google_docstring_2.pb new file mode 100644 index 0000000000..6ea6d4790e Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/102_core.flyte_basics.documented_workflow.google_docstring_2.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/102_core.flyte_basics.documented_workflow.google_docstring_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/102_core.flyte_basics.documented_workflow.google_docstring_2_wf.yaml new file mode 100755 index 0000000000..dd60d2fce1 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/102_core.flyte_basics.documented_workflow.google_docstring_2_wf.yaml @@ -0,0 +1,76 @@ +workflow: + id: + name: core.flyte_basics.documented_workflow.google_docstring + resource_type: 2 + interface: + inputs: + variables: + data: + description: A data class pertaining to the new record to be stored in the + DataFrame + type: + Type: + Simple: 9 + metadata: + $ref: '#/definitions/PandasdataSchema' + $schema: http://json-schema.org/draft-07/schema# + definitions: + PandasdataSchema: + additionalProperties: false + properties: + id: + default: 3 + title: id + type: integer + name: + default: Bonnie + title: name + type: string + type: object + df: + description: Pandas DataFrame + type: + Type: + StructuredDatasetType: + format: parquet + outputs: + variables: + o0: + description: Pandas DataFrame + type: + Type: + StructuredDatasetType: + format: parquet + metadata: {} + metadata_defaults: {} + nodes: + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.flyte_basics.documented_workflow.add_data + resource_type: 1 + overrides: {} + id: n0 + inputs: + - binding: + Value: + Promise: + var: data + var: data + - binding: + Value: + Promise: + var: df + var: df + metadata: + InterruptibleValue: null + name: add_data + retries: {} + outputs: + - binding: + Value: + Promise: + node_id: n0 + var: o0 + var: o0 diff --git a/pkg/compiler/test/testdata/snacks-core/103_core.flyte_basics.documented_workflow.google_docstring_3.pb b/pkg/compiler/test/testdata/snacks-core/103_core.flyte_basics.documented_workflow.google_docstring_3.pb new file mode 100644 index 0000000000..a4bfe6c611 Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/103_core.flyte_basics.documented_workflow.google_docstring_3.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/104_core.flyte_basics.files.normalize_columns_1.pb b/pkg/compiler/test/testdata/snacks-core/104_core.flyte_basics.files.normalize_columns_1.pb new file mode 100644 index 0000000000..7f80ccae9e Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/104_core.flyte_basics.files.normalize_columns_1.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/104_core.flyte_basics.files.normalize_columns_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/104_core.flyte_basics.files.normalize_columns_1_task.yaml new file mode 100755 index 0000000000..87f25cb951 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/104_core.flyte_basics.files.normalize_columns_1_task.yaml @@ -0,0 +1,69 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.files + - task-name + - normalize_columns + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.flyte_basics.files.normalize_columns + resource_type: 1 + interface: + inputs: + variables: + column_names: + description: column_names + type: + Type: + CollectionType: + Type: + Simple: 3 + columns_to_normalize: + description: columns_to_normalize + type: + Type: + CollectionType: + Type: + Simple: 3 + csv_url: + description: csv_url + type: + Type: + Blob: {} + output_location: + description: output_location + type: + Type: + Simple: 3 + outputs: + variables: + o0: + description: o0 + type: + Type: + Blob: {} + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/105_core.flyte_basics.files.normalize_csv_file_2.pb b/pkg/compiler/test/testdata/snacks-core/105_core.flyte_basics.files.normalize_csv_file_2.pb new file mode 100644 index 0000000000..fd1a511816 Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/105_core.flyte_basics.files.normalize_csv_file_2.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/105_core.flyte_basics.files.normalize_csv_file_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/105_core.flyte_basics.files.normalize_csv_file_2_wf.yaml new file mode 100755 index 0000000000..73152948fa --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/105_core.flyte_basics.files.normalize_csv_file_2_wf.yaml @@ -0,0 +1,81 @@ +workflow: + id: + name: core.flyte_basics.files.normalize_csv_file + resource_type: 2 + interface: + inputs: + variables: + column_names: + description: column_names + type: + Type: + CollectionType: + Type: + Simple: 3 + columns_to_normalize: + description: columns_to_normalize + type: + Type: + CollectionType: + Type: + Simple: 3 + csv_url: + description: csv_url + type: + Type: + Blob: {} + output_location: + description: output_location + type: + Type: + Simple: 3 + outputs: + variables: + o0: + description: o0 + type: + Type: + Blob: {} + metadata: {} + metadata_defaults: {} + nodes: + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.flyte_basics.files.normalize_columns + resource_type: 1 + overrides: {} + id: n0 + inputs: + - binding: + Value: + Promise: + var: column_names + var: column_names + - binding: + Value: + Promise: + var: columns_to_normalize + var: columns_to_normalize + - binding: + Value: + Promise: + var: csv_url + var: csv_url + - binding: + Value: + Promise: + var: output_location + var: output_location + metadata: + InterruptibleValue: null + name: normalize_columns + retries: {} + outputs: + - binding: + Value: + Promise: + node_id: n0 + var: o0 + var: o0 diff --git a/pkg/compiler/test/testdata/snacks-core/106_core.flyte_basics.files.normalize_csv_file_3.pb b/pkg/compiler/test/testdata/snacks-core/106_core.flyte_basics.files.normalize_csv_file_3.pb new file mode 100644 index 0000000000..8812d19702 Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/106_core.flyte_basics.files.normalize_csv_file_3.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/107_core.flyte_basics.folders.download_files_1.pb b/pkg/compiler/test/testdata/snacks-core/107_core.flyte_basics.folders.download_files_1.pb new file mode 100644 index 0000000000..fb5b33d34f Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/107_core.flyte_basics.folders.download_files_1.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/107_core.flyte_basics.folders.download_files_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/107_core.flyte_basics.folders.download_files_1_task.yaml new file mode 100755 index 0000000000..3ead97304e --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/107_core.flyte_basics.folders.download_files_1_task.yaml @@ -0,0 +1,53 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.folders + - task-name + - download_files + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.flyte_basics.folders.download_files + resource_type: 1 + interface: + inputs: + variables: + csv_urls: + description: csv_urls + type: + Type: + CollectionType: + Type: + Simple: 3 + outputs: + variables: + o0: + description: o0 + type: + Type: + Blob: + dimensionality: 1 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/108_core.flyte_basics.folders.normalize_all_files_1.pb b/pkg/compiler/test/testdata/snacks-core/108_core.flyte_basics.folders.normalize_all_files_1.pb new file mode 100644 index 0000000000..510ad9819b Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/108_core.flyte_basics.folders.normalize_all_files_1.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/108_core.flyte_basics.folders.normalize_all_files_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/108_core.flyte_basics.folders.normalize_all_files_1_task.yaml new file mode 100755 index 0000000000..c39b43cbfe --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/108_core.flyte_basics.folders.normalize_all_files_1_task.yaml @@ -0,0 +1,70 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.folders + - task-name + - normalize_all_files + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.flyte_basics.folders.normalize_all_files + resource_type: 1 + interface: + inputs: + variables: + columns_metadata: + description: columns_metadata + type: + Type: + CollectionType: + Type: + CollectionType: + Type: + Simple: 3 + columns_to_normalize_metadata: + description: columns_to_normalize_metadata + type: + Type: + CollectionType: + Type: + CollectionType: + Type: + Simple: 3 + csv_files_dir: + description: csv_files_dir + type: + Type: + Blob: + dimensionality: 1 + outputs: + variables: + o0: + description: o0 + type: + Type: + Blob: + dimensionality: 1 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/109_core.flyte_basics.folders.download_and_normalize_csv_files_2.pb b/pkg/compiler/test/testdata/snacks-core/109_core.flyte_basics.folders.download_and_normalize_csv_files_2.pb new file mode 100644 index 0000000000..c6caa3ff80 Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/109_core.flyte_basics.folders.download_and_normalize_csv_files_2.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/109_core.flyte_basics.folders.download_and_normalize_csv_files_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/109_core.flyte_basics.folders.download_and_normalize_csv_files_2_wf.yaml new file mode 100755 index 0000000000..d9d9f6a8a7 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/109_core.flyte_basics.folders.download_and_normalize_csv_files_2_wf.yaml @@ -0,0 +1,99 @@ +workflow: + id: + name: core.flyte_basics.folders.download_and_normalize_csv_files + resource_type: 2 + interface: + inputs: + variables: + columns_metadata: + description: columns_metadata + type: + Type: + CollectionType: + Type: + CollectionType: + Type: + Simple: 3 + columns_to_normalize_metadata: + description: columns_to_normalize_metadata + type: + Type: + CollectionType: + Type: + CollectionType: + Type: + Simple: 3 + csv_urls: + description: csv_urls + type: + Type: + CollectionType: + Type: + Simple: 3 + outputs: + variables: + o0: + description: o0 + type: + Type: + Blob: + dimensionality: 1 + metadata: {} + metadata_defaults: {} + nodes: + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.flyte_basics.folders.download_files + resource_type: 1 + overrides: {} + id: n0 + inputs: + - binding: + Value: + Promise: + var: csv_urls + var: csv_urls + metadata: + InterruptibleValue: null + name: download_files + retries: {} + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.flyte_basics.folders.normalize_all_files + resource_type: 1 + overrides: {} + id: n1 + inputs: + - binding: + Value: + Promise: + var: columns_metadata + var: columns_metadata + - binding: + Value: + Promise: + var: columns_to_normalize_metadata + var: columns_to_normalize_metadata + - binding: + Value: + Promise: + node_id: n0 + var: o0 + var: csv_files_dir + metadata: + InterruptibleValue: null + name: normalize_all_files + retries: {} + upstream_node_ids: + - n0 + outputs: + - binding: + Value: + Promise: + node_id: n1 + var: o0 + var: o0 diff --git a/pkg/compiler/test/testdata/snacks-core/110_core.flyte_basics.folders.download_and_normalize_csv_files_3.pb b/pkg/compiler/test/testdata/snacks-core/110_core.flyte_basics.folders.download_and_normalize_csv_files_3.pb new file mode 100644 index 0000000000..6c8000f55a Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/110_core.flyte_basics.folders.download_and_normalize_csv_files_3.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/111_core.flyte_basics.hello_world.say_hello_1.pb b/pkg/compiler/test/testdata/snacks-core/111_core.flyte_basics.hello_world.say_hello_1.pb new file mode 100644 index 0000000000..68279d1517 Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/111_core.flyte_basics.hello_world.say_hello_1.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/111_core.flyte_basics.hello_world.say_hello_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/111_core.flyte_basics.hello_world.say_hello_1_task.yaml new file mode 100755 index 0000000000..deaaaca069 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/111_core.flyte_basics.hello_world.say_hello_1_task.yaml @@ -0,0 +1,44 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.hello_world + - task-name + - say_hello + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.flyte_basics.hello_world.say_hello + resource_type: 1 + interface: + inputs: {} + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 3 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/112_core.flyte_basics.hello_world.my_wf_2.pb b/pkg/compiler/test/testdata/snacks-core/112_core.flyte_basics.hello_world.my_wf_2.pb new file mode 100644 index 0000000000..6a3c40d879 Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/112_core.flyte_basics.hello_world.my_wf_2.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/112_core.flyte_basics.hello_world.my_wf_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/112_core.flyte_basics.hello_world.my_wf_2_wf.yaml new file mode 100755 index 0000000000..1e21071068 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/112_core.flyte_basics.hello_world.my_wf_2_wf.yaml @@ -0,0 +1,35 @@ +workflow: + id: + name: core.flyte_basics.hello_world.my_wf + resource_type: 2 + interface: + inputs: {} + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 3 + metadata: {} + metadata_defaults: {} + nodes: + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.flyte_basics.hello_world.say_hello + resource_type: 1 + overrides: {} + id: n0 + metadata: + InterruptibleValue: null + name: say_hello + retries: {} + outputs: + - binding: + Value: + Promise: + node_id: n0 + var: o0 + var: o0 diff --git a/pkg/compiler/test/testdata/snacks-core/113_core.flyte_basics.hello_world.my_wf_3.pb b/pkg/compiler/test/testdata/snacks-core/113_core.flyte_basics.hello_world.my_wf_3.pb new file mode 100644 index 0000000000..aace6afbd7 Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/113_core.flyte_basics.hello_world.my_wf_3.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/114_core.flyte_basics.imperative_wf_style.t1_1.pb b/pkg/compiler/test/testdata/snacks-core/114_core.flyte_basics.imperative_wf_style.t1_1.pb new file mode 100644 index 0000000000..a754aaba01 Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/114_core.flyte_basics.imperative_wf_style.t1_1.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/114_core.flyte_basics.imperative_wf_style.t1_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/114_core.flyte_basics.imperative_wf_style.t1_1_task.yaml new file mode 100755 index 0000000000..39fec84612 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/114_core.flyte_basics.imperative_wf_style.t1_1_task.yaml @@ -0,0 +1,50 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.imperative_wf_style + - task-name + - t1 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.flyte_basics.imperative_wf_style.t1 + resource_type: 1 + interface: + inputs: + variables: + a: + description: a + type: + Type: + Simple: 3 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 3 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/115_core.flyte_basics.imperative_wf_style.t2_1.pb b/pkg/compiler/test/testdata/snacks-core/115_core.flyte_basics.imperative_wf_style.t2_1.pb new file mode 100644 index 0000000000..59489e9c2f Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/115_core.flyte_basics.imperative_wf_style.t2_1.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/115_core.flyte_basics.imperative_wf_style.t2_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/115_core.flyte_basics.imperative_wf_style.t2_1_task.yaml new file mode 100755 index 0000000000..59968930ec --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/115_core.flyte_basics.imperative_wf_style.t2_1_task.yaml @@ -0,0 +1,38 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.imperative_wf_style + - task-name + - t2 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.flyte_basics.imperative_wf_style.t2 + resource_type: 1 + interface: + inputs: {} + outputs: {} + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/116_core.flyte_basics.imperative_wf_style.t3_1.pb b/pkg/compiler/test/testdata/snacks-core/116_core.flyte_basics.imperative_wf_style.t3_1.pb new file mode 100644 index 0000000000..257cd78d5f Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/116_core.flyte_basics.imperative_wf_style.t3_1.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/116_core.flyte_basics.imperative_wf_style.t3_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/116_core.flyte_basics.imperative_wf_style.t3_1_task.yaml new file mode 100755 index 0000000000..e66300258c --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/116_core.flyte_basics.imperative_wf_style.t3_1_task.yaml @@ -0,0 +1,52 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.imperative_wf_style + - task-name + - t3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.flyte_basics.imperative_wf_style.t3 + resource_type: 1 + interface: + inputs: + variables: + a: + description: a + type: + Type: + CollectionType: + Type: + Simple: 3 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 3 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/117_my.imperative.workflow.example_2.pb b/pkg/compiler/test/testdata/snacks-core/117_my.imperative.workflow.example_2.pb new file mode 100644 index 0000000000..366bea9b10 Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/117_my.imperative.workflow.example_2.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/117_my.imperative.workflow.example_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/117_my.imperative.workflow.example_2_wf.yaml new file mode 100755 index 0000000000..ccfef725f2 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/117_my.imperative.workflow.example_2_wf.yaml @@ -0,0 +1,108 @@ +workflow: + id: + name: my.imperative.workflow.example + resource_type: 2 + interface: + inputs: + variables: + in1: + description: in1 + type: + Type: + Simple: 3 + in2: + description: in2 + type: + Type: + Simple: 3 + outputs: + variables: + output_from_t1: + description: output_from_t1 + type: + Type: + Simple: 3 + output_list: + description: output_list + type: + Type: + CollectionType: + Type: + Simple: 3 + metadata: {} + metadata_defaults: {} + nodes: + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.flyte_basics.imperative_wf_style.t1 + resource_type: 1 + overrides: {} + id: n0 + inputs: + - binding: + Value: + Promise: + var: in1 + var: a + metadata: + InterruptibleValue: null + name: t1 + retries: {} + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.flyte_basics.imperative_wf_style.t2 + resource_type: 1 + overrides: {} + id: n1 + metadata: + InterruptibleValue: null + name: t2 + retries: {} + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.flyte_basics.imperative_wf_style.t3 + resource_type: 1 + overrides: {} + id: n2 + inputs: + - binding: + Value: + Collection: + bindings: + - Value: + Promise: + var: in1 + - Value: + Promise: + var: in2 + var: a + metadata: + InterruptibleValue: null + name: t3 + retries: {} + outputs: + - binding: + Value: + Promise: + node_id: n0 + var: o0 + var: output_from_t1 + - binding: + Value: + Collection: + bindings: + - Value: + Promise: + node_id: n0 + var: o0 + - Value: + Promise: + node_id: n2 + var: o0 + var: output_list diff --git a/pkg/compiler/test/testdata/snacks-core/118_my.imperative.workflow.example_3.pb b/pkg/compiler/test/testdata/snacks-core/118_my.imperative.workflow.example_3.pb new file mode 100644 index 0000000000..f2e2b8e83b Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/118_my.imperative.workflow.example_3.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/119_core.flyte_basics.lp.square_1.pb b/pkg/compiler/test/testdata/snacks-core/119_core.flyte_basics.lp.square_1.pb new file mode 100644 index 0000000000..8196beb3ee Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/119_core.flyte_basics.lp.square_1.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/119_core.flyte_basics.lp.square_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/119_core.flyte_basics.lp.square_1_task.yaml new file mode 100755 index 0000000000..1123312891 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/119_core.flyte_basics.lp.square_1_task.yaml @@ -0,0 +1,50 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.lp + - task-name + - square + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.flyte_basics.lp.square + resource_type: 1 + interface: + inputs: + variables: + val: + description: val + type: + Type: + Simple: 1 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 1 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/120_core.flyte_basics.lp.my_wf_2.pb b/pkg/compiler/test/testdata/snacks-core/120_core.flyte_basics.lp.my_wf_2.pb new file mode 100644 index 0000000000..ccc83c58ad Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/120_core.flyte_basics.lp.my_wf_2.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/120_core.flyte_basics.lp.my_wf_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/120_core.flyte_basics.lp.my_wf_2_wf.yaml new file mode 100755 index 0000000000..2cdb50c526 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/120_core.flyte_basics.lp.my_wf_2_wf.yaml @@ -0,0 +1,47 @@ +workflow: + id: + name: core.flyte_basics.lp.my_wf + resource_type: 2 + interface: + inputs: + variables: + val: + description: val + type: + Type: + Simple: 1 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 1 + metadata: {} + metadata_defaults: {} + nodes: + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.flyte_basics.lp.square + resource_type: 1 + overrides: {} + id: n0 + inputs: + - binding: + Value: + Promise: + var: val + var: val + metadata: + InterruptibleValue: null + name: square + retries: {} + outputs: + - binding: + Value: + Promise: + node_id: n0 + var: o0 + var: o0 diff --git a/pkg/compiler/test/testdata/snacks-core/121_core.flyte_basics.lp.my_wf_3.pb b/pkg/compiler/test/testdata/snacks-core/121_core.flyte_basics.lp.my_wf_3.pb new file mode 100644 index 0000000000..1e81e36542 Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/121_core.flyte_basics.lp.my_wf_3.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/122_default_4_lp_3.pb b/pkg/compiler/test/testdata/snacks-core/122_default_4_lp_3.pb new file mode 100644 index 0000000000..e6373d472f Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/122_default_4_lp_3.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/123_always_2_lp_3.pb b/pkg/compiler/test/testdata/snacks-core/123_always_2_lp_3.pb new file mode 100644 index 0000000000..890a019e72 Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/123_always_2_lp_3.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/124_core.flyte_basics.lp.greet_1.pb b/pkg/compiler/test/testdata/snacks-core/124_core.flyte_basics.lp.greet_1.pb new file mode 100644 index 0000000000..7b60ea4fdf Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/124_core.flyte_basics.lp.greet_1.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/124_core.flyte_basics.lp.greet_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/124_core.flyte_basics.lp.greet_1_task.yaml new file mode 100755 index 0000000000..d7cee10b4e --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/124_core.flyte_basics.lp.greet_1_task.yaml @@ -0,0 +1,60 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.lp + - task-name + - greet + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.flyte_basics.lp.greet + resource_type: 1 + interface: + inputs: + variables: + am: + description: am + type: + Type: + Simple: 4 + day_of_week: + description: day_of_week + type: + Type: + Simple: 3 + number: + description: number + type: + Type: + Simple: 1 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 3 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/125_core.flyte_basics.lp.go_greet_2.pb b/pkg/compiler/test/testdata/snacks-core/125_core.flyte_basics.lp.go_greet_2.pb new file mode 100644 index 0000000000..2873aa9e00 Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/125_core.flyte_basics.lp.go_greet_2.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/125_core.flyte_basics.lp.go_greet_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/125_core.flyte_basics.lp.go_greet_2_wf.yaml new file mode 100755 index 0000000000..39d18be425 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/125_core.flyte_basics.lp.go_greet_2_wf.yaml @@ -0,0 +1,67 @@ +workflow: + id: + name: core.flyte_basics.lp.go_greet + resource_type: 2 + interface: + inputs: + variables: + am: + description: am + type: + Type: + Simple: 4 + day_of_week: + description: day_of_week + type: + Type: + Simple: 3 + number: + description: number + type: + Type: + Simple: 1 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 3 + metadata: {} + metadata_defaults: {} + nodes: + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.flyte_basics.lp.greet + resource_type: 1 + overrides: {} + id: n0 + inputs: + - binding: + Value: + Promise: + var: am + var: am + - binding: + Value: + Promise: + var: day_of_week + var: day_of_week + - binding: + Value: + Promise: + var: number + var: number + metadata: + InterruptibleValue: null + name: greet + retries: {} + outputs: + - binding: + Value: + Promise: + node_id: n0 + var: o0 + var: o0 diff --git a/pkg/compiler/test/testdata/snacks-core/126_core.flyte_basics.lp.go_greet_3.pb b/pkg/compiler/test/testdata/snacks-core/126_core.flyte_basics.lp.go_greet_3.pb new file mode 100644 index 0000000000..ad58b87dd7 Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/126_core.flyte_basics.lp.go_greet_3.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/127_morning_greeting_3.pb b/pkg/compiler/test/testdata/snacks-core/127_morning_greeting_3.pb new file mode 100644 index 0000000000..e422fb8534 Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/127_morning_greeting_3.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/128_core.flyte_basics.named_outputs.say_hello_1.pb b/pkg/compiler/test/testdata/snacks-core/128_core.flyte_basics.named_outputs.say_hello_1.pb new file mode 100644 index 0000000000..8232bfadab Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/128_core.flyte_basics.named_outputs.say_hello_1.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/128_core.flyte_basics.named_outputs.say_hello_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/128_core.flyte_basics.named_outputs.say_hello_1_task.yaml new file mode 100755 index 0000000000..c7c4dffa65 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/128_core.flyte_basics.named_outputs.say_hello_1_task.yaml @@ -0,0 +1,44 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.named_outputs + - task-name + - say_hello + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.flyte_basics.named_outputs.say_hello + resource_type: 1 + interface: + inputs: {} + outputs: + variables: + greet: + description: greet + type: + Type: + Simple: 3 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/129_core.flyte_basics.named_outputs.my_wf_2.pb b/pkg/compiler/test/testdata/snacks-core/129_core.flyte_basics.named_outputs.my_wf_2.pb new file mode 100644 index 0000000000..26b064ef88 Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/129_core.flyte_basics.named_outputs.my_wf_2.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/129_core.flyte_basics.named_outputs.my_wf_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/129_core.flyte_basics.named_outputs.my_wf_2_wf.yaml new file mode 100755 index 0000000000..1aead9b02f --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/129_core.flyte_basics.named_outputs.my_wf_2_wf.yaml @@ -0,0 +1,58 @@ +workflow: + id: + name: core.flyte_basics.named_outputs.my_wf + resource_type: 2 + interface: + inputs: {} + outputs: + variables: + greet1: + description: greet1 + type: + Type: + Simple: 3 + greet2: + description: greet2 + type: + Type: + Simple: 3 + metadata: {} + metadata_defaults: {} + nodes: + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.flyte_basics.named_outputs.say_hello + resource_type: 1 + overrides: {} + id: n0 + metadata: + InterruptibleValue: null + name: say_hello + retries: {} + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.flyte_basics.named_outputs.say_hello + resource_type: 1 + overrides: {} + id: n1 + metadata: + InterruptibleValue: null + name: say_hello + retries: {} + outputs: + - binding: + Value: + Promise: + node_id: n0 + var: greet + var: greet1 + - binding: + Value: + Promise: + node_id: n1 + var: greet + var: greet2 diff --git a/pkg/compiler/test/testdata/snacks-core/130_core.flyte_basics.named_outputs.my_wf_3.pb b/pkg/compiler/test/testdata/snacks-core/130_core.flyte_basics.named_outputs.my_wf_3.pb new file mode 100644 index 0000000000..fd6c964228 Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/130_core.flyte_basics.named_outputs.my_wf_3.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/133__bash.task_1_1.pb b/pkg/compiler/test/testdata/snacks-core/133__bash.task_1_1.pb new file mode 100644 index 0000000000..c740572bdd Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/133__bash.task_1_1.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/133__bash.task_1_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/133__bash.task_1_1_task.yaml new file mode 100755 index 0000000000..95b1fc07fd --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/133__bash.task_1_1_task.yaml @@ -0,0 +1,38 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - flytekit.extras.tasks.shell + - task-name + - _dummy_task_func + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: _bash.task_1 + resource_type: 1 + interface: + inputs: {} + outputs: {} + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/134_task_1_1.pb b/pkg/compiler/test/testdata/snacks-core/134_task_1_1.pb new file mode 100644 index 0000000000..911d2e9d2e Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/134_task_1_1.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/134_task_1_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/134_task_1_1_task.yaml new file mode 100755 index 0000000000..d069629a1e --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/134_task_1_1_task.yaml @@ -0,0 +1,50 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.shell_task + - task-name + - t1 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: task_1 + resource_type: 1 + interface: + inputs: + variables: + x: + description: x + type: + Type: + Blob: {} + outputs: + variables: + i: + description: i + type: + Type: + Blob: {} + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/135__bash.task_2_1.pb b/pkg/compiler/test/testdata/snacks-core/135__bash.task_2_1.pb new file mode 100644 index 0000000000..2e6140daeb Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/135__bash.task_2_1.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/135__bash.task_2_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/135__bash.task_2_1_task.yaml new file mode 100755 index 0000000000..2cf32a6742 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/135__bash.task_2_1_task.yaml @@ -0,0 +1,38 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - flytekit.extras.tasks.shell + - task-name + - _dummy_task_func + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: _bash.task_2 + resource_type: 1 + interface: + inputs: {} + outputs: {} + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/136_task_2_1.pb b/pkg/compiler/test/testdata/snacks-core/136_task_2_1.pb new file mode 100644 index 0000000000..27e63742ab Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/136_task_2_1.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/136_task_2_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/136_task_2_1_task.yaml new file mode 100755 index 0000000000..056cbbd355 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/136_task_2_1_task.yaml @@ -0,0 +1,56 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.shell_task + - task-name + - t2 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: task_2 + resource_type: 1 + interface: + inputs: + variables: + x: + description: x + type: + Type: + Blob: {} + "y": + description: "y" + type: + Type: + Blob: + dimensionality: 1 + outputs: + variables: + j: + description: j + type: + Type: + Blob: {} + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/137__bash.task_3_1.pb b/pkg/compiler/test/testdata/snacks-core/137__bash.task_3_1.pb new file mode 100644 index 0000000000..a16af86ba3 Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/137__bash.task_3_1.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/137__bash.task_3_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/137__bash.task_3_1_task.yaml new file mode 100755 index 0000000000..15e529af4c --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/137__bash.task_3_1_task.yaml @@ -0,0 +1,38 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - flytekit.extras.tasks.shell + - task-name + - _dummy_task_func + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: _bash.task_3 + resource_type: 1 + interface: + inputs: {} + outputs: {} + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/138_task_3_1.pb b/pkg/compiler/test/testdata/snacks-core/138_task_3_1.pb new file mode 100644 index 0000000000..a5670868a5 Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/138_task_3_1.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/138_task_3_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/138_task_3_1_task.yaml new file mode 100755 index 0000000000..5172896164 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/138_task_3_1_task.yaml @@ -0,0 +1,61 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.shell_task + - task-name + - t3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: task_3 + resource_type: 1 + interface: + inputs: + variables: + x: + description: x + type: + Type: + Blob: {} + "y": + description: "y" + type: + Type: + Blob: + dimensionality: 1 + z: + description: z + type: + Type: + Blob: {} + outputs: + variables: + k: + description: k + type: + Type: + Blob: {} + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/139_core.flyte_basics.shell_task.create_entities_1.pb b/pkg/compiler/test/testdata/snacks-core/139_core.flyte_basics.shell_task.create_entities_1.pb new file mode 100644 index 0000000000..ac1af36c26 Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/139_core.flyte_basics.shell_task.create_entities_1.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/139_core.flyte_basics.shell_task.create_entities_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/139_core.flyte_basics.shell_task.create_entities_1_task.yaml new file mode 100755 index 0000000000..eac9137e24 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/139_core.flyte_basics.shell_task.create_entities_1_task.yaml @@ -0,0 +1,50 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.shell_task + - task-name + - create_entities + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.flyte_basics.shell_task.create_entities + resource_type: 1 + interface: + inputs: {} + outputs: + variables: + o0: + description: o0 + type: + Type: + Blob: {} + o1: + description: o1 + type: + Type: + Blob: + dimensionality: 1 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/140_core.flyte_basics.shell_task.wf_2.pb b/pkg/compiler/test/testdata/snacks-core/140_core.flyte_basics.shell_task.wf_2.pb new file mode 100644 index 0000000000..271b0a6413 Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/140_core.flyte_basics.shell_task.wf_2.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/140_core.flyte_basics.shell_task.wf_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/140_core.flyte_basics.shell_task.wf_2_wf.yaml new file mode 100755 index 0000000000..877de0530b --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/140_core.flyte_basics.shell_task.wf_2_wf.yaml @@ -0,0 +1,118 @@ +workflow: + id: + name: core.flyte_basics.shell_task.wf + resource_type: 2 + interface: + inputs: {} + outputs: + variables: + o0: + description: o0 + type: + Type: + Blob: {} + metadata: {} + metadata_defaults: {} + nodes: + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.flyte_basics.shell_task.create_entities + resource_type: 1 + overrides: {} + id: n0 + metadata: + InterruptibleValue: null + name: create_entities + retries: {} + - Target: + TaskNode: + Reference: + ReferenceId: + name: task_1 + resource_type: 1 + overrides: {} + id: n1 + inputs: + - binding: + Value: + Promise: + node_id: n0 + var: o0 + var: x + metadata: + InterruptibleValue: null + name: task_1 + retries: {} + upstream_node_ids: + - n0 + - Target: + TaskNode: + Reference: + ReferenceId: + name: task_2 + resource_type: 1 + overrides: {} + id: n2 + inputs: + - binding: + Value: + Promise: + node_id: n1 + var: i + var: x + - binding: + Value: + Promise: + node_id: n0 + var: o1 + var: "y" + metadata: + InterruptibleValue: null + name: task_2 + retries: {} + upstream_node_ids: + - n1 + - n0 + - Target: + TaskNode: + Reference: + ReferenceId: + name: task_3 + resource_type: 1 + overrides: {} + id: n3 + inputs: + - binding: + Value: + Promise: + node_id: n0 + var: o0 + var: x + - binding: + Value: + Promise: + node_id: n0 + var: o1 + var: "y" + - binding: + Value: + Promise: + node_id: n2 + var: j + var: z + metadata: + InterruptibleValue: null + name: task_3 + retries: {} + upstream_node_ids: + - n2 + - n0 + outputs: + - binding: + Value: + Promise: + node_id: n3 + var: k + var: o0 diff --git a/pkg/compiler/test/testdata/snacks-core/141_core.flyte_basics.shell_task.wf_3.pb b/pkg/compiler/test/testdata/snacks-core/141_core.flyte_basics.shell_task.wf_3.pb new file mode 100644 index 0000000000..86fe1d46d9 Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/141_core.flyte_basics.shell_task.wf_3.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/142_core.flyte_basics.task.square_1.pb b/pkg/compiler/test/testdata/snacks-core/142_core.flyte_basics.task.square_1.pb new file mode 100644 index 0000000000..591bb9c7bf Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/142_core.flyte_basics.task.square_1.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/142_core.flyte_basics.task.square_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/142_core.flyte_basics.task.square_1_task.yaml new file mode 100755 index 0000000000..19184615f6 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/142_core.flyte_basics.task.square_1_task.yaml @@ -0,0 +1,50 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.task + - task-name + - square + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.flyte_basics.task.square + resource_type: 1 + interface: + inputs: + variables: + "n": + description: "n" + type: + Type: + Simple: 1 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 1 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/143_core.flyte_basics.task_cache.square_1.pb b/pkg/compiler/test/testdata/snacks-core/143_core.flyte_basics.task_cache.square_1.pb new file mode 100644 index 0000000000..ff14f7f1ec Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/143_core.flyte_basics.task_cache.square_1.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/143_core.flyte_basics.task_cache.square_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/143_core.flyte_basics.task_cache.square_1_task.yaml new file mode 100755 index 0000000000..898414991a --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/143_core.flyte_basics.task_cache.square_1_task.yaml @@ -0,0 +1,52 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.task_cache + - task-name + - square + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.flyte_basics.task_cache.square + resource_type: 1 + interface: + inputs: + variables: + "n": + description: "n" + type: + Type: + Simple: 1 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 1 + metadata: + InterruptibleValue: null + discoverable: true + discovery_version: "1.0" + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/144_core.flyte_basics.task_cache.uncached_data_reading_task_1.pb b/pkg/compiler/test/testdata/snacks-core/144_core.flyte_basics.task_cache.uncached_data_reading_task_1.pb new file mode 100644 index 0000000000..a79dcc5054 Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/144_core.flyte_basics.task_cache.uncached_data_reading_task_1.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/144_core.flyte_basics.task_cache.uncached_data_reading_task_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/144_core.flyte_basics.task_cache.uncached_data_reading_task_1_task.yaml new file mode 100755 index 0000000000..f4ba5b67fc --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/144_core.flyte_basics.task_cache.uncached_data_reading_task_1_task.yaml @@ -0,0 +1,45 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.task_cache + - task-name + - uncached_data_reading_task + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.flyte_basics.task_cache.uncached_data_reading_task + resource_type: 1 + interface: + inputs: {} + outputs: + variables: + o0: + description: o0 + type: + Type: + StructuredDatasetType: + format: parquet + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/145_core.flyte_basics.task_cache.cached_data_processing_task_1.pb b/pkg/compiler/test/testdata/snacks-core/145_core.flyte_basics.task_cache.cached_data_processing_task_1.pb new file mode 100644 index 0000000000..81e6594462 Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/145_core.flyte_basics.task_cache.cached_data_processing_task_1.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/145_core.flyte_basics.task_cache.cached_data_processing_task_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/145_core.flyte_basics.task_cache.cached_data_processing_task_1_task.yaml new file mode 100755 index 0000000000..7c110a1322 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/145_core.flyte_basics.task_cache.cached_data_processing_task_1_task.yaml @@ -0,0 +1,54 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.task_cache + - task-name + - cached_data_processing_task + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.flyte_basics.task_cache.cached_data_processing_task + resource_type: 1 + interface: + inputs: + variables: + df: + description: df + type: + Type: + StructuredDatasetType: + format: parquet + outputs: + variables: + o0: + description: o0 + type: + Type: + StructuredDatasetType: + format: parquet + metadata: + InterruptibleValue: null + discoverable: true + discovery_version: "1.0" + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/146_core.flyte_basics.task_cache.compare_dataframes_1.pb b/pkg/compiler/test/testdata/snacks-core/146_core.flyte_basics.task_cache.compare_dataframes_1.pb new file mode 100644 index 0000000000..5f6331e6a9 Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/146_core.flyte_basics.task_cache.compare_dataframes_1.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/146_core.flyte_basics.task_cache.compare_dataframes_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/146_core.flyte_basics.task_cache.compare_dataframes_1_task.yaml new file mode 100755 index 0000000000..eb9378d853 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/146_core.flyte_basics.task_cache.compare_dataframes_1_task.yaml @@ -0,0 +1,51 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.task_cache + - task-name + - compare_dataframes + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.flyte_basics.task_cache.compare_dataframes + resource_type: 1 + interface: + inputs: + variables: + df1: + description: df1 + type: + Type: + StructuredDatasetType: + format: parquet + df2: + description: df2 + type: + Type: + StructuredDatasetType: + format: parquet + outputs: {} + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/147_core.flyte_basics.task_cache.cached_dataframe_wf_2.pb b/pkg/compiler/test/testdata/snacks-core/147_core.flyte_basics.task_cache.cached_dataframe_wf_2.pb new file mode 100644 index 0000000000..9ea68d1b8b Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/147_core.flyte_basics.task_cache.cached_dataframe_wf_2.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/147_core.flyte_basics.task_cache.cached_dataframe_wf_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/147_core.flyte_basics.task_cache.cached_dataframe_wf_2_wf.yaml new file mode 100755 index 0000000000..a4a0937202 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/147_core.flyte_basics.task_cache.cached_dataframe_wf_2_wf.yaml @@ -0,0 +1,93 @@ +workflow: + id: + name: core.flyte_basics.task_cache.cached_dataframe_wf + resource_type: 2 + interface: + inputs: {} + outputs: {} + metadata: {} + metadata_defaults: {} + nodes: + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.flyte_basics.task_cache.uncached_data_reading_task + resource_type: 1 + overrides: {} + id: n0 + metadata: + InterruptibleValue: null + name: uncached_data_reading_task + retries: {} + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.flyte_basics.task_cache.cached_data_processing_task + resource_type: 1 + overrides: {} + id: n1 + inputs: + - binding: + Value: + Promise: + node_id: n0 + var: o0 + var: df + metadata: + InterruptibleValue: null + name: cached_data_processing_task + retries: {} + upstream_node_ids: + - n0 + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.flyte_basics.task_cache.cached_data_processing_task + resource_type: 1 + overrides: {} + id: n2 + inputs: + - binding: + Value: + Promise: + node_id: n0 + var: o0 + var: df + metadata: + InterruptibleValue: null + name: cached_data_processing_task + retries: {} + upstream_node_ids: + - n0 + - n1 + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.flyte_basics.task_cache.compare_dataframes + resource_type: 1 + overrides: {} + id: n3 + inputs: + - binding: + Value: + Promise: + node_id: n1 + var: o0 + var: df1 + - binding: + Value: + Promise: + node_id: n2 + var: o0 + var: df2 + metadata: + InterruptibleValue: null + name: compare_dataframes + retries: {} + upstream_node_ids: + - n2 + - n1 diff --git a/pkg/compiler/test/testdata/snacks-core/148_core.flyte_basics.task_cache.cached_dataframe_wf_3.pb b/pkg/compiler/test/testdata/snacks-core/148_core.flyte_basics.task_cache.cached_dataframe_wf_3.pb new file mode 100644 index 0000000000..ef58f942b3 Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/148_core.flyte_basics.task_cache.cached_dataframe_wf_3.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/149_core.flyte_basics.task_cache_serialize.square_1.pb b/pkg/compiler/test/testdata/snacks-core/149_core.flyte_basics.task_cache_serialize.square_1.pb new file mode 100644 index 0000000000..8e228a87cf Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/149_core.flyte_basics.task_cache_serialize.square_1.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/149_core.flyte_basics.task_cache_serialize.square_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/149_core.flyte_basics.task_cache_serialize.square_1_task.yaml new file mode 100755 index 0000000000..247184cc1e --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/149_core.flyte_basics.task_cache_serialize.square_1_task.yaml @@ -0,0 +1,53 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.task_cache_serialize + - task-name + - square + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.flyte_basics.task_cache_serialize.square + resource_type: 1 + interface: + inputs: + variables: + "n": + description: "n" + type: + Type: + Simple: 1 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 1 + metadata: + InterruptibleValue: null + cache_serializable: true + discoverable: true + discovery_version: "1.0" + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/150_core.scheduled_workflows.lp_schedules.format_date_1.pb b/pkg/compiler/test/testdata/snacks-core/150_core.scheduled_workflows.lp_schedules.format_date_1.pb new file mode 100644 index 0000000000..9caac3d38e Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/150_core.scheduled_workflows.lp_schedules.format_date_1.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/150_core.scheduled_workflows.lp_schedules.format_date_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/150_core.scheduled_workflows.lp_schedules.format_date_1_task.yaml new file mode 100755 index 0000000000..b381aae673 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/150_core.scheduled_workflows.lp_schedules.format_date_1_task.yaml @@ -0,0 +1,50 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.scheduled_workflows.lp_schedules + - task-name + - format_date + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.scheduled_workflows.lp_schedules.format_date + resource_type: 1 + interface: + inputs: + variables: + run_date: + description: run_date + type: + Type: + Simple: 5 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 3 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/151_core.scheduled_workflows.lp_schedules.date_formatter_wf_2.pb b/pkg/compiler/test/testdata/snacks-core/151_core.scheduled_workflows.lp_schedules.date_formatter_wf_2.pb new file mode 100644 index 0000000000..3c3f9de1b4 Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/151_core.scheduled_workflows.lp_schedules.date_formatter_wf_2.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/151_core.scheduled_workflows.lp_schedules.date_formatter_wf_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/151_core.scheduled_workflows.lp_schedules.date_formatter_wf_2_wf.yaml new file mode 100755 index 0000000000..0077991ced --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/151_core.scheduled_workflows.lp_schedules.date_formatter_wf_2_wf.yaml @@ -0,0 +1,34 @@ +workflow: + id: + name: core.scheduled_workflows.lp_schedules.date_formatter_wf + resource_type: 2 + interface: + inputs: + variables: + kickoff_time: + description: kickoff_time + type: + Type: + Simple: 5 + outputs: {} + metadata: {} + metadata_defaults: {} + nodes: + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.scheduled_workflows.lp_schedules.format_date + resource_type: 1 + overrides: {} + id: n0 + inputs: + - binding: + Value: + Promise: + var: kickoff_time + var: run_date + metadata: + InterruptibleValue: null + name: format_date + retries: {} diff --git a/pkg/compiler/test/testdata/snacks-core/152_core.scheduled_workflows.lp_schedules.date_formatter_wf_3.pb b/pkg/compiler/test/testdata/snacks-core/152_core.scheduled_workflows.lp_schedules.date_formatter_wf_3.pb new file mode 100644 index 0000000000..5b21a394f2 Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/152_core.scheduled_workflows.lp_schedules.date_formatter_wf_3.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/153_my_cron_scheduled_lp_3.pb b/pkg/compiler/test/testdata/snacks-core/153_my_cron_scheduled_lp_3.pb new file mode 100644 index 0000000000..f146a877d8 Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/153_my_cron_scheduled_lp_3.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/154_core.scheduled_workflows.lp_schedules.be_positive_1.pb b/pkg/compiler/test/testdata/snacks-core/154_core.scheduled_workflows.lp_schedules.be_positive_1.pb new file mode 100644 index 0000000000..084789b0e3 Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/154_core.scheduled_workflows.lp_schedules.be_positive_1.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/154_core.scheduled_workflows.lp_schedules.be_positive_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/154_core.scheduled_workflows.lp_schedules.be_positive_1_task.yaml new file mode 100755 index 0000000000..d7ca0c8950 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/154_core.scheduled_workflows.lp_schedules.be_positive_1_task.yaml @@ -0,0 +1,50 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.scheduled_workflows.lp_schedules + - task-name + - be_positive + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.scheduled_workflows.lp_schedules.be_positive + resource_type: 1 + interface: + inputs: + variables: + name: + description: name + type: + Type: + Simple: 3 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 3 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/155_core.scheduled_workflows.lp_schedules.positive_wf_2.pb b/pkg/compiler/test/testdata/snacks-core/155_core.scheduled_workflows.lp_schedules.positive_wf_2.pb new file mode 100644 index 0000000000..c4f0963bb1 Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/155_core.scheduled_workflows.lp_schedules.positive_wf_2.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/155_core.scheduled_workflows.lp_schedules.positive_wf_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/155_core.scheduled_workflows.lp_schedules.positive_wf_2_wf.yaml new file mode 100755 index 0000000000..a21c9e7e96 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/155_core.scheduled_workflows.lp_schedules.positive_wf_2_wf.yaml @@ -0,0 +1,34 @@ +workflow: + id: + name: core.scheduled_workflows.lp_schedules.positive_wf + resource_type: 2 + interface: + inputs: + variables: + name: + description: name + type: + Type: + Simple: 3 + outputs: {} + metadata: {} + metadata_defaults: {} + nodes: + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.scheduled_workflows.lp_schedules.be_positive + resource_type: 1 + overrides: {} + id: n0 + inputs: + - binding: + Value: + Promise: + var: name + var: name + metadata: + InterruptibleValue: null + name: be_positive + retries: {} diff --git a/pkg/compiler/test/testdata/snacks-core/156_core.scheduled_workflows.lp_schedules.positive_wf_3.pb b/pkg/compiler/test/testdata/snacks-core/156_core.scheduled_workflows.lp_schedules.positive_wf_3.pb new file mode 100644 index 0000000000..c679d7d33b Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/156_core.scheduled_workflows.lp_schedules.positive_wf_3.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/157_my_fixed_rate_lp_3.pb b/pkg/compiler/test/testdata/snacks-core/157_my_fixed_rate_lp_3.pb new file mode 100644 index 0000000000..febcd3ed65 Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/157_my_fixed_rate_lp_3.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/158_core.type_system.custom_objects.stringify_1.pb b/pkg/compiler/test/testdata/snacks-core/158_core.type_system.custom_objects.stringify_1.pb new file mode 100644 index 0000000000..f2951f5f8e Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/158_core.type_system.custom_objects.stringify_1.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/158_core.type_system.custom_objects.stringify_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/158_core.type_system.custom_objects.stringify_1_task.yaml new file mode 100755 index 0000000000..cd0a6af180 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/158_core.type_system.custom_objects.stringify_1_task.yaml @@ -0,0 +1,70 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.type_system.custom_objects + - task-name + - stringify + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.type_system.custom_objects.stringify + resource_type: 1 + interface: + inputs: + variables: + x: + description: x + type: + Type: + Simple: 1 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 9 + metadata: + $ref: '#/definitions/DatumSchema' + $schema: http://json-schema.org/draft-07/schema# + definitions: + DatumSchema: + additionalProperties: false + properties: + x: + title: x + type: integer + "y": + title: "y" + type: string + z: + additionalProperties: + title: z + type: string + title: z + type: object + type: object + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/159_core.type_system.custom_objects.add_1.pb b/pkg/compiler/test/testdata/snacks-core/159_core.type_system.custom_objects.add_1.pb new file mode 100644 index 0000000000..989661dbdb Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/159_core.type_system.custom_objects.add_1.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/159_core.type_system.custom_objects.add_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/159_core.type_system.custom_objects.add_1_task.yaml new file mode 100755 index 0000000000..56614c1955 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/159_core.type_system.custom_objects.add_1_task.yaml @@ -0,0 +1,115 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.type_system.custom_objects + - task-name + - add + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.type_system.custom_objects.add + resource_type: 1 + interface: + inputs: + variables: + x: + description: x + type: + Type: + Simple: 9 + metadata: + $ref: '#/definitions/DatumSchema' + $schema: http://json-schema.org/draft-07/schema# + definitions: + DatumSchema: + additionalProperties: false + properties: + x: + title: x + type: integer + "y": + title: "y" + type: string + z: + additionalProperties: + title: z + type: string + title: z + type: object + type: object + "y": + description: "y" + type: + Type: + Simple: 9 + metadata: + $ref: '#/definitions/DatumSchema' + $schema: http://json-schema.org/draft-07/schema# + definitions: + DatumSchema: + additionalProperties: false + properties: + x: + title: x + type: integer + "y": + title: "y" + type: string + z: + additionalProperties: + title: z + type: string + title: z + type: object + type: object + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 9 + metadata: + $ref: '#/definitions/DatumSchema' + $schema: http://json-schema.org/draft-07/schema# + definitions: + DatumSchema: + additionalProperties: false + properties: + x: + title: x + type: integer + "y": + title: "y" + type: string + z: + additionalProperties: + title: z + type: string + title: z + type: object + type: object + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/160_core.type_system.custom_objects.upload_result_1.pb b/pkg/compiler/test/testdata/snacks-core/160_core.type_system.custom_objects.upload_result_1.pb new file mode 100644 index 0000000000..200004414a Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/160_core.type_system.custom_objects.upload_result_1.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/160_core.type_system.custom_objects.upload_result_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/160_core.type_system.custom_objects.upload_result_1_task.yaml new file mode 100755 index 0000000000..85f50ef26b --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/160_core.type_system.custom_objects.upload_result_1_task.yaml @@ -0,0 +1,85 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.type_system.custom_objects + - task-name + - upload_result + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.type_system.custom_objects.upload_result + resource_type: 1 + interface: + inputs: {} + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 9 + metadata: + $ref: '#/definitions/ResultSchema' + $schema: http://json-schema.org/draft-07/schema# + definitions: + FlytedirectorySchema: + additionalProperties: false + properties: + path: + title: path + type: string + type: object + FlytefileSchema: + additionalProperties: false + properties: + path: + title: path + type: string + type: object + FlyteschemaSchema: + additionalProperties: false + properties: + remote_path: + title: remote_path + type: string + type: object + ResultSchema: + additionalProperties: false + properties: + directory: + $ref: '#/definitions/FlytedirectorySchema' + field_many: false + type: object + file: + $ref: '#/definitions/FlytefileSchema' + field_many: false + type: object + schema: + $ref: '#/definitions/FlyteschemaSchema' + field_many: false + type: object + type: object + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/161_core.type_system.custom_objects.download_result_1.pb b/pkg/compiler/test/testdata/snacks-core/161_core.type_system.custom_objects.download_result_1.pb new file mode 100644 index 0000000000..a80bb1fcd7 Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/161_core.type_system.custom_objects.download_result_1.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/161_core.type_system.custom_objects.download_result_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/161_core.type_system.custom_objects.download_result_1_task.yaml new file mode 100755 index 0000000000..a5bc7415f3 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/161_core.type_system.custom_objects.download_result_1_task.yaml @@ -0,0 +1,85 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.type_system.custom_objects + - task-name + - download_result + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.type_system.custom_objects.download_result + resource_type: 1 + interface: + inputs: + variables: + res: + description: res + type: + Type: + Simple: 9 + metadata: + $ref: '#/definitions/ResultSchema' + $schema: http://json-schema.org/draft-07/schema# + definitions: + FlytedirectorySchema: + additionalProperties: false + properties: + path: + title: path + type: string + type: object + FlytefileSchema: + additionalProperties: false + properties: + path: + title: path + type: string + type: object + FlyteschemaSchema: + additionalProperties: false + properties: + remote_path: + title: remote_path + type: string + type: object + ResultSchema: + additionalProperties: false + properties: + directory: + $ref: '#/definitions/FlytedirectorySchema' + field_many: false + type: object + file: + $ref: '#/definitions/FlytefileSchema' + field_many: false + type: object + schema: + $ref: '#/definitions/FlyteschemaSchema' + field_many: false + type: object + type: object + outputs: {} + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/162_core.type_system.custom_objects.wf_2.pb b/pkg/compiler/test/testdata/snacks-core/162_core.type_system.custom_objects.wf_2.pb new file mode 100644 index 0000000000..caeaa5bc4b Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/162_core.type_system.custom_objects.wf_2.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/162_core.type_system.custom_objects.wf_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/162_core.type_system.custom_objects.wf_2_wf.yaml new file mode 100755 index 0000000000..7df4424f27 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/162_core.type_system.custom_objects.wf_2_wf.yaml @@ -0,0 +1,203 @@ +workflow: + id: + name: core.type_system.custom_objects.wf + resource_type: 2 + interface: + inputs: + variables: + x: + description: x + type: + Type: + Simple: 1 + "y": + description: "y" + type: + Type: + Simple: 1 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 9 + metadata: + $ref: '#/definitions/DatumSchema' + $schema: http://json-schema.org/draft-07/schema# + definitions: + DatumSchema: + additionalProperties: false + properties: + x: + title: x + type: integer + "y": + title: "y" + type: string + z: + additionalProperties: + title: z + type: string + title: z + type: object + type: object + o1: + description: o1 + type: + Type: + Simple: 9 + metadata: + $ref: '#/definitions/ResultSchema' + $schema: http://json-schema.org/draft-07/schema# + definitions: + FlytedirectorySchema: + additionalProperties: false + properties: + path: + title: path + type: string + type: object + FlytefileSchema: + additionalProperties: false + properties: + path: + title: path + type: string + type: object + FlyteschemaSchema: + additionalProperties: false + properties: + remote_path: + title: remote_path + type: string + type: object + ResultSchema: + additionalProperties: false + properties: + directory: + $ref: '#/definitions/FlytedirectorySchema' + field_many: false + type: object + file: + $ref: '#/definitions/FlytefileSchema' + field_many: false + type: object + schema: + $ref: '#/definitions/FlyteschemaSchema' + field_many: false + type: object + type: object + metadata: {} + metadata_defaults: {} + nodes: + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.type_system.custom_objects.upload_result + resource_type: 1 + overrides: {} + id: n0 + metadata: + InterruptibleValue: null + name: upload_result + retries: {} + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.type_system.custom_objects.download_result + resource_type: 1 + overrides: {} + id: n1 + inputs: + - binding: + Value: + Promise: + node_id: n0 + var: o0 + var: res + metadata: + InterruptibleValue: null + name: download_result + retries: {} + upstream_node_ids: + - n0 + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.type_system.custom_objects.stringify + resource_type: 1 + overrides: {} + id: n2 + inputs: + - binding: + Value: + Promise: + var: x + var: x + metadata: + InterruptibleValue: null + name: stringify + retries: {} + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.type_system.custom_objects.stringify + resource_type: 1 + overrides: {} + id: n3 + inputs: + - binding: + Value: + Promise: + var: "y" + var: x + metadata: + InterruptibleValue: null + name: stringify + retries: {} + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.type_system.custom_objects.add + resource_type: 1 + overrides: {} + id: n4 + inputs: + - binding: + Value: + Promise: + node_id: n2 + var: o0 + var: x + - binding: + Value: + Promise: + node_id: n3 + var: o0 + var: "y" + metadata: + InterruptibleValue: null + name: add + retries: {} + upstream_node_ids: + - n2 + - n3 + outputs: + - binding: + Value: + Promise: + node_id: n4 + var: o0 + var: o0 + - binding: + Value: + Promise: + node_id: n0 + var: o0 + var: o1 diff --git a/pkg/compiler/test/testdata/snacks-core/163_core.type_system.custom_objects.wf_3.pb b/pkg/compiler/test/testdata/snacks-core/163_core.type_system.custom_objects.wf_3.pb new file mode 100644 index 0000000000..fe133c08cd Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/163_core.type_system.custom_objects.wf_3.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/164_core.type_system.enums.enum_stringify_1.pb b/pkg/compiler/test/testdata/snacks-core/164_core.type_system.enums.enum_stringify_1.pb new file mode 100644 index 0000000000..05764a25a9 Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/164_core.type_system.enums.enum_stringify_1.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/164_core.type_system.enums.enum_stringify_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/164_core.type_system.enums.enum_stringify_1_task.yaml new file mode 100755 index 0000000000..fb59b6a035 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/164_core.type_system.enums.enum_stringify_1_task.yaml @@ -0,0 +1,54 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.type_system.enums + - task-name + - enum_stringify + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.type_system.enums.enum_stringify + resource_type: 1 + interface: + inputs: + variables: + c: + description: c + type: + Type: + EnumType: + values: + - red + - green + - blue + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 3 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/165_core.type_system.enums.string_to_enum_1.pb b/pkg/compiler/test/testdata/snacks-core/165_core.type_system.enums.string_to_enum_1.pb new file mode 100644 index 0000000000..065f2fb5b9 Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/165_core.type_system.enums.string_to_enum_1.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/165_core.type_system.enums.string_to_enum_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/165_core.type_system.enums.string_to_enum_1_task.yaml new file mode 100755 index 0000000000..b3c3a41c62 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/165_core.type_system.enums.string_to_enum_1_task.yaml @@ -0,0 +1,54 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.type_system.enums + - task-name + - string_to_enum + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.type_system.enums.string_to_enum + resource_type: 1 + interface: + inputs: + variables: + c: + description: c + type: + Type: + Simple: 3 + outputs: + variables: + o0: + description: o0 + type: + Type: + EnumType: + values: + - red + - green + - blue + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/166_core.type_system.enums.enum_wf_2.pb b/pkg/compiler/test/testdata/snacks-core/166_core.type_system.enums.enum_wf_2.pb new file mode 100644 index 0000000000..c42171c455 Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/166_core.type_system.enums.enum_wf_2.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/166_core.type_system.enums.enum_wf_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/166_core.type_system.enums.enum_wf_2_wf.yaml new file mode 100755 index 0000000000..df1745603c --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/166_core.type_system.enums.enum_wf_2_wf.yaml @@ -0,0 +1,87 @@ +workflow: + id: + name: core.type_system.enums.enum_wf + resource_type: 2 + interface: + inputs: + variables: + c: + description: c + type: + Type: + EnumType: + values: + - red + - green + - blue + outputs: + variables: + o0: + description: o0 + type: + Type: + EnumType: + values: + - red + - green + - blue + o1: + description: o1 + type: + Type: + Simple: 3 + metadata: {} + metadata_defaults: {} + nodes: + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.type_system.enums.enum_stringify + resource_type: 1 + overrides: {} + id: n0 + inputs: + - binding: + Value: + Promise: + var: c + var: c + metadata: + InterruptibleValue: null + name: enum_stringify + retries: {} + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.type_system.enums.string_to_enum + resource_type: 1 + overrides: {} + id: n1 + inputs: + - binding: + Value: + Promise: + node_id: n0 + var: o0 + var: c + metadata: + InterruptibleValue: null + name: string_to_enum + retries: {} + upstream_node_ids: + - n0 + outputs: + - binding: + Value: + Promise: + node_id: n1 + var: o0 + var: o0 + - binding: + Value: + Promise: + node_id: n0 + var: o0 + var: o1 diff --git a/pkg/compiler/test/testdata/snacks-core/167_core.type_system.enums.enum_wf_3.pb b/pkg/compiler/test/testdata/snacks-core/167_core.type_system.enums.enum_wf_3.pb new file mode 100644 index 0000000000..18dcf27cef Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/167_core.type_system.enums.enum_wf_3.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/168_core.type_system.flyte_pickle.greet_1.pb b/pkg/compiler/test/testdata/snacks-core/168_core.type_system.flyte_pickle.greet_1.pb new file mode 100644 index 0000000000..ef3aeeb56a Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/168_core.type_system.flyte_pickle.greet_1.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/168_core.type_system.flyte_pickle.greet_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/168_core.type_system.flyte_pickle.greet_1_task.yaml new file mode 100755 index 0000000000..716ad92010 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/168_core.type_system.flyte_pickle.greet_1_task.yaml @@ -0,0 +1,53 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.type_system.flyte_pickle + - task-name + - greet + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.type_system.flyte_pickle.greet + resource_type: 1 + interface: + inputs: + variables: + name: + description: name + type: + Type: + Simple: 3 + outputs: + variables: + o0: + description: o0 + type: + Type: + Blob: + format: PythonPickle + metadata: + python_class_name: People + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/169_core.type_system.flyte_pickle.welcome_2.pb b/pkg/compiler/test/testdata/snacks-core/169_core.type_system.flyte_pickle.welcome_2.pb new file mode 100644 index 0000000000..9b0a2640e6 Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/169_core.type_system.flyte_pickle.welcome_2.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/169_core.type_system.flyte_pickle.welcome_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/169_core.type_system.flyte_pickle.welcome_2_wf.yaml new file mode 100755 index 0000000000..a08c6a8c46 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/169_core.type_system.flyte_pickle.welcome_2_wf.yaml @@ -0,0 +1,50 @@ +workflow: + id: + name: core.type_system.flyte_pickle.welcome + resource_type: 2 + interface: + inputs: + variables: + name: + description: name + type: + Type: + Simple: 3 + outputs: + variables: + o0: + description: o0 + type: + Type: + Blob: + format: PythonPickle + metadata: + python_class_name: People + metadata: {} + metadata_defaults: {} + nodes: + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.type_system.flyte_pickle.greet + resource_type: 1 + overrides: {} + id: n0 + inputs: + - binding: + Value: + Promise: + var: name + var: name + metadata: + InterruptibleValue: null + name: greet + retries: {} + outputs: + - binding: + Value: + Promise: + node_id: n0 + var: o0 + var: o0 diff --git a/pkg/compiler/test/testdata/snacks-core/170_core.type_system.flyte_pickle.welcome_3.pb b/pkg/compiler/test/testdata/snacks-core/170_core.type_system.flyte_pickle.welcome_3.pb new file mode 100644 index 0000000000..30d4a771ad Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/170_core.type_system.flyte_pickle.welcome_3.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/171_core.type_system.schema.get_df_1.pb b/pkg/compiler/test/testdata/snacks-core/171_core.type_system.schema.get_df_1.pb new file mode 100644 index 0000000000..ac63219948 Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/171_core.type_system.schema.get_df_1.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/171_core.type_system.schema.get_df_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/171_core.type_system.schema.get_df_1_task.yaml new file mode 100755 index 0000000000..141241395e --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/171_core.type_system.schema.get_df_1_task.yaml @@ -0,0 +1,51 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.type_system.schema + - task-name + - get_df + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.type_system.schema.get_df + resource_type: 1 + interface: + inputs: + variables: + a: + description: a + type: + Type: + Simple: 1 + outputs: + variables: + o0: + description: o0 + type: + Type: + StructuredDatasetType: + format: parquet + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/172_core.type_system.schema.add_df_1.pb b/pkg/compiler/test/testdata/snacks-core/172_core.type_system.schema.add_df_1.pb new file mode 100644 index 0000000000..f51e0d76e4 Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/172_core.type_system.schema.add_df_1.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/172_core.type_system.schema.add_df_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/172_core.type_system.schema.add_df_1_task.yaml new file mode 100755 index 0000000000..7d1938b4ad --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/172_core.type_system.schema.add_df_1_task.yaml @@ -0,0 +1,52 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.type_system.schema + - task-name + - add_df + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.type_system.schema.add_df + resource_type: 1 + interface: + inputs: + variables: + df: + description: df + type: + Type: + StructuredDatasetType: + format: parquet + outputs: + variables: + o0: + description: o0 + type: + Type: + StructuredDatasetType: + format: parquet + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/173_core.type_system.schema.df_wf_2.pb b/pkg/compiler/test/testdata/snacks-core/173_core.type_system.schema.df_wf_2.pb new file mode 100644 index 0000000000..2149d50afc Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/173_core.type_system.schema.df_wf_2.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/173_core.type_system.schema.df_wf_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/173_core.type_system.schema.df_wf_2_wf.yaml new file mode 100755 index 0000000000..95fa8cfc23 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/173_core.type_system.schema.df_wf_2_wf.yaml @@ -0,0 +1,69 @@ +workflow: + id: + name: core.type_system.schema.df_wf + resource_type: 2 + interface: + inputs: + variables: + a: + description: a + type: + Type: + Simple: 1 + outputs: + variables: + o0: + description: o0 + type: + Type: + StructuredDatasetType: + format: parquet + metadata: {} + metadata_defaults: {} + nodes: + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.type_system.schema.get_df + resource_type: 1 + overrides: {} + id: n0 + inputs: + - binding: + Value: + Promise: + var: a + var: a + metadata: + InterruptibleValue: null + name: get_df + retries: {} + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.type_system.schema.add_df + resource_type: 1 + overrides: {} + id: n1 + inputs: + - binding: + Value: + Promise: + node_id: n0 + var: o0 + var: df + metadata: + InterruptibleValue: null + name: add_df + retries: {} + upstream_node_ids: + - n0 + outputs: + - binding: + Value: + Promise: + node_id: n1 + var: o0 + var: o0 diff --git a/pkg/compiler/test/testdata/snacks-core/174_core.type_system.schema.df_wf_3.pb b/pkg/compiler/test/testdata/snacks-core/174_core.type_system.schema.df_wf_3.pb new file mode 100644 index 0000000000..adf02dbd81 Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/174_core.type_system.schema.df_wf_3.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/175_core.type_system.structured_dataset.get_df_1.pb b/pkg/compiler/test/testdata/snacks-core/175_core.type_system.structured_dataset.get_df_1.pb new file mode 100644 index 0000000000..928a6a330f Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/175_core.type_system.structured_dataset.get_df_1.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/175_core.type_system.structured_dataset.get_df_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/175_core.type_system.structured_dataset.get_df_1_task.yaml new file mode 100755 index 0000000000..f4187772be --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/175_core.type_system.structured_dataset.get_df_1_task.yaml @@ -0,0 +1,64 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.type_system.structured_dataset + - task-name + - get_df + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.type_system.structured_dataset.get_df + resource_type: 1 + interface: + inputs: + variables: + a: + description: a + type: + Type: + Simple: 1 + outputs: + variables: + o0: + description: o0 + type: + Type: + StructuredDatasetType: + columns: + - literal_type: + Type: + Simple: 3 + name: Name + - literal_type: + Type: + Simple: 1 + name: Age + - literal_type: + Type: + Simple: 1 + name: Height + format: parquet + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/176_core.type_system.structured_dataset.get_schema_df_1.pb b/pkg/compiler/test/testdata/snacks-core/176_core.type_system.structured_dataset.get_schema_df_1.pb new file mode 100644 index 0000000000..42aa4ccdb0 Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/176_core.type_system.structured_dataset.get_schema_df_1.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/176_core.type_system.structured_dataset.get_schema_df_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/176_core.type_system.structured_dataset.get_schema_df_1_task.yaml new file mode 100755 index 0000000000..8e52f683f9 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/176_core.type_system.structured_dataset.get_schema_df_1_task.yaml @@ -0,0 +1,55 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.type_system.structured_dataset + - task-name + - get_schema_df + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.type_system.structured_dataset.get_schema_df + resource_type: 1 + interface: + inputs: + variables: + a: + description: a + type: + Type: + Simple: 1 + outputs: + variables: + o0: + description: o0 + type: + Type: + Schema: + columns: + - name: Name + type: 2 + - name: Age + - name: Height + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/177_core.type_system.structured_dataset.get_subset_df_1.pb b/pkg/compiler/test/testdata/snacks-core/177_core.type_system.structured_dataset.get_subset_df_1.pb new file mode 100644 index 0000000000..e369b2b853 Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/177_core.type_system.structured_dataset.get_subset_df_1.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/177_core.type_system.structured_dataset.get_subset_df_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/177_core.type_system.structured_dataset.get_subset_df_1_task.yaml new file mode 100755 index 0000000000..6888780842 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/177_core.type_system.structured_dataset.get_subset_df_1_task.yaml @@ -0,0 +1,62 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.type_system.structured_dataset + - task-name + - get_subset_df + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.type_system.structured_dataset.get_subset_df + resource_type: 1 + interface: + inputs: + variables: + df: + description: df + type: + Type: + StructuredDatasetType: + columns: + - literal_type: + Type: + Simple: 1 + name: Age + format: parquet + outputs: + variables: + o0: + description: o0 + type: + Type: + StructuredDatasetType: + columns: + - literal_type: + Type: + Simple: 1 + name: Age + format: parquet + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/178_core.type_system.structured_dataset.to_numpy_1.pb b/pkg/compiler/test/testdata/snacks-core/178_core.type_system.structured_dataset.to_numpy_1.pb new file mode 100644 index 0000000000..68f5b15bb6 Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/178_core.type_system.structured_dataset.to_numpy_1.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/178_core.type_system.structured_dataset.to_numpy_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/178_core.type_system.structured_dataset.to_numpy_1_task.yaml new file mode 100755 index 0000000000..137da95164 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/178_core.type_system.structured_dataset.to_numpy_1_task.yaml @@ -0,0 +1,62 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.type_system.structured_dataset + - task-name + - to_numpy + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.type_system.structured_dataset.to_numpy + resource_type: 1 + interface: + inputs: + variables: + ds: + description: ds + type: + Type: + StructuredDatasetType: + columns: + - literal_type: + Type: + Simple: 1 + name: Age + format: parquet + outputs: + variables: + o0: + description: o0 + type: + Type: + StructuredDatasetType: + columns: + - literal_type: + Type: + Simple: 1 + name: Age + format: parquet + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/179_core.type_system.structured_dataset.pandas_compatibility_wf_2.pb b/pkg/compiler/test/testdata/snacks-core/179_core.type_system.structured_dataset.pandas_compatibility_wf_2.pb new file mode 100644 index 0000000000..5cf8eb061f Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/179_core.type_system.structured_dataset.pandas_compatibility_wf_2.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/179_core.type_system.structured_dataset.pandas_compatibility_wf_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/179_core.type_system.structured_dataset.pandas_compatibility_wf_2_wf.yaml new file mode 100755 index 0000000000..5006091f3b --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/179_core.type_system.structured_dataset.pandas_compatibility_wf_2_wf.yaml @@ -0,0 +1,95 @@ +workflow: + id: + name: core.type_system.structured_dataset.pandas_compatibility_wf + resource_type: 2 + interface: + inputs: + variables: + a: + description: a + type: + Type: + Simple: 1 + outputs: + variables: + o0: + description: o0 + type: + Type: + StructuredDatasetType: + columns: + - literal_type: + Type: + Simple: 1 + name: Age + format: parquet + metadata: {} + metadata_defaults: {} + nodes: + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.type_system.structured_dataset.get_df + resource_type: 1 + overrides: {} + id: n0 + inputs: + - binding: + Value: + Promise: + var: a + var: a + metadata: + InterruptibleValue: null + name: get_df + retries: {} + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.type_system.structured_dataset.get_subset_df + resource_type: 1 + overrides: {} + id: n1 + inputs: + - binding: + Value: + Promise: + node_id: n0 + var: o0 + var: df + metadata: + InterruptibleValue: null + name: get_subset_df + retries: {} + upstream_node_ids: + - n0 + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.type_system.structured_dataset.to_numpy + resource_type: 1 + overrides: {} + id: n2 + inputs: + - binding: + Value: + Promise: + node_id: n1 + var: o0 + var: ds + metadata: + InterruptibleValue: null + name: to_numpy + retries: {} + upstream_node_ids: + - n1 + outputs: + - binding: + Value: + Promise: + node_id: n2 + var: o0 + var: o0 diff --git a/pkg/compiler/test/testdata/snacks-core/180_core.type_system.structured_dataset.pandas_compatibility_wf_3.pb b/pkg/compiler/test/testdata/snacks-core/180_core.type_system.structured_dataset.pandas_compatibility_wf_3.pb new file mode 100644 index 0000000000..89af64ab0c Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/180_core.type_system.structured_dataset.pandas_compatibility_wf_3.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/181_core.type_system.structured_dataset.schema_compatibility_wf_2.pb b/pkg/compiler/test/testdata/snacks-core/181_core.type_system.structured_dataset.schema_compatibility_wf_2.pb new file mode 100644 index 0000000000..a0c3ca27f0 Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/181_core.type_system.structured_dataset.schema_compatibility_wf_2.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/181_core.type_system.structured_dataset.schema_compatibility_wf_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/181_core.type_system.structured_dataset.schema_compatibility_wf_2_wf.yaml new file mode 100755 index 0000000000..102fa2b16d --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/181_core.type_system.structured_dataset.schema_compatibility_wf_2_wf.yaml @@ -0,0 +1,95 @@ +workflow: + id: + name: core.type_system.structured_dataset.schema_compatibility_wf + resource_type: 2 + interface: + inputs: + variables: + a: + description: a + type: + Type: + Simple: 1 + outputs: + variables: + o0: + description: o0 + type: + Type: + StructuredDatasetType: + columns: + - literal_type: + Type: + Simple: 1 + name: Age + format: parquet + metadata: {} + metadata_defaults: {} + nodes: + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.type_system.structured_dataset.get_schema_df + resource_type: 1 + overrides: {} + id: n0 + inputs: + - binding: + Value: + Promise: + var: a + var: a + metadata: + InterruptibleValue: null + name: get_schema_df + retries: {} + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.type_system.structured_dataset.get_subset_df + resource_type: 1 + overrides: {} + id: n1 + inputs: + - binding: + Value: + Promise: + node_id: n0 + var: o0 + var: df + metadata: + InterruptibleValue: null + name: get_subset_df + retries: {} + upstream_node_ids: + - n0 + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.type_system.structured_dataset.to_numpy + resource_type: 1 + overrides: {} + id: n2 + inputs: + - binding: + Value: + Promise: + node_id: n1 + var: o0 + var: ds + metadata: + InterruptibleValue: null + name: to_numpy + retries: {} + upstream_node_ids: + - n1 + outputs: + - binding: + Value: + Promise: + node_id: n2 + var: o0 + var: o0 diff --git a/pkg/compiler/test/testdata/snacks-core/182_core.type_system.structured_dataset.schema_compatibility_wf_3.pb b/pkg/compiler/test/testdata/snacks-core/182_core.type_system.structured_dataset.schema_compatibility_wf_3.pb new file mode 100644 index 0000000000..95617566db Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/182_core.type_system.structured_dataset.schema_compatibility_wf_3.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/183_core.type_system.typed_schema.t1_1.pb b/pkg/compiler/test/testdata/snacks-core/183_core.type_system.typed_schema.t1_1.pb new file mode 100644 index 0000000000..41bec7a027 Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/183_core.type_system.typed_schema.t1_1.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/183_core.type_system.typed_schema.t1_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/183_core.type_system.typed_schema.t1_1_task.yaml new file mode 100755 index 0000000000..268b43a319 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/183_core.type_system.typed_schema.t1_1_task.yaml @@ -0,0 +1,48 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.type_system.typed_schema + - task-name + - t1 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.type_system.typed_schema.t1 + resource_type: 1 + interface: + inputs: {} + outputs: + variables: + o0: + description: o0 + type: + Type: + Schema: + columns: + - name: x + - name: "y" + type: 2 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/184_core.type_system.typed_schema.t2_1.pb b/pkg/compiler/test/testdata/snacks-core/184_core.type_system.typed_schema.t2_1.pb new file mode 100644 index 0000000000..123178709a Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/184_core.type_system.typed_schema.t2_1.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/184_core.type_system.typed_schema.t2_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/184_core.type_system.typed_schema.t2_1_task.yaml new file mode 100755 index 0000000000..7d0c955a2b --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/184_core.type_system.typed_schema.t2_1_task.yaml @@ -0,0 +1,56 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.type_system.typed_schema + - task-name + - t2 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.type_system.typed_schema.t2 + resource_type: 1 + interface: + inputs: + variables: + schema: + description: schema + type: + Type: + Schema: + columns: + - name: x + - name: "y" + type: 2 + outputs: + variables: + o0: + description: o0 + type: + Type: + Schema: + columns: + - name: x + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/185_core.type_system.typed_schema.wf_2.pb b/pkg/compiler/test/testdata/snacks-core/185_core.type_system.typed_schema.wf_2.pb new file mode 100644 index 0000000000..0836d23815 Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/185_core.type_system.typed_schema.wf_2.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/185_core.type_system.typed_schema.wf_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/185_core.type_system.typed_schema.wf_2_wf.yaml new file mode 100755 index 0000000000..e1c92bd2fe --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/185_core.type_system.typed_schema.wf_2_wf.yaml @@ -0,0 +1,58 @@ +workflow: + id: + name: core.type_system.typed_schema.wf + resource_type: 2 + interface: + inputs: {} + outputs: + variables: + o0: + description: o0 + type: + Type: + Schema: + columns: + - name: x + metadata: {} + metadata_defaults: {} + nodes: + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.type_system.typed_schema.t1 + resource_type: 1 + overrides: {} + id: n0 + metadata: + InterruptibleValue: null + name: t1 + retries: {} + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.type_system.typed_schema.t2 + resource_type: 1 + overrides: {} + id: n1 + inputs: + - binding: + Value: + Promise: + node_id: n0 + var: o0 + var: schema + metadata: + InterruptibleValue: null + name: t2 + retries: {} + upstream_node_ids: + - n0 + outputs: + - binding: + Value: + Promise: + node_id: n1 + var: o0 + var: o0 diff --git a/pkg/compiler/test/testdata/snacks-core/186_core.type_system.typed_schema.wf_3.pb b/pkg/compiler/test/testdata/snacks-core/186_core.type_system.typed_schema.wf_3.pb new file mode 100644 index 0000000000..ba608b6c75 Binary files /dev/null and b/pkg/compiler/test/testdata/snacks-core/186_core.type_system.typed_schema.wf_3.pb differ diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/000_core.containerization.multi_images.svm_trainer_1_task.json b/pkg/compiler/test/testdata/snacks-core/compiled/000_core.containerization.multi_images.svm_trainer_1_task.json new file mode 100755 index 0000000000..05f2933329 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/000_core.containerization.multi_images.svm_trainer_1_task.json @@ -0,0 +1 @@ +{"template":{"id":{"resourceType":"TASK","name":"core.containerization.multi_images.svm_trainer"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{},"outputs":{"variables":{"test_features":{"type":{"structuredDatasetType":{"format":"parquet"}},"description":"test_features"},"test_labels":{"type":{"structuredDatasetType":{"format":"parquet"}},"description":"test_labels"},"train_features":{"type":{"structuredDatasetType":{"format":"parquet"}},"description":"train_features"},"train_labels":{"type":{"structuredDatasetType":{"format":"parquet"}},"description":"train_labels"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-with-sklearn-baa17ccf39aa667c5950bd713a4366ce7d5fccaf7f85e6be8c07fe4b522f92c3","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.containerization.multi_images","task-name","svm_trainer"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/000_core.containerization.multi_images.svm_trainer_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/000_core.containerization.multi_images.svm_trainer_1_task.yaml new file mode 100755 index 0000000000..95be6ba9cf --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/000_core.containerization.multi_images.svm_trainer_1_task.yaml @@ -0,0 +1,70 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.containerization.multi_images + - task-name + - svm_trainer + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-with-sklearn-baa17ccf39aa667c5950bd713a4366ce7d5fccaf7f85e6be8c07fe4b522f92c3 + resources: {} + id: + name: core.containerization.multi_images.svm_trainer + resource_type: 1 + interface: + inputs: {} + outputs: + variables: + test_features: + description: test_features + type: + Type: + StructuredDatasetType: + format: parquet + test_labels: + description: test_labels + type: + Type: + StructuredDatasetType: + format: parquet + train_features: + description: train_features + type: + Type: + StructuredDatasetType: + format: parquet + train_labels: + description: train_labels + type: + Type: + StructuredDatasetType: + format: parquet + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/001_core.containerization.multi_images.svm_predictor_1_task.json b/pkg/compiler/test/testdata/snacks-core/compiled/001_core.containerization.multi_images.svm_predictor_1_task.json new file mode 100755 index 0000000000..6fbaca5fea --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/001_core.containerization.multi_images.svm_predictor_1_task.json @@ -0,0 +1 @@ +{"template":{"id":{"resourceType":"TASK","name":"core.containerization.multi_images.svm_predictor"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"X_test":{"type":{"structuredDatasetType":{"format":"parquet"}},"description":"X_test"},"X_train":{"type":{"structuredDatasetType":{"format":"parquet"}},"description":"X_train"},"y_test":{"type":{"structuredDatasetType":{"format":"parquet"}},"description":"y_test"},"y_train":{"type":{"structuredDatasetType":{"format":"parquet"}},"description":"y_train"}}},"outputs":{"variables":{"o0":{"type":{"simple":"FLOAT"},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:multi-image-predict-98b125fd57d20594026941c2ebe7ef662e5acb7d6423660a65f493ca2d9aa267","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.containerization.multi_images","task-name","svm_predictor"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/001_core.containerization.multi_images.svm_predictor_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/001_core.containerization.multi_images.svm_predictor_1_task.yaml new file mode 100755 index 0000000000..18a6297ddb --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/001_core.containerization.multi_images.svm_predictor_1_task.yaml @@ -0,0 +1,76 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.containerization.multi_images + - task-name + - svm_predictor + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:multi-image-predict-98b125fd57d20594026941c2ebe7ef662e5acb7d6423660a65f493ca2d9aa267 + resources: {} + id: + name: core.containerization.multi_images.svm_predictor + resource_type: 1 + interface: + inputs: + variables: + X_test: + description: X_test + type: + Type: + StructuredDatasetType: + format: parquet + X_train: + description: X_train + type: + Type: + StructuredDatasetType: + format: parquet + y_test: + description: y_test + type: + Type: + StructuredDatasetType: + format: parquet + y_train: + description: y_train + type: + Type: + StructuredDatasetType: + format: parquet + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 2 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/002_core.containerization.multi_images.my_workflow_2_wf.json b/pkg/compiler/test/testdata/snacks-core/compiled/002_core.containerization.multi_images.my_workflow_2_wf.json new file mode 100755 index 0000000000..145107a042 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/002_core.containerization.multi_images.my_workflow_2_wf.json @@ -0,0 +1 @@ +{"primary":{"template":{"id":{"resourceType":"WORKFLOW","name":"core.containerization.multi_images.my_workflow"},"metadata":{},"interface":{"inputs":{},"outputs":{"variables":{"o0":{"type":{"simple":"FLOAT"},"description":"o0"}}}},"nodes":[{"id":"start-node"},{"id":"end-node","inputs":[{"var":"o0","binding":{"promise":{"nodeId":"n1","var":"o0"}}}]},{"id":"n0","metadata":{"name":"svm_trainer","retries":{}},"taskNode":{"referenceId":{"resourceType":"TASK","name":"core.containerization.multi_images.svm_trainer"},"overrides":{}}},{"id":"n1","metadata":{"name":"svm_predictor","retries":{}},"inputs":[{"var":"X_test","binding":{"promise":{"nodeId":"n0","var":"test_features"}}},{"var":"X_train","binding":{"promise":{"nodeId":"n0","var":"train_features"}}},{"var":"y_test","binding":{"promise":{"nodeId":"n0","var":"test_labels"}}},{"var":"y_train","binding":{"promise":{"nodeId":"n0","var":"train_labels"}}}],"upstreamNodeIds":["n0"],"taskNode":{"referenceId":{"resourceType":"TASK","name":"core.containerization.multi_images.svm_predictor"},"overrides":{}}}],"outputs":[{"var":"o0","binding":{"promise":{"nodeId":"n1","var":"o0"}}}],"metadataDefaults":{}},"connections":{"downstream":{"n0":{"ids":["n1"]},"n1":{"ids":["end-node"]},"start-node":{"ids":["n0"]}},"upstream":{"end-node":{"ids":["n1"]},"n0":{"ids":["start-node"]},"n1":{"ids":["n0"]}}}},"tasks":[{"template":{"id":{"resourceType":"TASK","name":"core.containerization.multi_images.svm_predictor"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"X_test":{"type":{"structuredDatasetType":{"format":"parquet"}},"description":"X_test"},"X_train":{"type":{"structuredDatasetType":{"format":"parquet"}},"description":"X_train"},"y_test":{"type":{"structuredDatasetType":{"format":"parquet"}},"description":"y_test"},"y_train":{"type":{"structuredDatasetType":{"format":"parquet"}},"description":"y_train"}}},"outputs":{"variables":{"o0":{"type":{"simple":"FLOAT"},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:multi-image-predict-98b125fd57d20594026941c2ebe7ef662e5acb7d6423660a65f493ca2d9aa267","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.containerization.multi_images","task-name","svm_predictor"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},{"template":{"id":{"resourceType":"TASK","name":"core.containerization.multi_images.svm_trainer"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{},"outputs":{"variables":{"test_features":{"type":{"structuredDatasetType":{"format":"parquet"}},"description":"test_features"},"test_labels":{"type":{"structuredDatasetType":{"format":"parquet"}},"description":"test_labels"},"train_features":{"type":{"structuredDatasetType":{"format":"parquet"}},"description":"train_features"},"train_labels":{"type":{"structuredDatasetType":{"format":"parquet"}},"description":"train_labels"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-with-sklearn-baa17ccf39aa667c5950bd713a4366ce7d5fccaf7f85e6be8c07fe4b522f92c3","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.containerization.multi_images","task-name","svm_trainer"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}}]} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/002_core.containerization.multi_images.my_workflow_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/002_core.containerization.multi_images.my_workflow_2_wf.yaml new file mode 100755 index 0000000000..fffe031b06 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/002_core.containerization.multi_images.my_workflow_2_wf.yaml @@ -0,0 +1,254 @@ +primary: + connections: + downstream: + n0: + ids: + - n1 + n1: + ids: + - end-node + start-node: + ids: + - n0 + upstream: + end-node: + ids: + - n1 + n0: + ids: + - start-node + n1: + ids: + - n0 + template: + id: + name: core.containerization.multi_images.my_workflow + resource_type: 2 + interface: + inputs: {} + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 2 + metadata: {} + metadata_defaults: {} + nodes: + - Target: null + id: start-node + - Target: null + id: end-node + inputs: + - binding: + Value: + Promise: + node_id: n1 + var: o0 + var: o0 + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.containerization.multi_images.svm_trainer + resource_type: 1 + overrides: {} + id: n0 + metadata: + InterruptibleValue: null + name: svm_trainer + retries: {} + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.containerization.multi_images.svm_predictor + resource_type: 1 + overrides: {} + id: n1 + inputs: + - binding: + Value: + Promise: + node_id: n0 + var: test_features + var: X_test + - binding: + Value: + Promise: + node_id: n0 + var: train_features + var: X_train + - binding: + Value: + Promise: + node_id: n0 + var: test_labels + var: y_test + - binding: + Value: + Promise: + node_id: n0 + var: train_labels + var: y_train + metadata: + InterruptibleValue: null + name: svm_predictor + retries: {} + upstream_node_ids: + - n0 + outputs: + - binding: + Value: + Promise: + node_id: n1 + var: o0 + var: o0 +tasks: +- template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.containerization.multi_images + - task-name + - svm_predictor + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:multi-image-predict-98b125fd57d20594026941c2ebe7ef662e5acb7d6423660a65f493ca2d9aa267 + resources: {} + id: + name: core.containerization.multi_images.svm_predictor + resource_type: 1 + interface: + inputs: + variables: + X_test: + description: X_test + type: + Type: + StructuredDatasetType: + format: parquet + X_train: + description: X_train + type: + Type: + StructuredDatasetType: + format: parquet + y_test: + description: y_test + type: + Type: + StructuredDatasetType: + format: parquet + y_train: + description: y_train + type: + Type: + StructuredDatasetType: + format: parquet + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 2 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task +- template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.containerization.multi_images + - task-name + - svm_trainer + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-with-sklearn-baa17ccf39aa667c5950bd713a4366ce7d5fccaf7f85e6be8c07fe4b522f92c3 + resources: {} + id: + name: core.containerization.multi_images.svm_trainer + resource_type: 1 + interface: + inputs: {} + outputs: + variables: + test_features: + description: test_features + type: + Type: + StructuredDatasetType: + format: parquet + test_labels: + description: test_labels + type: + Type: + StructuredDatasetType: + format: parquet + train_features: + description: train_features + type: + Type: + StructuredDatasetType: + format: parquet + train_labels: + description: train_labels + type: + Type: + StructuredDatasetType: + format: parquet + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/004_ellipse-area-metadata-shell_1_task.json b/pkg/compiler/test/testdata/snacks-core/compiled/004_ellipse-area-metadata-shell_1_task.json new file mode 100755 index 0000000000..3c1b8893c7 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/004_ellipse-area-metadata-shell_1_task.json @@ -0,0 +1 @@ +{"template":{"id":{"resourceType":"TASK","name":"ellipse-area-metadata-shell"},"type":"raw-container","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"a":{"type":{"simple":"FLOAT"},"description":"a"},"b":{"type":{"simple":"FLOAT"},"description":"b"}}},"outputs":{"variables":{"area":{"type":{"simple":"FLOAT"},"description":"area"},"metadata":{"type":{"simple":"STRING"},"description":"metadata"}}}},"container":{"image":"ghcr.io/flyteorg/rawcontainers-shell:v1","command":["./calculate-ellipse-area.sh","/var/inputs","/var/outputs"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}],"dataConfig":{"enabled":true,"inputPath":"/var/inputs","outputPath":"/var/outputs"}}}} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/004_ellipse-area-metadata-shell_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/004_ellipse-area-metadata-shell_1_task.yaml new file mode 100755 index 0000000000..50241dad8a --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/004_ellipse-area-metadata-shell_1_task.yaml @@ -0,0 +1,56 @@ +template: + Target: + Container: + command: + - ./calculate-ellipse-area.sh + - /var/inputs + - /var/outputs + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + data_config: + enabled: true + input_path: /var/inputs + output_path: /var/outputs + image: ghcr.io/flyteorg/rawcontainers-shell:v1 + resources: {} + id: + name: ellipse-area-metadata-shell + resource_type: 1 + interface: + inputs: + variables: + a: + description: a + type: + Type: + Simple: 2 + b: + description: b + type: + Type: + Simple: 2 + outputs: + variables: + area: + description: area + type: + Type: + Simple: 2 + metadata: + description: metadata + type: + Type: + Simple: 3 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: raw-container diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/005_ellipse-area-metadata-python_1_task.json b/pkg/compiler/test/testdata/snacks-core/compiled/005_ellipse-area-metadata-python_1_task.json new file mode 100755 index 0000000000..49d5d0f583 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/005_ellipse-area-metadata-python_1_task.json @@ -0,0 +1 @@ +{"template":{"id":{"resourceType":"TASK","name":"ellipse-area-metadata-python"},"type":"raw-container","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"a":{"type":{"simple":"FLOAT"},"description":"a"},"b":{"type":{"simple":"FLOAT"},"description":"b"}}},"outputs":{"variables":{"area":{"type":{"simple":"FLOAT"},"description":"area"},"metadata":{"type":{"simple":"STRING"},"description":"metadata"}}}},"container":{"image":"ghcr.io/flyteorg/rawcontainers-python:v1","command":["python","calculate-ellipse-area.py","/var/inputs","/var/outputs"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}],"dataConfig":{"enabled":true,"inputPath":"/var/inputs","outputPath":"/var/outputs"}}}} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/005_ellipse-area-metadata-python_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/005_ellipse-area-metadata-python_1_task.yaml new file mode 100755 index 0000000000..0f66634a95 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/005_ellipse-area-metadata-python_1_task.yaml @@ -0,0 +1,57 @@ +template: + Target: + Container: + command: + - python + - calculate-ellipse-area.py + - /var/inputs + - /var/outputs + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + data_config: + enabled: true + input_path: /var/inputs + output_path: /var/outputs + image: ghcr.io/flyteorg/rawcontainers-python:v1 + resources: {} + id: + name: ellipse-area-metadata-python + resource_type: 1 + interface: + inputs: + variables: + a: + description: a + type: + Type: + Simple: 2 + b: + description: b + type: + Type: + Simple: 2 + outputs: + variables: + area: + description: area + type: + Type: + Simple: 2 + metadata: + description: metadata + type: + Type: + Simple: 3 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: raw-container diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/006_ellipse-area-metadata-r_1_task.json b/pkg/compiler/test/testdata/snacks-core/compiled/006_ellipse-area-metadata-r_1_task.json new file mode 100755 index 0000000000..a2cb0f3f2f --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/006_ellipse-area-metadata-r_1_task.json @@ -0,0 +1 @@ +{"template":{"id":{"resourceType":"TASK","name":"ellipse-area-metadata-r"},"type":"raw-container","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"a":{"type":{"simple":"FLOAT"},"description":"a"},"b":{"type":{"simple":"FLOAT"},"description":"b"}}},"outputs":{"variables":{"area":{"type":{"simple":"FLOAT"},"description":"area"},"metadata":{"type":{"simple":"STRING"},"description":"metadata"}}}},"container":{"image":"ghcr.io/flyteorg/rawcontainers-r:v1","command":["Rscript","--vanilla","calculate-ellipse-area.R","/var/inputs","/var/outputs"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}],"dataConfig":{"enabled":true,"inputPath":"/var/inputs","outputPath":"/var/outputs"}}}} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/006_ellipse-area-metadata-r_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/006_ellipse-area-metadata-r_1_task.yaml new file mode 100755 index 0000000000..033b2e4714 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/006_ellipse-area-metadata-r_1_task.yaml @@ -0,0 +1,58 @@ +template: + Target: + Container: + command: + - Rscript + - --vanilla + - calculate-ellipse-area.R + - /var/inputs + - /var/outputs + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + data_config: + enabled: true + input_path: /var/inputs + output_path: /var/outputs + image: ghcr.io/flyteorg/rawcontainers-r:v1 + resources: {} + id: + name: ellipse-area-metadata-r + resource_type: 1 + interface: + inputs: + variables: + a: + description: a + type: + Type: + Simple: 2 + b: + description: b + type: + Type: + Simple: 2 + outputs: + variables: + area: + description: area + type: + Type: + Simple: 2 + metadata: + description: metadata + type: + Type: + Simple: 3 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: raw-container diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/007_ellipse-area-metadata-haskell_1_task.json b/pkg/compiler/test/testdata/snacks-core/compiled/007_ellipse-area-metadata-haskell_1_task.json new file mode 100755 index 0000000000..613bfa82b6 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/007_ellipse-area-metadata-haskell_1_task.json @@ -0,0 +1 @@ +{"template":{"id":{"resourceType":"TASK","name":"ellipse-area-metadata-haskell"},"type":"raw-container","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"a":{"type":{"simple":"FLOAT"},"description":"a"},"b":{"type":{"simple":"FLOAT"},"description":"b"}}},"outputs":{"variables":{"area":{"type":{"simple":"FLOAT"},"description":"area"},"metadata":{"type":{"simple":"STRING"},"description":"metadata"}}}},"container":{"image":"ghcr.io/flyteorg/rawcontainers-haskell:v1","command":["./calculate-ellipse-area","/var/inputs","/var/outputs"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}],"dataConfig":{"enabled":true,"inputPath":"/var/inputs","outputPath":"/var/outputs"}}}} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/007_ellipse-area-metadata-haskell_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/007_ellipse-area-metadata-haskell_1_task.yaml new file mode 100755 index 0000000000..642f9c108b --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/007_ellipse-area-metadata-haskell_1_task.yaml @@ -0,0 +1,56 @@ +template: + Target: + Container: + command: + - ./calculate-ellipse-area + - /var/inputs + - /var/outputs + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + data_config: + enabled: true + input_path: /var/inputs + output_path: /var/outputs + image: ghcr.io/flyteorg/rawcontainers-haskell:v1 + resources: {} + id: + name: ellipse-area-metadata-haskell + resource_type: 1 + interface: + inputs: + variables: + a: + description: a + type: + Type: + Simple: 2 + b: + description: b + type: + Type: + Simple: 2 + outputs: + variables: + area: + description: area + type: + Type: + Simple: 2 + metadata: + description: metadata + type: + Type: + Simple: 3 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: raw-container diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/008_ellipse-area-metadata-julia_1_task.json b/pkg/compiler/test/testdata/snacks-core/compiled/008_ellipse-area-metadata-julia_1_task.json new file mode 100755 index 0000000000..1c18dd0bff --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/008_ellipse-area-metadata-julia_1_task.json @@ -0,0 +1 @@ +{"template":{"id":{"resourceType":"TASK","name":"ellipse-area-metadata-julia"},"type":"raw-container","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"a":{"type":{"simple":"FLOAT"},"description":"a"},"b":{"type":{"simple":"FLOAT"},"description":"b"}}},"outputs":{"variables":{"area":{"type":{"simple":"FLOAT"},"description":"area"},"metadata":{"type":{"simple":"STRING"},"description":"metadata"}}}},"container":{"image":"ghcr.io/flyteorg/rawcontainers-julia:v1","command":["julia","calculate-ellipse-area.jl","/var/inputs","/var/outputs"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}],"dataConfig":{"enabled":true,"inputPath":"/var/inputs","outputPath":"/var/outputs"}}}} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/008_ellipse-area-metadata-julia_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/008_ellipse-area-metadata-julia_1_task.yaml new file mode 100755 index 0000000000..0a85c274cf --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/008_ellipse-area-metadata-julia_1_task.yaml @@ -0,0 +1,57 @@ +template: + Target: + Container: + command: + - julia + - calculate-ellipse-area.jl + - /var/inputs + - /var/outputs + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + data_config: + enabled: true + input_path: /var/inputs + output_path: /var/outputs + image: ghcr.io/flyteorg/rawcontainers-julia:v1 + resources: {} + id: + name: ellipse-area-metadata-julia + resource_type: 1 + interface: + inputs: + variables: + a: + description: a + type: + Type: + Simple: 2 + b: + description: b + type: + Type: + Simple: 2 + outputs: + variables: + area: + description: area + type: + Type: + Simple: 2 + metadata: + description: metadata + type: + Type: + Simple: 3 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: raw-container diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/009_core.containerization.raw_container.report_all_calculated_areas_1_task.json b/pkg/compiler/test/testdata/snacks-core/compiled/009_core.containerization.raw_container.report_all_calculated_areas_1_task.json new file mode 100755 index 0000000000..0fa9f05a6c --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/009_core.containerization.raw_container.report_all_calculated_areas_1_task.json @@ -0,0 +1 @@ +{"template":{"id":{"resourceType":"TASK","name":"core.containerization.raw_container.report_all_calculated_areas"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"area_haskell":{"type":{"simple":"FLOAT"},"description":"area_haskell"},"area_julia":{"type":{"simple":"FLOAT"},"description":"area_julia"},"area_python":{"type":{"simple":"FLOAT"},"description":"area_python"},"area_r":{"type":{"simple":"FLOAT"},"description":"area_r"},"area_shell":{"type":{"simple":"FLOAT"},"description":"area_shell"},"metadata_haskell":{"type":{"simple":"STRING"},"description":"metadata_haskell"},"metadata_julia":{"type":{"simple":"STRING"},"description":"metadata_julia"},"metadata_python":{"type":{"simple":"STRING"},"description":"metadata_python"},"metadata_r":{"type":{"simple":"STRING"},"description":"metadata_r"},"metadata_shell":{"type":{"simple":"STRING"},"description":"metadata_shell"}}},"outputs":{}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.containerization.raw_container","task-name","report_all_calculated_areas"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/009_core.containerization.raw_container.report_all_calculated_areas_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/009_core.containerization.raw_container.report_all_calculated_areas_1_task.yaml new file mode 100755 index 0000000000..a909e7c1af --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/009_core.containerization.raw_container.report_all_calculated_areas_1_task.yaml @@ -0,0 +1,96 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.containerization.raw_container + - task-name + - report_all_calculated_areas + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.containerization.raw_container.report_all_calculated_areas + resource_type: 1 + interface: + inputs: + variables: + area_haskell: + description: area_haskell + type: + Type: + Simple: 2 + area_julia: + description: area_julia + type: + Type: + Simple: 2 + area_python: + description: area_python + type: + Type: + Simple: 2 + area_r: + description: area_r + type: + Type: + Simple: 2 + area_shell: + description: area_shell + type: + Type: + Simple: 2 + metadata_haskell: + description: metadata_haskell + type: + Type: + Simple: 3 + metadata_julia: + description: metadata_julia + type: + Type: + Simple: 3 + metadata_python: + description: metadata_python + type: + Type: + Simple: 3 + metadata_r: + description: metadata_r + type: + Type: + Simple: 3 + metadata_shell: + description: metadata_shell + type: + Type: + Simple: 3 + outputs: {} + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/010_core.containerization.raw_container.wf_2_wf.json b/pkg/compiler/test/testdata/snacks-core/compiled/010_core.containerization.raw_container.wf_2_wf.json new file mode 100755 index 0000000000..abeb832c67 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/010_core.containerization.raw_container.wf_2_wf.json @@ -0,0 +1 @@ +{"primary":{"template":{"id":{"resourceType":"WORKFLOW","name":"core.containerization.raw_container.wf"},"metadata":{},"interface":{"inputs":{"variables":{"a":{"type":{"simple":"FLOAT"},"description":"a"},"b":{"type":{"simple":"FLOAT"},"description":"b"}}},"outputs":{}},"nodes":[{"id":"start-node"},{"id":"end-node"},{"id":"n0","metadata":{"name":"ellipse-area-metadata-shell","retries":{}},"inputs":[{"var":"a","binding":{"promise":{"nodeId":"start-node","var":"a"}}},{"var":"b","binding":{"promise":{"nodeId":"start-node","var":"b"}}}],"taskNode":{"referenceId":{"resourceType":"TASK","name":"ellipse-area-metadata-shell"},"overrides":{}}},{"id":"n1","metadata":{"name":"ellipse-area-metadata-python","retries":{}},"inputs":[{"var":"a","binding":{"promise":{"nodeId":"start-node","var":"a"}}},{"var":"b","binding":{"promise":{"nodeId":"start-node","var":"b"}}}],"taskNode":{"referenceId":{"resourceType":"TASK","name":"ellipse-area-metadata-python"},"overrides":{}}},{"id":"n2","metadata":{"name":"ellipse-area-metadata-r","retries":{}},"inputs":[{"var":"a","binding":{"promise":{"nodeId":"start-node","var":"a"}}},{"var":"b","binding":{"promise":{"nodeId":"start-node","var":"b"}}}],"taskNode":{"referenceId":{"resourceType":"TASK","name":"ellipse-area-metadata-r"},"overrides":{}}},{"id":"n3","metadata":{"name":"ellipse-area-metadata-haskell","retries":{}},"inputs":[{"var":"a","binding":{"promise":{"nodeId":"start-node","var":"a"}}},{"var":"b","binding":{"promise":{"nodeId":"start-node","var":"b"}}}],"taskNode":{"referenceId":{"resourceType":"TASK","name":"ellipse-area-metadata-haskell"},"overrides":{}}},{"id":"n4","metadata":{"name":"ellipse-area-metadata-julia","retries":{}},"inputs":[{"var":"a","binding":{"promise":{"nodeId":"start-node","var":"a"}}},{"var":"b","binding":{"promise":{"nodeId":"start-node","var":"b"}}}],"taskNode":{"referenceId":{"resourceType":"TASK","name":"ellipse-area-metadata-julia"},"overrides":{}}},{"id":"n5","metadata":{"name":"report_all_calculated_areas","retries":{}},"inputs":[{"var":"area_haskell","binding":{"promise":{"nodeId":"n3","var":"area"}}},{"var":"area_julia","binding":{"promise":{"nodeId":"n4","var":"area"}}},{"var":"area_python","binding":{"promise":{"nodeId":"n1","var":"area"}}},{"var":"area_r","binding":{"promise":{"nodeId":"n2","var":"area"}}},{"var":"area_shell","binding":{"promise":{"nodeId":"n0","var":"area"}}},{"var":"metadata_haskell","binding":{"promise":{"nodeId":"n3","var":"metadata"}}},{"var":"metadata_julia","binding":{"promise":{"nodeId":"n4","var":"metadata"}}},{"var":"metadata_python","binding":{"promise":{"nodeId":"n1","var":"metadata"}}},{"var":"metadata_r","binding":{"promise":{"nodeId":"n2","var":"metadata"}}},{"var":"metadata_shell","binding":{"promise":{"nodeId":"n0","var":"metadata"}}}],"upstreamNodeIds":["n0","n1","n2","n3","n4"],"taskNode":{"referenceId":{"resourceType":"TASK","name":"core.containerization.raw_container.report_all_calculated_areas"},"overrides":{}}}],"metadataDefaults":{}},"connections":{"downstream":{"n0":{"ids":["n5"]},"n1":{"ids":["n5"]},"n2":{"ids":["n5"]},"n3":{"ids":["n5"]},"n4":{"ids":["n5"]},"n5":{"ids":["end-node"]},"start-node":{"ids":["n0","n1","n2","n3","n4"]}},"upstream":{"end-node":{"ids":["n5"]},"n0":{"ids":["start-node"]},"n1":{"ids":["start-node"]},"n2":{"ids":["start-node"]},"n3":{"ids":["start-node"]},"n4":{"ids":["start-node"]},"n5":{"ids":["n0","n1","n2","n3","n4"]}}}},"tasks":[{"template":{"id":{"resourceType":"TASK","name":"core.containerization.raw_container.report_all_calculated_areas"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"area_haskell":{"type":{"simple":"FLOAT"},"description":"area_haskell"},"area_julia":{"type":{"simple":"FLOAT"},"description":"area_julia"},"area_python":{"type":{"simple":"FLOAT"},"description":"area_python"},"area_r":{"type":{"simple":"FLOAT"},"description":"area_r"},"area_shell":{"type":{"simple":"FLOAT"},"description":"area_shell"},"metadata_haskell":{"type":{"simple":"STRING"},"description":"metadata_haskell"},"metadata_julia":{"type":{"simple":"STRING"},"description":"metadata_julia"},"metadata_python":{"type":{"simple":"STRING"},"description":"metadata_python"},"metadata_r":{"type":{"simple":"STRING"},"description":"metadata_r"},"metadata_shell":{"type":{"simple":"STRING"},"description":"metadata_shell"}}},"outputs":{}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.containerization.raw_container","task-name","report_all_calculated_areas"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},{"template":{"id":{"resourceType":"TASK","name":"ellipse-area-metadata-haskell"},"type":"raw-container","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"a":{"type":{"simple":"FLOAT"},"description":"a"},"b":{"type":{"simple":"FLOAT"},"description":"b"}}},"outputs":{"variables":{"area":{"type":{"simple":"FLOAT"},"description":"area"},"metadata":{"type":{"simple":"STRING"},"description":"metadata"}}}},"container":{"image":"ghcr.io/flyteorg/rawcontainers-haskell:v1","command":["./calculate-ellipse-area","/var/inputs","/var/outputs"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}],"dataConfig":{"enabled":true,"inputPath":"/var/inputs","outputPath":"/var/outputs"}}}},{"template":{"id":{"resourceType":"TASK","name":"ellipse-area-metadata-julia"},"type":"raw-container","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"a":{"type":{"simple":"FLOAT"},"description":"a"},"b":{"type":{"simple":"FLOAT"},"description":"b"}}},"outputs":{"variables":{"area":{"type":{"simple":"FLOAT"},"description":"area"},"metadata":{"type":{"simple":"STRING"},"description":"metadata"}}}},"container":{"image":"ghcr.io/flyteorg/rawcontainers-julia:v1","command":["julia","calculate-ellipse-area.jl","/var/inputs","/var/outputs"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}],"dataConfig":{"enabled":true,"inputPath":"/var/inputs","outputPath":"/var/outputs"}}}},{"template":{"id":{"resourceType":"TASK","name":"ellipse-area-metadata-python"},"type":"raw-container","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"a":{"type":{"simple":"FLOAT"},"description":"a"},"b":{"type":{"simple":"FLOAT"},"description":"b"}}},"outputs":{"variables":{"area":{"type":{"simple":"FLOAT"},"description":"area"},"metadata":{"type":{"simple":"STRING"},"description":"metadata"}}}},"container":{"image":"ghcr.io/flyteorg/rawcontainers-python:v1","command":["python","calculate-ellipse-area.py","/var/inputs","/var/outputs"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}],"dataConfig":{"enabled":true,"inputPath":"/var/inputs","outputPath":"/var/outputs"}}}},{"template":{"id":{"resourceType":"TASK","name":"ellipse-area-metadata-r"},"type":"raw-container","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"a":{"type":{"simple":"FLOAT"},"description":"a"},"b":{"type":{"simple":"FLOAT"},"description":"b"}}},"outputs":{"variables":{"area":{"type":{"simple":"FLOAT"},"description":"area"},"metadata":{"type":{"simple":"STRING"},"description":"metadata"}}}},"container":{"image":"ghcr.io/flyteorg/rawcontainers-r:v1","command":["Rscript","--vanilla","calculate-ellipse-area.R","/var/inputs","/var/outputs"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}],"dataConfig":{"enabled":true,"inputPath":"/var/inputs","outputPath":"/var/outputs"}}}},{"template":{"id":{"resourceType":"TASK","name":"ellipse-area-metadata-shell"},"type":"raw-container","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"a":{"type":{"simple":"FLOAT"},"description":"a"},"b":{"type":{"simple":"FLOAT"},"description":"b"}}},"outputs":{"variables":{"area":{"type":{"simple":"FLOAT"},"description":"area"},"metadata":{"type":{"simple":"STRING"},"description":"metadata"}}}},"container":{"image":"ghcr.io/flyteorg/rawcontainers-shell:v1","command":["./calculate-ellipse-area.sh","/var/inputs","/var/outputs"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}],"dataConfig":{"enabled":true,"inputPath":"/var/inputs","outputPath":"/var/outputs"}}}}]} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/010_core.containerization.raw_container.wf_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/010_core.containerization.raw_container.wf_2_wf.yaml new file mode 100755 index 0000000000..534f0b9374 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/010_core.containerization.raw_container.wf_2_wf.yaml @@ -0,0 +1,664 @@ +primary: + connections: + downstream: + n0: + ids: + - n5 + n1: + ids: + - n5 + n2: + ids: + - n5 + n3: + ids: + - n5 + n4: + ids: + - n5 + n5: + ids: + - end-node + start-node: + ids: + - n0 + - n1 + - n2 + - n3 + - n4 + upstream: + end-node: + ids: + - n5 + n0: + ids: + - start-node + n1: + ids: + - start-node + n2: + ids: + - start-node + n3: + ids: + - start-node + n4: + ids: + - start-node + n5: + ids: + - n0 + - n1 + - n2 + - n3 + - n4 + template: + id: + name: core.containerization.raw_container.wf + resource_type: 2 + interface: + inputs: + variables: + a: + description: a + type: + Type: + Simple: 2 + b: + description: b + type: + Type: + Simple: 2 + outputs: {} + metadata: {} + metadata_defaults: {} + nodes: + - Target: null + id: start-node + - Target: null + id: end-node + - Target: + TaskNode: + Reference: + ReferenceId: + name: ellipse-area-metadata-shell + resource_type: 1 + overrides: {} + id: n0 + inputs: + - binding: + Value: + Promise: + node_id: start-node + var: a + var: a + - binding: + Value: + Promise: + node_id: start-node + var: b + var: b + metadata: + InterruptibleValue: null + name: ellipse-area-metadata-shell + retries: {} + - Target: + TaskNode: + Reference: + ReferenceId: + name: ellipse-area-metadata-python + resource_type: 1 + overrides: {} + id: n1 + inputs: + - binding: + Value: + Promise: + node_id: start-node + var: a + var: a + - binding: + Value: + Promise: + node_id: start-node + var: b + var: b + metadata: + InterruptibleValue: null + name: ellipse-area-metadata-python + retries: {} + - Target: + TaskNode: + Reference: + ReferenceId: + name: ellipse-area-metadata-r + resource_type: 1 + overrides: {} + id: n2 + inputs: + - binding: + Value: + Promise: + node_id: start-node + var: a + var: a + - binding: + Value: + Promise: + node_id: start-node + var: b + var: b + metadata: + InterruptibleValue: null + name: ellipse-area-metadata-r + retries: {} + - Target: + TaskNode: + Reference: + ReferenceId: + name: ellipse-area-metadata-haskell + resource_type: 1 + overrides: {} + id: n3 + inputs: + - binding: + Value: + Promise: + node_id: start-node + var: a + var: a + - binding: + Value: + Promise: + node_id: start-node + var: b + var: b + metadata: + InterruptibleValue: null + name: ellipse-area-metadata-haskell + retries: {} + - Target: + TaskNode: + Reference: + ReferenceId: + name: ellipse-area-metadata-julia + resource_type: 1 + overrides: {} + id: n4 + inputs: + - binding: + Value: + Promise: + node_id: start-node + var: a + var: a + - binding: + Value: + Promise: + node_id: start-node + var: b + var: b + metadata: + InterruptibleValue: null + name: ellipse-area-metadata-julia + retries: {} + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.containerization.raw_container.report_all_calculated_areas + resource_type: 1 + overrides: {} + id: n5 + inputs: + - binding: + Value: + Promise: + node_id: n3 + var: area + var: area_haskell + - binding: + Value: + Promise: + node_id: n4 + var: area + var: area_julia + - binding: + Value: + Promise: + node_id: n1 + var: area + var: area_python + - binding: + Value: + Promise: + node_id: n2 + var: area + var: area_r + - binding: + Value: + Promise: + node_id: n0 + var: area + var: area_shell + - binding: + Value: + Promise: + node_id: n3 + var: metadata + var: metadata_haskell + - binding: + Value: + Promise: + node_id: n4 + var: metadata + var: metadata_julia + - binding: + Value: + Promise: + node_id: n1 + var: metadata + var: metadata_python + - binding: + Value: + Promise: + node_id: n2 + var: metadata + var: metadata_r + - binding: + Value: + Promise: + node_id: n0 + var: metadata + var: metadata_shell + metadata: + InterruptibleValue: null + name: report_all_calculated_areas + retries: {} + upstream_node_ids: + - n0 + - n1 + - n2 + - n3 + - n4 +tasks: +- template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.containerization.raw_container + - task-name + - report_all_calculated_areas + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.containerization.raw_container.report_all_calculated_areas + resource_type: 1 + interface: + inputs: + variables: + area_haskell: + description: area_haskell + type: + Type: + Simple: 2 + area_julia: + description: area_julia + type: + Type: + Simple: 2 + area_python: + description: area_python + type: + Type: + Simple: 2 + area_r: + description: area_r + type: + Type: + Simple: 2 + area_shell: + description: area_shell + type: + Type: + Simple: 2 + metadata_haskell: + description: metadata_haskell + type: + Type: + Simple: 3 + metadata_julia: + description: metadata_julia + type: + Type: + Simple: 3 + metadata_python: + description: metadata_python + type: + Type: + Simple: 3 + metadata_r: + description: metadata_r + type: + Type: + Simple: 3 + metadata_shell: + description: metadata_shell + type: + Type: + Simple: 3 + outputs: {} + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task +- template: + Target: + Container: + command: + - ./calculate-ellipse-area + - /var/inputs + - /var/outputs + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + data_config: + enabled: true + input_path: /var/inputs + output_path: /var/outputs + image: ghcr.io/flyteorg/rawcontainers-haskell:v1 + resources: {} + id: + name: ellipse-area-metadata-haskell + resource_type: 1 + interface: + inputs: + variables: + a: + description: a + type: + Type: + Simple: 2 + b: + description: b + type: + Type: + Simple: 2 + outputs: + variables: + area: + description: area + type: + Type: + Simple: 2 + metadata: + description: metadata + type: + Type: + Simple: 3 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: raw-container +- template: + Target: + Container: + command: + - julia + - calculate-ellipse-area.jl + - /var/inputs + - /var/outputs + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + data_config: + enabled: true + input_path: /var/inputs + output_path: /var/outputs + image: ghcr.io/flyteorg/rawcontainers-julia:v1 + resources: {} + id: + name: ellipse-area-metadata-julia + resource_type: 1 + interface: + inputs: + variables: + a: + description: a + type: + Type: + Simple: 2 + b: + description: b + type: + Type: + Simple: 2 + outputs: + variables: + area: + description: area + type: + Type: + Simple: 2 + metadata: + description: metadata + type: + Type: + Simple: 3 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: raw-container +- template: + Target: + Container: + command: + - python + - calculate-ellipse-area.py + - /var/inputs + - /var/outputs + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + data_config: + enabled: true + input_path: /var/inputs + output_path: /var/outputs + image: ghcr.io/flyteorg/rawcontainers-python:v1 + resources: {} + id: + name: ellipse-area-metadata-python + resource_type: 1 + interface: + inputs: + variables: + a: + description: a + type: + Type: + Simple: 2 + b: + description: b + type: + Type: + Simple: 2 + outputs: + variables: + area: + description: area + type: + Type: + Simple: 2 + metadata: + description: metadata + type: + Type: + Simple: 3 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: raw-container +- template: + Target: + Container: + command: + - Rscript + - --vanilla + - calculate-ellipse-area.R + - /var/inputs + - /var/outputs + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + data_config: + enabled: true + input_path: /var/inputs + output_path: /var/outputs + image: ghcr.io/flyteorg/rawcontainers-r:v1 + resources: {} + id: + name: ellipse-area-metadata-r + resource_type: 1 + interface: + inputs: + variables: + a: + description: a + type: + Type: + Simple: 2 + b: + description: b + type: + Type: + Simple: 2 + outputs: + variables: + area: + description: area + type: + Type: + Simple: 2 + metadata: + description: metadata + type: + Type: + Simple: 3 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: raw-container +- template: + Target: + Container: + command: + - ./calculate-ellipse-area.sh + - /var/inputs + - /var/outputs + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + data_config: + enabled: true + input_path: /var/inputs + output_path: /var/outputs + image: ghcr.io/flyteorg/rawcontainers-shell:v1 + resources: {} + id: + name: ellipse-area-metadata-shell + resource_type: 1 + interface: + inputs: + variables: + a: + description: a + type: + Type: + Simple: 2 + b: + description: b + type: + Type: + Simple: 2 + outputs: + variables: + area: + description: area + type: + Type: + Simple: 2 + metadata: + description: metadata + type: + Type: + Simple: 3 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: raw-container diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/012_core.containerization.use_secrets.secret_task_1_task.json b/pkg/compiler/test/testdata/snacks-core/compiled/012_core.containerization.use_secrets.secret_task_1_task.json new file mode 100755 index 0000000000..18afcad0b8 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/012_core.containerization.use_secrets.secret_task_1_task.json @@ -0,0 +1 @@ +{"template":{"id":{"resourceType":"TASK","name":"core.containerization.use_secrets.secret_task"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{},"outputs":{"variables":{"o0":{"type":{"simple":"STRING"},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.containerization.use_secrets","task-name","secret_task"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]},"securityContext":{"secrets":[{"group":"user-info","key":"user_secret"}]}}} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/012_core.containerization.use_secrets.secret_task_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/012_core.containerization.use_secrets.secret_task_1_task.yaml new file mode 100755 index 0000000000..8ffeba7168 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/012_core.containerization.use_secrets.secret_task_1_task.yaml @@ -0,0 +1,55 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.containerization.use_secrets + - task-name + - secret_task + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.containerization.use_secrets.secret_task + resource_type: 1 + interface: + inputs: {} + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 3 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + security_context: + secrets: + - group: user-info + key: user_secret + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/013_core.containerization.use_secrets.user_info_task_1_task.json b/pkg/compiler/test/testdata/snacks-core/compiled/013_core.containerization.use_secrets.user_info_task_1_task.json new file mode 100755 index 0000000000..bbedc28c50 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/013_core.containerization.use_secrets.user_info_task_1_task.json @@ -0,0 +1 @@ +{"template":{"id":{"resourceType":"TASK","name":"core.containerization.use_secrets.user_info_task"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{},"outputs":{"variables":{"o0":{"type":{"simple":"STRING"},"description":"o0"},"o1":{"type":{"simple":"STRING"},"description":"o1"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.containerization.use_secrets","task-name","user_info_task"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]},"securityContext":{"secrets":[{"group":"user-info","key":"username"},{"group":"user-info","key":"password"}]}}} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/013_core.containerization.use_secrets.user_info_task_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/013_core.containerization.use_secrets.user_info_task_1_task.yaml new file mode 100755 index 0000000000..b341ea9aac --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/013_core.containerization.use_secrets.user_info_task_1_task.yaml @@ -0,0 +1,62 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.containerization.use_secrets + - task-name + - user_info_task + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.containerization.use_secrets.user_info_task + resource_type: 1 + interface: + inputs: {} + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 3 + o1: + description: o1 + type: + Type: + Simple: 3 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + security_context: + secrets: + - group: user-info + key: username + - group: user-info + key: password + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/014_core.containerization.use_secrets.secret_file_task_1_task.json b/pkg/compiler/test/testdata/snacks-core/compiled/014_core.containerization.use_secrets.secret_file_task_1_task.json new file mode 100755 index 0000000000..e6079debcf --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/014_core.containerization.use_secrets.secret_file_task_1_task.json @@ -0,0 +1 @@ +{"template":{"id":{"resourceType":"TASK","name":"core.containerization.use_secrets.secret_file_task"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{},"outputs":{"variables":{"o0":{"type":{"simple":"STRING"},"description":"o0"},"o1":{"type":{"simple":"STRING"},"description":"o1"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.containerization.use_secrets","task-name","secret_file_task"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]},"securityContext":{"secrets":[{"group":"user-info","key":"user_secret","mountRequirement":"ENV_VAR"}]}}} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/014_core.containerization.use_secrets.secret_file_task_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/014_core.containerization.use_secrets.secret_file_task_1_task.yaml new file mode 100755 index 0000000000..4a0fb267a3 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/014_core.containerization.use_secrets.secret_file_task_1_task.yaml @@ -0,0 +1,61 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.containerization.use_secrets + - task-name + - secret_file_task + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.containerization.use_secrets.secret_file_task + resource_type: 1 + interface: + inputs: {} + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 3 + o1: + description: o1 + type: + Type: + Simple: 3 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + security_context: + secrets: + - group: user-info + key: user_secret + mount_requirement: 1 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/015_core.containerization.use_secrets.my_secret_workflow_2_wf.json b/pkg/compiler/test/testdata/snacks-core/compiled/015_core.containerization.use_secrets.my_secret_workflow_2_wf.json new file mode 100755 index 0000000000..0bfcb31ac3 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/015_core.containerization.use_secrets.my_secret_workflow_2_wf.json @@ -0,0 +1 @@ +{"primary":{"template":{"id":{"resourceType":"WORKFLOW","name":"core.containerization.use_secrets.my_secret_workflow"},"metadata":{},"interface":{"inputs":{},"outputs":{"variables":{"o0":{"type":{"simple":"STRING"},"description":"o0"},"o1":{"type":{"simple":"STRING"},"description":"o1"},"o2":{"type":{"simple":"STRING"},"description":"o2"},"o3":{"type":{"simple":"STRING"},"description":"o3"},"o4":{"type":{"simple":"STRING"},"description":"o4"}}}},"nodes":[{"id":"start-node"},{"id":"end-node","inputs":[{"var":"o0","binding":{"promise":{"nodeId":"n0","var":"o0"}}},{"var":"o1","binding":{"promise":{"nodeId":"n1","var":"o0"}}},{"var":"o2","binding":{"promise":{"nodeId":"n1","var":"o1"}}},{"var":"o3","binding":{"promise":{"nodeId":"n2","var":"o0"}}},{"var":"o4","binding":{"promise":{"nodeId":"n2","var":"o1"}}}]},{"id":"n0","metadata":{"name":"secret_task","retries":{}},"taskNode":{"referenceId":{"resourceType":"TASK","name":"core.containerization.use_secrets.secret_task"},"overrides":{}}},{"id":"n1","metadata":{"name":"user_info_task","retries":{}},"taskNode":{"referenceId":{"resourceType":"TASK","name":"core.containerization.use_secrets.user_info_task"},"overrides":{}}},{"id":"n2","metadata":{"name":"secret_file_task","retries":{}},"taskNode":{"referenceId":{"resourceType":"TASK","name":"core.containerization.use_secrets.secret_file_task"},"overrides":{}}}],"outputs":[{"var":"o0","binding":{"promise":{"nodeId":"n0","var":"o0"}}},{"var":"o1","binding":{"promise":{"nodeId":"n1","var":"o0"}}},{"var":"o2","binding":{"promise":{"nodeId":"n1","var":"o1"}}},{"var":"o3","binding":{"promise":{"nodeId":"n2","var":"o0"}}},{"var":"o4","binding":{"promise":{"nodeId":"n2","var":"o1"}}}],"metadataDefaults":{}},"connections":{"downstream":{"n0":{"ids":["end-node"]},"n1":{"ids":["end-node"]},"n2":{"ids":["end-node"]},"start-node":{"ids":["n0","n1","n2"]}},"upstream":{"end-node":{"ids":["n0","n1","n2"]},"n0":{"ids":["start-node"]},"n1":{"ids":["start-node"]},"n2":{"ids":["start-node"]}}}},"tasks":[{"template":{"id":{"resourceType":"TASK","name":"core.containerization.use_secrets.secret_file_task"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{},"outputs":{"variables":{"o0":{"type":{"simple":"STRING"},"description":"o0"},"o1":{"type":{"simple":"STRING"},"description":"o1"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.containerization.use_secrets","task-name","secret_file_task"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]},"securityContext":{"secrets":[{"group":"user-info","key":"user_secret","mountRequirement":"ENV_VAR"}]}}},{"template":{"id":{"resourceType":"TASK","name":"core.containerization.use_secrets.secret_task"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{},"outputs":{"variables":{"o0":{"type":{"simple":"STRING"},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.containerization.use_secrets","task-name","secret_task"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]},"securityContext":{"secrets":[{"group":"user-info","key":"user_secret"}]}}},{"template":{"id":{"resourceType":"TASK","name":"core.containerization.use_secrets.user_info_task"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{},"outputs":{"variables":{"o0":{"type":{"simple":"STRING"},"description":"o0"},"o1":{"type":{"simple":"STRING"},"description":"o1"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.containerization.use_secrets","task-name","user_info_task"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]},"securityContext":{"secrets":[{"group":"user-info","key":"username"},{"group":"user-info","key":"password"}]}}}]} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/015_core.containerization.use_secrets.my_secret_workflow_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/015_core.containerization.use_secrets.my_secret_workflow_2_wf.yaml new file mode 100755 index 0000000000..65ecd563a2 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/015_core.containerization.use_secrets.my_secret_workflow_2_wf.yaml @@ -0,0 +1,349 @@ +primary: + connections: + downstream: + n0: + ids: + - end-node + n1: + ids: + - end-node + n2: + ids: + - end-node + start-node: + ids: + - n0 + - n1 + - n2 + upstream: + end-node: + ids: + - n0 + - n1 + - n2 + n0: + ids: + - start-node + n1: + ids: + - start-node + n2: + ids: + - start-node + template: + id: + name: core.containerization.use_secrets.my_secret_workflow + resource_type: 2 + interface: + inputs: {} + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 3 + o1: + description: o1 + type: + Type: + Simple: 3 + o2: + description: o2 + type: + Type: + Simple: 3 + o3: + description: o3 + type: + Type: + Simple: 3 + o4: + description: o4 + type: + Type: + Simple: 3 + metadata: {} + metadata_defaults: {} + nodes: + - Target: null + id: start-node + - Target: null + id: end-node + inputs: + - binding: + Value: + Promise: + node_id: n0 + var: o0 + var: o0 + - binding: + Value: + Promise: + node_id: n1 + var: o0 + var: o1 + - binding: + Value: + Promise: + node_id: n1 + var: o1 + var: o2 + - binding: + Value: + Promise: + node_id: n2 + var: o0 + var: o3 + - binding: + Value: + Promise: + node_id: n2 + var: o1 + var: o4 + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.containerization.use_secrets.secret_task + resource_type: 1 + overrides: {} + id: n0 + metadata: + InterruptibleValue: null + name: secret_task + retries: {} + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.containerization.use_secrets.user_info_task + resource_type: 1 + overrides: {} + id: n1 + metadata: + InterruptibleValue: null + name: user_info_task + retries: {} + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.containerization.use_secrets.secret_file_task + resource_type: 1 + overrides: {} + id: n2 + metadata: + InterruptibleValue: null + name: secret_file_task + retries: {} + outputs: + - binding: + Value: + Promise: + node_id: n0 + var: o0 + var: o0 + - binding: + Value: + Promise: + node_id: n1 + var: o0 + var: o1 + - binding: + Value: + Promise: + node_id: n1 + var: o1 + var: o2 + - binding: + Value: + Promise: + node_id: n2 + var: o0 + var: o3 + - binding: + Value: + Promise: + node_id: n2 + var: o1 + var: o4 +tasks: +- template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.containerization.use_secrets + - task-name + - secret_file_task + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.containerization.use_secrets.secret_file_task + resource_type: 1 + interface: + inputs: {} + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 3 + o1: + description: o1 + type: + Type: + Simple: 3 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + security_context: + secrets: + - group: user-info + key: user_secret + mount_requirement: 1 + type: python-task +- template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.containerization.use_secrets + - task-name + - secret_task + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.containerization.use_secrets.secret_task + resource_type: 1 + interface: + inputs: {} + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 3 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + security_context: + secrets: + - group: user-info + key: user_secret + type: python-task +- template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.containerization.use_secrets + - task-name + - user_info_task + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.containerization.use_secrets.user_info_task + resource_type: 1 + interface: + inputs: {} + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 3 + o1: + description: o1 + type: + Type: + Simple: 3 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + security_context: + secrets: + - group: user-info + key: username + - group: user-info + key: password + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/017_core.control_flow.chain_tasks.read_1_task.json b/pkg/compiler/test/testdata/snacks-core/compiled/017_core.control_flow.chain_tasks.read_1_task.json new file mode 100755 index 0000000000..c86790fea7 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/017_core.control_flow.chain_tasks.read_1_task.json @@ -0,0 +1 @@ +{"template":{"id":{"resourceType":"TASK","name":"core.control_flow.chain_tasks.read"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{},"outputs":{"variables":{"o0":{"type":{"structuredDatasetType":{"format":"parquet"}},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.control_flow.chain_tasks","task-name","read"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/017_core.control_flow.chain_tasks.read_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/017_core.control_flow.chain_tasks.read_1_task.yaml new file mode 100755 index 0000000000..06da97bcd4 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/017_core.control_flow.chain_tasks.read_1_task.yaml @@ -0,0 +1,52 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.control_flow.chain_tasks + - task-name + - read + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.control_flow.chain_tasks.read + resource_type: 1 + interface: + inputs: {} + outputs: + variables: + o0: + description: o0 + type: + Type: + StructuredDatasetType: + format: parquet + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/018_core.control_flow.chain_tasks.write_1_task.json b/pkg/compiler/test/testdata/snacks-core/compiled/018_core.control_flow.chain_tasks.write_1_task.json new file mode 100755 index 0000000000..6a3711b08b --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/018_core.control_flow.chain_tasks.write_1_task.json @@ -0,0 +1 @@ +{"template":{"id":{"resourceType":"TASK","name":"core.control_flow.chain_tasks.write"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{},"outputs":{}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.control_flow.chain_tasks","task-name","write"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/018_core.control_flow.chain_tasks.write_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/018_core.control_flow.chain_tasks.write_1_task.yaml new file mode 100755 index 0000000000..db095c5303 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/018_core.control_flow.chain_tasks.write_1_task.yaml @@ -0,0 +1,45 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.control_flow.chain_tasks + - task-name + - write + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.control_flow.chain_tasks.write + resource_type: 1 + interface: + inputs: {} + outputs: {} + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/019_core.control_flow.chain_tasks.chain_tasks_wf_2_wf.json b/pkg/compiler/test/testdata/snacks-core/compiled/019_core.control_flow.chain_tasks.chain_tasks_wf_2_wf.json new file mode 100755 index 0000000000..d92d169c15 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/019_core.control_flow.chain_tasks.chain_tasks_wf_2_wf.json @@ -0,0 +1 @@ +{"primary":{"template":{"id":{"resourceType":"WORKFLOW","name":"core.control_flow.chain_tasks.chain_tasks_wf"},"metadata":{},"interface":{"inputs":{},"outputs":{"variables":{"o0":{"type":{"structuredDatasetType":{"format":"parquet"}},"description":"o0"}}}},"nodes":[{"id":"start-node"},{"id":"end-node","inputs":[{"var":"o0","binding":{"promise":{"nodeId":"n1","var":"o0"}}}]},{"id":"n0","metadata":{"name":"write","retries":{}},"taskNode":{"referenceId":{"resourceType":"TASK","name":"core.control_flow.chain_tasks.write"},"overrides":{}}},{"id":"n1","metadata":{"name":"read","retries":{}},"upstreamNodeIds":["n0"],"taskNode":{"referenceId":{"resourceType":"TASK","name":"core.control_flow.chain_tasks.read"},"overrides":{}}}],"outputs":[{"var":"o0","binding":{"promise":{"nodeId":"n1","var":"o0"}}}],"metadataDefaults":{}},"connections":{"downstream":{"n0":{"ids":["n1"]},"n1":{"ids":["end-node"]},"start-node":{"ids":["n0"]}},"upstream":{"end-node":{"ids":["n1"]},"n0":{"ids":["start-node"]},"n1":{"ids":["n0"]}}}},"tasks":[{"template":{"id":{"resourceType":"TASK","name":"core.control_flow.chain_tasks.read"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{},"outputs":{"variables":{"o0":{"type":{"structuredDatasetType":{"format":"parquet"}},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.control_flow.chain_tasks","task-name","read"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},{"template":{"id":{"resourceType":"TASK","name":"core.control_flow.chain_tasks.write"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{},"outputs":{}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.control_flow.chain_tasks","task-name","write"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}}]} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/019_core.control_flow.chain_tasks.chain_tasks_wf_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/019_core.control_flow.chain_tasks.chain_tasks_wf_2_wf.yaml new file mode 100755 index 0000000000..58205b8d6c --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/019_core.control_flow.chain_tasks.chain_tasks_wf_2_wf.yaml @@ -0,0 +1,181 @@ +primary: + connections: + downstream: + n0: + ids: + - n1 + n1: + ids: + - end-node + start-node: + ids: + - n0 + upstream: + end-node: + ids: + - n1 + n0: + ids: + - start-node + n1: + ids: + - n0 + template: + id: + name: core.control_flow.chain_tasks.chain_tasks_wf + resource_type: 2 + interface: + inputs: {} + outputs: + variables: + o0: + description: o0 + type: + Type: + StructuredDatasetType: + format: parquet + metadata: {} + metadata_defaults: {} + nodes: + - Target: null + id: start-node + - Target: null + id: end-node + inputs: + - binding: + Value: + Promise: + node_id: n1 + var: o0 + var: o0 + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.control_flow.chain_tasks.write + resource_type: 1 + overrides: {} + id: n0 + metadata: + InterruptibleValue: null + name: write + retries: {} + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.control_flow.chain_tasks.read + resource_type: 1 + overrides: {} + id: n1 + metadata: + InterruptibleValue: null + name: read + retries: {} + upstream_node_ids: + - n0 + outputs: + - binding: + Value: + Promise: + node_id: n1 + var: o0 + var: o0 +tasks: +- template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.control_flow.chain_tasks + - task-name + - read + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.control_flow.chain_tasks.read + resource_type: 1 + interface: + inputs: {} + outputs: + variables: + o0: + description: o0 + type: + Type: + StructuredDatasetType: + format: parquet + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task +- template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.control_flow.chain_tasks + - task-name + - write + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.control_flow.chain_tasks.write + resource_type: 1 + interface: + inputs: {} + outputs: {} + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/021_core.control_flow.checkpoint.use_checkpoint_1_task.json b/pkg/compiler/test/testdata/snacks-core/compiled/021_core.control_flow.checkpoint.use_checkpoint_1_task.json new file mode 100755 index 0000000000..927ed1eebc --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/021_core.control_flow.checkpoint.use_checkpoint_1_task.json @@ -0,0 +1 @@ +{"template":{"id":{"resourceType":"TASK","name":"core.control_flow.checkpoint.use_checkpoint"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{"retries":3}},"interface":{"inputs":{"variables":{"n_iterations":{"type":{"simple":"INTEGER"},"description":"n_iterations"}}},"outputs":{"variables":{"o0":{"type":{"simple":"INTEGER"},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.control_flow.checkpoint","task-name","use_checkpoint"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/021_core.control_flow.checkpoint.use_checkpoint_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/021_core.control_flow.checkpoint.use_checkpoint_1_task.yaml new file mode 100755 index 0000000000..f08dc2d352 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/021_core.control_flow.checkpoint.use_checkpoint_1_task.yaml @@ -0,0 +1,58 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.control_flow.checkpoint + - task-name + - use_checkpoint + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.control_flow.checkpoint.use_checkpoint + resource_type: 1 + interface: + inputs: + variables: + n_iterations: + description: n_iterations + type: + Type: + Simple: 1 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 1 + metadata: + InterruptibleValue: null + retries: + retries: 3 + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/022_core.control_flow.checkpoint.example_2_wf.json b/pkg/compiler/test/testdata/snacks-core/compiled/022_core.control_flow.checkpoint.example_2_wf.json new file mode 100755 index 0000000000..a82eda7129 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/022_core.control_flow.checkpoint.example_2_wf.json @@ -0,0 +1 @@ +{"primary":{"template":{"id":{"resourceType":"WORKFLOW","name":"core.control_flow.checkpoint.example"},"metadata":{},"interface":{"inputs":{"variables":{"n_iterations":{"type":{"simple":"INTEGER"},"description":"n_iterations"}}},"outputs":{"variables":{"o0":{"type":{"simple":"INTEGER"},"description":"o0"}}}},"nodes":[{"id":"start-node"},{"id":"end-node","inputs":[{"var":"o0","binding":{"promise":{"nodeId":"n0","var":"o0"}}}]},{"id":"n0","metadata":{"name":"use_checkpoint","retries":{"retries":3}},"inputs":[{"var":"n_iterations","binding":{"promise":{"nodeId":"start-node","var":"n_iterations"}}}],"taskNode":{"referenceId":{"resourceType":"TASK","name":"core.control_flow.checkpoint.use_checkpoint"},"overrides":{}}}],"outputs":[{"var":"o0","binding":{"promise":{"nodeId":"n0","var":"o0"}}}],"metadataDefaults":{}},"connections":{"downstream":{"n0":{"ids":["end-node"]},"start-node":{"ids":["n0"]}},"upstream":{"end-node":{"ids":["n0"]},"n0":{"ids":["start-node"]}}}},"tasks":[{"template":{"id":{"resourceType":"TASK","name":"core.control_flow.checkpoint.use_checkpoint"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{"retries":3}},"interface":{"inputs":{"variables":{"n_iterations":{"type":{"simple":"INTEGER"},"description":"n_iterations"}}},"outputs":{"variables":{"o0":{"type":{"simple":"INTEGER"},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.control_flow.checkpoint","task-name","use_checkpoint"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}}]} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/022_core.control_flow.checkpoint.example_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/022_core.control_flow.checkpoint.example_2_wf.yaml new file mode 100755 index 0000000000..c5499fa617 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/022_core.control_flow.checkpoint.example_2_wf.yaml @@ -0,0 +1,135 @@ +primary: + connections: + downstream: + n0: + ids: + - end-node + start-node: + ids: + - n0 + upstream: + end-node: + ids: + - n0 + n0: + ids: + - start-node + template: + id: + name: core.control_flow.checkpoint.example + resource_type: 2 + interface: + inputs: + variables: + n_iterations: + description: n_iterations + type: + Type: + Simple: 1 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 1 + metadata: {} + metadata_defaults: {} + nodes: + - Target: null + id: start-node + - Target: null + id: end-node + inputs: + - binding: + Value: + Promise: + node_id: n0 + var: o0 + var: o0 + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.control_flow.checkpoint.use_checkpoint + resource_type: 1 + overrides: {} + id: n0 + inputs: + - binding: + Value: + Promise: + node_id: start-node + var: n_iterations + var: n_iterations + metadata: + InterruptibleValue: null + name: use_checkpoint + retries: + retries: 3 + outputs: + - binding: + Value: + Promise: + node_id: n0 + var: o0 + var: o0 +tasks: +- template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.control_flow.checkpoint + - task-name + - use_checkpoint + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.control_flow.checkpoint.use_checkpoint + resource_type: 1 + interface: + inputs: + variables: + n_iterations: + description: n_iterations + type: + Type: + Simple: 1 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 1 + metadata: + InterruptibleValue: null + retries: + retries: 3 + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/024_core.control_flow.conditions.square_1_task.json b/pkg/compiler/test/testdata/snacks-core/compiled/024_core.control_flow.conditions.square_1_task.json new file mode 100755 index 0000000000..cc91f92256 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/024_core.control_flow.conditions.square_1_task.json @@ -0,0 +1 @@ +{"template":{"id":{"resourceType":"TASK","name":"core.control_flow.conditions.square"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"n":{"type":{"simple":"FLOAT"},"description":"name of the parameter for the task is derived from the name of the input variable, and\nthe type is automatically mapped to Types.Integer"}}},"outputs":{"variables":{"o0":{"type":{"simple":"FLOAT"},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.control_flow.conditions","task-name","square"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/024_core.control_flow.conditions.square_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/024_core.control_flow.conditions.square_1_task.yaml new file mode 100755 index 0000000000..d53228e9f2 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/024_core.control_flow.conditions.square_1_task.yaml @@ -0,0 +1,59 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.control_flow.conditions + - task-name + - square + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.control_flow.conditions.square + resource_type: 1 + interface: + inputs: + variables: + "n": + description: |- + name of the parameter for the task is derived from the name of the input variable, and + the type is automatically mapped to Types.Integer + type: + Type: + Simple: 2 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 2 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/025_core.control_flow.conditions.double_1_task.json b/pkg/compiler/test/testdata/snacks-core/compiled/025_core.control_flow.conditions.double_1_task.json new file mode 100755 index 0000000000..2e84f4e067 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/025_core.control_flow.conditions.double_1_task.json @@ -0,0 +1 @@ +{"template":{"id":{"resourceType":"TASK","name":"core.control_flow.conditions.double"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"n":{"type":{"simple":"FLOAT"},"description":"name of the parameter for the task is derived from the name of the input variable\nand the type is mapped to ``Types.Integer``"}}},"outputs":{"variables":{"o0":{"type":{"simple":"FLOAT"},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.control_flow.conditions","task-name","double"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/025_core.control_flow.conditions.double_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/025_core.control_flow.conditions.double_1_task.yaml new file mode 100755 index 0000000000..c9567909a6 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/025_core.control_flow.conditions.double_1_task.yaml @@ -0,0 +1,59 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.control_flow.conditions + - task-name + - double + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.control_flow.conditions.double + resource_type: 1 + interface: + inputs: + variables: + "n": + description: |- + name of the parameter for the task is derived from the name of the input variable + and the type is mapped to ``Types.Integer`` + type: + Type: + Simple: 2 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 2 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/026_core.control_flow.conditions.multiplier_2_wf.json b/pkg/compiler/test/testdata/snacks-core/compiled/026_core.control_flow.conditions.multiplier_2_wf.json new file mode 100755 index 0000000000..67165cc1c4 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/026_core.control_flow.conditions.multiplier_2_wf.json @@ -0,0 +1 @@ +{"primary":{"template":{"id":{"resourceType":"WORKFLOW","name":"core.control_flow.conditions.multiplier"},"metadata":{},"interface":{"inputs":{"variables":{"my_input":{"type":{"simple":"FLOAT"},"description":"my_input"}}},"outputs":{"variables":{"o0":{"type":{"simple":"FLOAT"},"description":"o0"}}}},"nodes":[{"id":"start-node"},{"id":"end-node","inputs":[{"var":"o0","binding":{"promise":{"nodeId":"n0","var":"o0"}}}]},{"id":"n0","metadata":{"name":"fractions","retries":{}},"inputs":[{"var":".my_input","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"branchNode":{"ifElse":{"case":{"condition":{"conjunction":{"leftExpression":{"comparison":{"operator":"GTE","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":0.1}}}},"rightExpression":{"comparison":{"operator":"LTE","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":1}}}}}},"thenNode":{"id":"n0-n0","metadata":{"name":"double","retries":{}},"inputs":[{"var":"n","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"taskNode":{"referenceId":{"resourceType":"TASK","name":"core.control_flow.conditions.double"},"overrides":{}}}},"elseNode":{"id":"n0-n1","metadata":{"name":"square","retries":{}},"inputs":[{"var":"n","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"taskNode":{"referenceId":{"resourceType":"TASK","name":"core.control_flow.conditions.square"},"overrides":{}}}}}}],"outputs":[{"var":"o0","binding":{"promise":{"nodeId":"n0","var":"o0"}}}],"metadataDefaults":{}},"connections":{"downstream":{"n0":{"ids":["end-node"]},"start-node":{"ids":["n0"]}},"upstream":{"end-node":{"ids":["n0"]},"n0":{"ids":["start-node"]},"n0-n0":{"ids":["start-node"]},"n0-n1":{"ids":["start-node"]}}}},"tasks":[{"template":{"id":{"resourceType":"TASK","name":"core.control_flow.conditions.double"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"n":{"type":{"simple":"FLOAT"},"description":"name of the parameter for the task is derived from the name of the input variable\nand the type is mapped to ``Types.Integer``"}}},"outputs":{"variables":{"o0":{"type":{"simple":"FLOAT"},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.control_flow.conditions","task-name","double"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},{"template":{"id":{"resourceType":"TASK","name":"core.control_flow.conditions.square"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"n":{"type":{"simple":"FLOAT"},"description":"name of the parameter for the task is derived from the name of the input variable, and\nthe type is automatically mapped to Types.Integer"}}},"outputs":{"variables":{"o0":{"type":{"simple":"FLOAT"},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.control_flow.conditions","task-name","square"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}}]} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/026_core.control_flow.conditions.multiplier_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/026_core.control_flow.conditions.multiplier_2_wf.yaml new file mode 100755 index 0000000000..035ad43712 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/026_core.control_flow.conditions.multiplier_2_wf.yaml @@ -0,0 +1,265 @@ +primary: + connections: + downstream: + n0: + ids: + - end-node + start-node: + ids: + - n0 + upstream: + end-node: + ids: + - n0 + n0: + ids: + - start-node + n0-n0: + ids: + - start-node + n0-n1: + ids: + - start-node + template: + id: + name: core.control_flow.conditions.multiplier + resource_type: 2 + interface: + inputs: + variables: + my_input: + description: my_input + type: + Type: + Simple: 2 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 2 + metadata: {} + metadata_defaults: {} + nodes: + - Target: null + id: start-node + - Target: null + id: end-node + inputs: + - binding: + Value: + Promise: + node_id: n0 + var: o0 + var: o0 + - Target: + BranchNode: + if_else: + Default: + ElseNode: + Target: + TaskNode: + Reference: + ReferenceId: + name: core.control_flow.conditions.square + resource_type: 1 + overrides: {} + id: n0-n1 + inputs: + - binding: + Value: + Promise: + node_id: start-node + var: my_input + var: "n" + metadata: + InterruptibleValue: null + name: square + retries: {} + case: + condition: + Expr: + Conjunction: + left_expression: + Expr: + Comparison: + left_value: + Val: + Var: .my_input + operator: 3 + right_value: + Val: + Primitive: + Value: + FloatValue: 0.1 + right_expression: + Expr: + Comparison: + left_value: + Val: + Var: .my_input + operator: 5 + right_value: + Val: + Primitive: + Value: + FloatValue: 1 + then_node: + Target: + TaskNode: + Reference: + ReferenceId: + name: core.control_flow.conditions.double + resource_type: 1 + overrides: {} + id: n0-n0 + inputs: + - binding: + Value: + Promise: + node_id: start-node + var: my_input + var: "n" + metadata: + InterruptibleValue: null + name: double + retries: {} + id: n0 + inputs: + - binding: + Value: + Promise: + node_id: start-node + var: my_input + var: .my_input + metadata: + InterruptibleValue: null + name: fractions + retries: {} + outputs: + - binding: + Value: + Promise: + node_id: n0 + var: o0 + var: o0 +tasks: +- template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.control_flow.conditions + - task-name + - double + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.control_flow.conditions.double + resource_type: 1 + interface: + inputs: + variables: + "n": + description: |- + name of the parameter for the task is derived from the name of the input variable + and the type is mapped to ``Types.Integer`` + type: + Type: + Simple: 2 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 2 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task +- template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.control_flow.conditions + - task-name + - square + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.control_flow.conditions.square + resource_type: 1 + interface: + inputs: + variables: + "n": + description: |- + name of the parameter for the task is derived from the name of the input variable, and + the type is automatically mapped to Types.Integer + type: + Type: + Simple: 2 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 2 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/028_core.control_flow.conditions.multiplier_2_2_wf.json b/pkg/compiler/test/testdata/snacks-core/compiled/028_core.control_flow.conditions.multiplier_2_2_wf.json new file mode 100755 index 0000000000..b7ad917433 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/028_core.control_flow.conditions.multiplier_2_2_wf.json @@ -0,0 +1 @@ +{"primary":{"template":{"id":{"resourceType":"WORKFLOW","name":"core.control_flow.conditions.multiplier_2"},"metadata":{},"interface":{"inputs":{"variables":{"my_input":{"type":{"simple":"FLOAT"},"description":"my_input"}}},"outputs":{"variables":{"o0":{"type":{"simple":"FLOAT"},"description":"o0"}}}},"nodes":[{"id":"start-node"},{"id":"end-node","inputs":[{"var":"o0","binding":{"promise":{"nodeId":"n0","var":"o0"}}}]},{"id":"n0","metadata":{"name":"fractions","retries":{}},"inputs":[{"var":".my_input","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"branchNode":{"ifElse":{"case":{"condition":{"conjunction":{"leftExpression":{"comparison":{"operator":"GT","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":0.1}}}},"rightExpression":{"comparison":{"operator":"LT","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":1}}}}}},"thenNode":{"id":"n0-n0","metadata":{"name":"double","retries":{}},"inputs":[{"var":"n","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"taskNode":{"referenceId":{"resourceType":"TASK","name":"core.control_flow.conditions.double"},"overrides":{}}}},"other":[{"condition":{"conjunction":{"leftExpression":{"comparison":{"operator":"GT","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":1}}}},"rightExpression":{"comparison":{"operator":"LTE","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":10}}}}}},"thenNode":{"id":"n0-n1","metadata":{"name":"square","retries":{}},"inputs":[{"var":"n","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"taskNode":{"referenceId":{"resourceType":"TASK","name":"core.control_flow.conditions.square"},"overrides":{}}}}],"error":{"failedNodeId":"fractions","message":"The input must be between 0 and 10"}}}}],"outputs":[{"var":"o0","binding":{"promise":{"nodeId":"n0","var":"o0"}}}],"metadataDefaults":{}},"connections":{"downstream":{"n0":{"ids":["end-node"]},"start-node":{"ids":["n0"]}},"upstream":{"end-node":{"ids":["n0"]},"n0":{"ids":["start-node"]},"n0-n0":{"ids":["start-node"]},"n0-n1":{"ids":["start-node"]}}}},"tasks":[{"template":{"id":{"resourceType":"TASK","name":"core.control_flow.conditions.double"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"n":{"type":{"simple":"FLOAT"},"description":"name of the parameter for the task is derived from the name of the input variable\nand the type is mapped to ``Types.Integer``"}}},"outputs":{"variables":{"o0":{"type":{"simple":"FLOAT"},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.control_flow.conditions","task-name","double"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},{"template":{"id":{"resourceType":"TASK","name":"core.control_flow.conditions.square"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"n":{"type":{"simple":"FLOAT"},"description":"name of the parameter for the task is derived from the name of the input variable, and\nthe type is automatically mapped to Types.Integer"}}},"outputs":{"variables":{"o0":{"type":{"simple":"FLOAT"},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.control_flow.conditions","task-name","square"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}}]} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/028_core.control_flow.conditions.multiplier_2_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/028_core.control_flow.conditions.multiplier_2_2_wf.yaml new file mode 100755 index 0000000000..95759eaa41 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/028_core.control_flow.conditions.multiplier_2_2_wf.yaml @@ -0,0 +1,296 @@ +primary: + connections: + downstream: + n0: + ids: + - end-node + start-node: + ids: + - n0 + upstream: + end-node: + ids: + - n0 + n0: + ids: + - start-node + n0-n0: + ids: + - start-node + n0-n1: + ids: + - start-node + template: + id: + name: core.control_flow.conditions.multiplier_2 + resource_type: 2 + interface: + inputs: + variables: + my_input: + description: my_input + type: + Type: + Simple: 2 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 2 + metadata: {} + metadata_defaults: {} + nodes: + - Target: null + id: start-node + - Target: null + id: end-node + inputs: + - binding: + Value: + Promise: + node_id: n0 + var: o0 + var: o0 + - Target: + BranchNode: + if_else: + Default: + Error: + failed_node_id: fractions + message: The input must be between 0 and 10 + case: + condition: + Expr: + Conjunction: + left_expression: + Expr: + Comparison: + left_value: + Val: + Var: .my_input + operator: 2 + right_value: + Val: + Primitive: + Value: + FloatValue: 0.1 + right_expression: + Expr: + Comparison: + left_value: + Val: + Var: .my_input + operator: 4 + right_value: + Val: + Primitive: + Value: + FloatValue: 1 + then_node: + Target: + TaskNode: + Reference: + ReferenceId: + name: core.control_flow.conditions.double + resource_type: 1 + overrides: {} + id: n0-n0 + inputs: + - binding: + Value: + Promise: + node_id: start-node + var: my_input + var: "n" + metadata: + InterruptibleValue: null + name: double + retries: {} + other: + - condition: + Expr: + Conjunction: + left_expression: + Expr: + Comparison: + left_value: + Val: + Var: .my_input + operator: 2 + right_value: + Val: + Primitive: + Value: + FloatValue: 1 + right_expression: + Expr: + Comparison: + left_value: + Val: + Var: .my_input + operator: 5 + right_value: + Val: + Primitive: + Value: + FloatValue: 10 + then_node: + Target: + TaskNode: + Reference: + ReferenceId: + name: core.control_flow.conditions.square + resource_type: 1 + overrides: {} + id: n0-n1 + inputs: + - binding: + Value: + Promise: + node_id: start-node + var: my_input + var: "n" + metadata: + InterruptibleValue: null + name: square + retries: {} + id: n0 + inputs: + - binding: + Value: + Promise: + node_id: start-node + var: my_input + var: .my_input + metadata: + InterruptibleValue: null + name: fractions + retries: {} + outputs: + - binding: + Value: + Promise: + node_id: n0 + var: o0 + var: o0 +tasks: +- template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.control_flow.conditions + - task-name + - double + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.control_flow.conditions.double + resource_type: 1 + interface: + inputs: + variables: + "n": + description: |- + name of the parameter for the task is derived from the name of the input variable + and the type is mapped to ``Types.Integer`` + type: + Type: + Simple: 2 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 2 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task +- template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.control_flow.conditions + - task-name + - square + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.control_flow.conditions.square + resource_type: 1 + interface: + inputs: + variables: + "n": + description: |- + name of the parameter for the task is derived from the name of the input variable, and + the type is automatically mapped to Types.Integer + type: + Type: + Simple: 2 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 2 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/030_core.control_flow.conditions.multiplier_3_2_wf.json b/pkg/compiler/test/testdata/snacks-core/compiled/030_core.control_flow.conditions.multiplier_3_2_wf.json new file mode 100755 index 0000000000..831689dee0 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/030_core.control_flow.conditions.multiplier_3_2_wf.json @@ -0,0 +1 @@ +{"primary":{"template":{"id":{"resourceType":"WORKFLOW","name":"core.control_flow.conditions.multiplier_3"},"metadata":{},"interface":{"inputs":{"variables":{"my_input":{"type":{"simple":"FLOAT"},"description":"my_input"}}},"outputs":{"variables":{"o0":{"type":{"simple":"FLOAT"},"description":"o0"}}}},"nodes":[{"id":"start-node"},{"id":"end-node","inputs":[{"var":"o0","binding":{"promise":{"nodeId":"n1","var":"o0"}}}]},{"id":"n0","metadata":{"name":"fractions","retries":{}},"inputs":[{"var":".my_input","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"branchNode":{"ifElse":{"case":{"condition":{"conjunction":{"leftExpression":{"comparison":{"operator":"GT","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":0.1}}}},"rightExpression":{"comparison":{"operator":"LT","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":1}}}}}},"thenNode":{"id":"n0-n0","metadata":{"name":"double","retries":{}},"inputs":[{"var":"n","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"taskNode":{"referenceId":{"resourceType":"TASK","name":"core.control_flow.conditions.double"},"overrides":{}}}},"other":[{"condition":{"conjunction":{"leftExpression":{"comparison":{"operator":"GT","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":1}}}},"rightExpression":{"comparison":{"operator":"LT","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":10}}}}}},"thenNode":{"id":"n0-n1","metadata":{"name":"square","retries":{}},"inputs":[{"var":"n","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"taskNode":{"referenceId":{"resourceType":"TASK","name":"core.control_flow.conditions.square"},"overrides":{}}}}],"error":{"failedNodeId":"fractions","message":"The input must be between 0 and 10"}}}},{"id":"n1","metadata":{"name":"double","retries":{}},"inputs":[{"var":"n","binding":{"promise":{"nodeId":"n0","var":"o0"}}}],"upstreamNodeIds":["n0"],"taskNode":{"referenceId":{"resourceType":"TASK","name":"core.control_flow.conditions.double"},"overrides":{}}}],"outputs":[{"var":"o0","binding":{"promise":{"nodeId":"n1","var":"o0"}}}],"metadataDefaults":{}},"connections":{"downstream":{"n0":{"ids":["n1"]},"n1":{"ids":["end-node"]},"start-node":{"ids":["n0"]}},"upstream":{"end-node":{"ids":["n1"]},"n0":{"ids":["start-node"]},"n0-n0":{"ids":["start-node"]},"n0-n1":{"ids":["start-node"]},"n1":{"ids":["n0"]}}}},"tasks":[{"template":{"id":{"resourceType":"TASK","name":"core.control_flow.conditions.double"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"n":{"type":{"simple":"FLOAT"},"description":"name of the parameter for the task is derived from the name of the input variable\nand the type is mapped to ``Types.Integer``"}}},"outputs":{"variables":{"o0":{"type":{"simple":"FLOAT"},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.control_flow.conditions","task-name","double"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},{"template":{"id":{"resourceType":"TASK","name":"core.control_flow.conditions.square"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"n":{"type":{"simple":"FLOAT"},"description":"name of the parameter for the task is derived from the name of the input variable, and\nthe type is automatically mapped to Types.Integer"}}},"outputs":{"variables":{"o0":{"type":{"simple":"FLOAT"},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.control_flow.conditions","task-name","square"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}}]} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/030_core.control_flow.conditions.multiplier_3_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/030_core.control_flow.conditions.multiplier_3_2_wf.yaml new file mode 100755 index 0000000000..fbe3fe679a --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/030_core.control_flow.conditions.multiplier_3_2_wf.yaml @@ -0,0 +1,323 @@ +primary: + connections: + downstream: + n0: + ids: + - n1 + n1: + ids: + - end-node + start-node: + ids: + - n0 + upstream: + end-node: + ids: + - n1 + n0: + ids: + - start-node + n0-n0: + ids: + - start-node + n0-n1: + ids: + - start-node + n1: + ids: + - n0 + template: + id: + name: core.control_flow.conditions.multiplier_3 + resource_type: 2 + interface: + inputs: + variables: + my_input: + description: my_input + type: + Type: + Simple: 2 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 2 + metadata: {} + metadata_defaults: {} + nodes: + - Target: null + id: start-node + - Target: null + id: end-node + inputs: + - binding: + Value: + Promise: + node_id: n1 + var: o0 + var: o0 + - Target: + BranchNode: + if_else: + Default: + Error: + failed_node_id: fractions + message: The input must be between 0 and 10 + case: + condition: + Expr: + Conjunction: + left_expression: + Expr: + Comparison: + left_value: + Val: + Var: .my_input + operator: 2 + right_value: + Val: + Primitive: + Value: + FloatValue: 0.1 + right_expression: + Expr: + Comparison: + left_value: + Val: + Var: .my_input + operator: 4 + right_value: + Val: + Primitive: + Value: + FloatValue: 1 + then_node: + Target: + TaskNode: + Reference: + ReferenceId: + name: core.control_flow.conditions.double + resource_type: 1 + overrides: {} + id: n0-n0 + inputs: + - binding: + Value: + Promise: + node_id: start-node + var: my_input + var: "n" + metadata: + InterruptibleValue: null + name: double + retries: {} + other: + - condition: + Expr: + Conjunction: + left_expression: + Expr: + Comparison: + left_value: + Val: + Var: .my_input + operator: 2 + right_value: + Val: + Primitive: + Value: + FloatValue: 1 + right_expression: + Expr: + Comparison: + left_value: + Val: + Var: .my_input + operator: 4 + right_value: + Val: + Primitive: + Value: + FloatValue: 10 + then_node: + Target: + TaskNode: + Reference: + ReferenceId: + name: core.control_flow.conditions.square + resource_type: 1 + overrides: {} + id: n0-n1 + inputs: + - binding: + Value: + Promise: + node_id: start-node + var: my_input + var: "n" + metadata: + InterruptibleValue: null + name: square + retries: {} + id: n0 + inputs: + - binding: + Value: + Promise: + node_id: start-node + var: my_input + var: .my_input + metadata: + InterruptibleValue: null + name: fractions + retries: {} + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.control_flow.conditions.double + resource_type: 1 + overrides: {} + id: n1 + inputs: + - binding: + Value: + Promise: + node_id: n0 + var: o0 + var: "n" + metadata: + InterruptibleValue: null + name: double + retries: {} + upstream_node_ids: + - n0 + outputs: + - binding: + Value: + Promise: + node_id: n1 + var: o0 + var: o0 +tasks: +- template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.control_flow.conditions + - task-name + - double + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.control_flow.conditions.double + resource_type: 1 + interface: + inputs: + variables: + "n": + description: |- + name of the parameter for the task is derived from the name of the input variable + and the type is mapped to ``Types.Integer`` + type: + Type: + Simple: 2 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 2 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task +- template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.control_flow.conditions + - task-name + - square + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.control_flow.conditions.square + resource_type: 1 + interface: + inputs: + variables: + "n": + description: |- + name of the parameter for the task is derived from the name of the input variable, and + the type is automatically mapped to Types.Integer + type: + Type: + Simple: 2 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 2 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/032_core.control_flow.conditions.coin_toss_1_task.json b/pkg/compiler/test/testdata/snacks-core/compiled/032_core.control_flow.conditions.coin_toss_1_task.json new file mode 100755 index 0000000000..1889b13bbd --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/032_core.control_flow.conditions.coin_toss_1_task.json @@ -0,0 +1 @@ +{"template":{"id":{"resourceType":"TASK","name":"core.control_flow.conditions.coin_toss"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"seed":{"type":{"simple":"INTEGER"},"description":"seed"}}},"outputs":{"variables":{"o0":{"type":{"simple":"BOOLEAN"},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.control_flow.conditions","task-name","coin_toss"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/032_core.control_flow.conditions.coin_toss_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/032_core.control_flow.conditions.coin_toss_1_task.yaml new file mode 100755 index 0000000000..8870ff8d73 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/032_core.control_flow.conditions.coin_toss_1_task.yaml @@ -0,0 +1,57 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.control_flow.conditions + - task-name + - coin_toss + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.control_flow.conditions.coin_toss + resource_type: 1 + interface: + inputs: + variables: + seed: + description: seed + type: + Type: + Simple: 1 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 4 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/033_core.control_flow.conditions.failed_1_task.json b/pkg/compiler/test/testdata/snacks-core/compiled/033_core.control_flow.conditions.failed_1_task.json new file mode 100755 index 0000000000..04f8995434 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/033_core.control_flow.conditions.failed_1_task.json @@ -0,0 +1 @@ +{"template":{"id":{"resourceType":"TASK","name":"core.control_flow.conditions.failed"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{},"outputs":{"variables":{"o0":{"type":{"simple":"INTEGER"},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.control_flow.conditions","task-name","failed"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/033_core.control_flow.conditions.failed_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/033_core.control_flow.conditions.failed_1_task.yaml new file mode 100755 index 0000000000..5e0aa7d5d7 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/033_core.control_flow.conditions.failed_1_task.yaml @@ -0,0 +1,51 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.control_flow.conditions + - task-name + - failed + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.control_flow.conditions.failed + resource_type: 1 + interface: + inputs: {} + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 1 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/034_core.control_flow.conditions.success_1_task.json b/pkg/compiler/test/testdata/snacks-core/compiled/034_core.control_flow.conditions.success_1_task.json new file mode 100755 index 0000000000..e223be296a --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/034_core.control_flow.conditions.success_1_task.json @@ -0,0 +1 @@ +{"template":{"id":{"resourceType":"TASK","name":"core.control_flow.conditions.success"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{},"outputs":{"variables":{"o0":{"type":{"simple":"INTEGER"},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.control_flow.conditions","task-name","success"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/034_core.control_flow.conditions.success_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/034_core.control_flow.conditions.success_1_task.yaml new file mode 100755 index 0000000000..bc6b50e4fc --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/034_core.control_flow.conditions.success_1_task.yaml @@ -0,0 +1,51 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.control_flow.conditions + - task-name + - success + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.control_flow.conditions.success + resource_type: 1 + interface: + inputs: {} + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 1 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/035_core.control_flow.conditions.basic_boolean_wf_2_wf.json b/pkg/compiler/test/testdata/snacks-core/compiled/035_core.control_flow.conditions.basic_boolean_wf_2_wf.json new file mode 100755 index 0000000000..a50902bfbb --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/035_core.control_flow.conditions.basic_boolean_wf_2_wf.json @@ -0,0 +1 @@ +{"primary":{"template":{"id":{"resourceType":"WORKFLOW","name":"core.control_flow.conditions.basic_boolean_wf"},"metadata":{},"interface":{"inputs":{"variables":{"seed":{"type":{"simple":"INTEGER"},"description":"seed"}}},"outputs":{"variables":{"o0":{"type":{"simple":"INTEGER"},"description":"o0"}}}},"nodes":[{"id":"start-node"},{"id":"end-node","inputs":[{"var":"o0","binding":{"promise":{"nodeId":"n1","var":"o0"}}}]},{"id":"n0","metadata":{"name":"coin_toss","retries":{}},"inputs":[{"var":"seed","binding":{"promise":{"nodeId":"start-node","var":"seed"}}}],"taskNode":{"referenceId":{"resourceType":"TASK","name":"core.control_flow.conditions.coin_toss"},"overrides":{}}},{"id":"n1","metadata":{"name":"test","retries":{}},"inputs":[{"var":"n0.o0","binding":{"promise":{"nodeId":"n0","var":"o0"}}}],"upstreamNodeIds":["n0"],"branchNode":{"ifElse":{"case":{"condition":{"comparison":{"leftValue":{"var":"n0.o0"},"rightValue":{"primitive":{"boolean":true}}}},"thenNode":{"id":"n1-n0","metadata":{"name":"success","retries":{}},"taskNode":{"referenceId":{"resourceType":"TASK","name":"core.control_flow.conditions.success"},"overrides":{}}}},"elseNode":{"id":"n1-n1","metadata":{"name":"failed","retries":{}},"taskNode":{"referenceId":{"resourceType":"TASK","name":"core.control_flow.conditions.failed"},"overrides":{}}}}}}],"outputs":[{"var":"o0","binding":{"promise":{"nodeId":"n1","var":"o0"}}}],"metadataDefaults":{}},"connections":{"downstream":{"n0":{"ids":["n1"]},"n1":{"ids":["end-node"]},"start-node":{"ids":["n0"]}},"upstream":{"end-node":{"ids":["n1"]},"n0":{"ids":["start-node"]},"n1":{"ids":["n0"]}}}},"tasks":[{"template":{"id":{"resourceType":"TASK","name":"core.control_flow.conditions.coin_toss"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"seed":{"type":{"simple":"INTEGER"},"description":"seed"}}},"outputs":{"variables":{"o0":{"type":{"simple":"BOOLEAN"},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.control_flow.conditions","task-name","coin_toss"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},{"template":{"id":{"resourceType":"TASK","name":"core.control_flow.conditions.failed"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{},"outputs":{"variables":{"o0":{"type":{"simple":"INTEGER"},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.control_flow.conditions","task-name","failed"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},{"template":{"id":{"resourceType":"TASK","name":"core.control_flow.conditions.success"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{},"outputs":{"variables":{"o0":{"type":{"simple":"INTEGER"},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.control_flow.conditions","task-name","success"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}}]} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/035_core.control_flow.conditions.basic_boolean_wf_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/035_core.control_flow.conditions.basic_boolean_wf_2_wf.yaml new file mode 100755 index 0000000000..e1774834c9 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/035_core.control_flow.conditions.basic_boolean_wf_2_wf.yaml @@ -0,0 +1,297 @@ +primary: + connections: + downstream: + n0: + ids: + - n1 + n1: + ids: + - end-node + start-node: + ids: + - n0 + upstream: + end-node: + ids: + - n1 + n0: + ids: + - start-node + n1: + ids: + - n0 + template: + id: + name: core.control_flow.conditions.basic_boolean_wf + resource_type: 2 + interface: + inputs: + variables: + seed: + description: seed + type: + Type: + Simple: 1 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 1 + metadata: {} + metadata_defaults: {} + nodes: + - Target: null + id: start-node + - Target: null + id: end-node + inputs: + - binding: + Value: + Promise: + node_id: n1 + var: o0 + var: o0 + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.control_flow.conditions.coin_toss + resource_type: 1 + overrides: {} + id: n0 + inputs: + - binding: + Value: + Promise: + node_id: start-node + var: seed + var: seed + metadata: + InterruptibleValue: null + name: coin_toss + retries: {} + - Target: + BranchNode: + if_else: + Default: + ElseNode: + Target: + TaskNode: + Reference: + ReferenceId: + name: core.control_flow.conditions.failed + resource_type: 1 + overrides: {} + id: n1-n1 + metadata: + InterruptibleValue: null + name: failed + retries: {} + case: + condition: + Expr: + Comparison: + left_value: + Val: + Var: n0.o0 + right_value: + Val: + Primitive: + Value: + Boolean: true + then_node: + Target: + TaskNode: + Reference: + ReferenceId: + name: core.control_flow.conditions.success + resource_type: 1 + overrides: {} + id: n1-n0 + metadata: + InterruptibleValue: null + name: success + retries: {} + id: n1 + inputs: + - binding: + Value: + Promise: + node_id: n0 + var: o0 + var: n0.o0 + metadata: + InterruptibleValue: null + name: test + retries: {} + upstream_node_ids: + - n0 + outputs: + - binding: + Value: + Promise: + node_id: n1 + var: o0 + var: o0 +tasks: +- template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.control_flow.conditions + - task-name + - coin_toss + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.control_flow.conditions.coin_toss + resource_type: 1 + interface: + inputs: + variables: + seed: + description: seed + type: + Type: + Simple: 1 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 4 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task +- template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.control_flow.conditions + - task-name + - failed + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.control_flow.conditions.failed + resource_type: 1 + interface: + inputs: {} + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 1 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task +- template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.control_flow.conditions + - task-name + - success + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.control_flow.conditions.success + resource_type: 1 + interface: + inputs: {} + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 1 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/037_core.control_flow.conditions.bool_input_wf_2_wf.json b/pkg/compiler/test/testdata/snacks-core/compiled/037_core.control_flow.conditions.bool_input_wf_2_wf.json new file mode 100755 index 0000000000..0c11f49798 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/037_core.control_flow.conditions.bool_input_wf_2_wf.json @@ -0,0 +1 @@ +{"primary":{"template":{"id":{"resourceType":"WORKFLOW","name":"core.control_flow.conditions.bool_input_wf"},"metadata":{},"interface":{"inputs":{"variables":{"b":{"type":{"simple":"BOOLEAN"},"description":"b"}}},"outputs":{"variables":{"o0":{"type":{"simple":"INTEGER"},"description":"o0"}}}},"nodes":[{"id":"start-node"},{"id":"end-node","inputs":[{"var":"o0","binding":{"promise":{"nodeId":"n0","var":"o0"}}}]},{"id":"n0","metadata":{"name":"test","retries":{}},"inputs":[{"var":".b","binding":{"promise":{"nodeId":"start-node","var":"b"}}}],"branchNode":{"ifElse":{"case":{"condition":{"comparison":{"leftValue":{"var":".b"},"rightValue":{"primitive":{"boolean":true}}}},"thenNode":{"id":"n0-n0","metadata":{"name":"success","retries":{}},"taskNode":{"referenceId":{"resourceType":"TASK","name":"core.control_flow.conditions.success"},"overrides":{}}}},"elseNode":{"id":"n0-n1","metadata":{"name":"failed","retries":{}},"taskNode":{"referenceId":{"resourceType":"TASK","name":"core.control_flow.conditions.failed"},"overrides":{}}}}}}],"outputs":[{"var":"o0","binding":{"promise":{"nodeId":"n0","var":"o0"}}}],"metadataDefaults":{}},"connections":{"downstream":{"n0":{"ids":["end-node"]},"start-node":{"ids":["n0"]}},"upstream":{"end-node":{"ids":["n0"]},"n0":{"ids":["start-node"]}}}},"tasks":[{"template":{"id":{"resourceType":"TASK","name":"core.control_flow.conditions.failed"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{},"outputs":{"variables":{"o0":{"type":{"simple":"INTEGER"},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.control_flow.conditions","task-name","failed"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},{"template":{"id":{"resourceType":"TASK","name":"core.control_flow.conditions.success"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{},"outputs":{"variables":{"o0":{"type":{"simple":"INTEGER"},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.control_flow.conditions","task-name","success"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}}]} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/037_core.control_flow.conditions.bool_input_wf_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/037_core.control_flow.conditions.bool_input_wf_2_wf.yaml new file mode 100755 index 0000000000..949321dda2 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/037_core.control_flow.conditions.bool_input_wf_2_wf.yaml @@ -0,0 +1,213 @@ +primary: + connections: + downstream: + n0: + ids: + - end-node + start-node: + ids: + - n0 + upstream: + end-node: + ids: + - n0 + n0: + ids: + - start-node + template: + id: + name: core.control_flow.conditions.bool_input_wf + resource_type: 2 + interface: + inputs: + variables: + b: + description: b + type: + Type: + Simple: 4 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 1 + metadata: {} + metadata_defaults: {} + nodes: + - Target: null + id: start-node + - Target: null + id: end-node + inputs: + - binding: + Value: + Promise: + node_id: n0 + var: o0 + var: o0 + - Target: + BranchNode: + if_else: + Default: + ElseNode: + Target: + TaskNode: + Reference: + ReferenceId: + name: core.control_flow.conditions.failed + resource_type: 1 + overrides: {} + id: n0-n1 + metadata: + InterruptibleValue: null + name: failed + retries: {} + case: + condition: + Expr: + Comparison: + left_value: + Val: + Var: .b + right_value: + Val: + Primitive: + Value: + Boolean: true + then_node: + Target: + TaskNode: + Reference: + ReferenceId: + name: core.control_flow.conditions.success + resource_type: 1 + overrides: {} + id: n0-n0 + metadata: + InterruptibleValue: null + name: success + retries: {} + id: n0 + inputs: + - binding: + Value: + Promise: + node_id: start-node + var: b + var: .b + metadata: + InterruptibleValue: null + name: test + retries: {} + outputs: + - binding: + Value: + Promise: + node_id: n0 + var: o0 + var: o0 +tasks: +- template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.control_flow.conditions + - task-name + - failed + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.control_flow.conditions.failed + resource_type: 1 + interface: + inputs: {} + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 1 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task +- template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.control_flow.conditions + - task-name + - success + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.control_flow.conditions.success + resource_type: 1 + interface: + inputs: {} + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 1 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/039_core.control_flow.conditions.nested_conditions_2_wf.json b/pkg/compiler/test/testdata/snacks-core/compiled/039_core.control_flow.conditions.nested_conditions_2_wf.json new file mode 100755 index 0000000000..a3d53ddfa8 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/039_core.control_flow.conditions.nested_conditions_2_wf.json @@ -0,0 +1 @@ +{"primary":{"template":{"id":{"resourceType":"WORKFLOW","name":"core.control_flow.conditions.nested_conditions"},"metadata":{},"interface":{"inputs":{"variables":{"my_input":{"type":{"simple":"FLOAT"},"description":"my_input"}}},"outputs":{"variables":{"o0":{"type":{"simple":"FLOAT"},"description":"o0"}}}},"nodes":[{"id":"start-node"},{"id":"end-node","inputs":[{"var":"o0","binding":{"promise":{"nodeId":"n0","var":"o0"}}}]},{"id":"n0","metadata":{"name":"fractions","retries":{}},"inputs":[{"var":".my_input","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"branchNode":{"ifElse":{"case":{"condition":{"conjunction":{"leftExpression":{"comparison":{"operator":"GT","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":0.1}}}},"rightExpression":{"comparison":{"operator":"LT","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":1}}}}}},"thenNode":{"id":"n0-n0","metadata":{"name":"inner_fractions","retries":{}},"inputs":[{"var":".my_input","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"branchNode":{"ifElse":{"case":{"condition":{"comparison":{"operator":"LT","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":0.5}}}},"thenNode":{"id":"n0-n0-n0","metadata":{"name":"double","retries":{}},"inputs":[{"var":"n","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"taskNode":{"referenceId":{"resourceType":"TASK","name":"core.control_flow.conditions.double"},"overrides":{}}}},"other":[{"condition":{"conjunction":{"leftExpression":{"comparison":{"operator":"GT","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":0.5}}}},"rightExpression":{"comparison":{"operator":"LT","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":0.7}}}}}},"thenNode":{"id":"n0-n0-n1","metadata":{"name":"square","retries":{}},"inputs":[{"var":"n","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"taskNode":{"referenceId":{"resourceType":"TASK","name":"core.control_flow.conditions.square"},"overrides":{}}}}],"error":{"failedNodeId":"inner_fractions","message":"Only \u003c0.7 allowed"}}}}},"other":[{"condition":{"conjunction":{"leftExpression":{"comparison":{"operator":"GT","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":1}}}},"rightExpression":{"comparison":{"operator":"LT","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":10}}}}}},"thenNode":{"id":"n0-n1","metadata":{"name":"square","retries":{}},"inputs":[{"var":"n","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"taskNode":{"referenceId":{"resourceType":"TASK","name":"core.control_flow.conditions.square"},"overrides":{}}}}],"elseNode":{"id":"n0-n2","metadata":{"name":"double","retries":{}},"inputs":[{"var":"n","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"taskNode":{"referenceId":{"resourceType":"TASK","name":"core.control_flow.conditions.double"},"overrides":{}}}}}}],"outputs":[{"var":"o0","binding":{"promise":{"nodeId":"n0","var":"o0"}}}],"metadataDefaults":{}},"connections":{"downstream":{"n0":{"ids":["end-node"]},"start-node":{"ids":["n0"]}},"upstream":{"end-node":{"ids":["n0"]},"n0":{"ids":["start-node"]},"n0-n0":{"ids":["start-node"]},"n0-n0-n0":{"ids":["start-node"]},"n0-n0-n1":{"ids":["start-node"]},"n0-n1":{"ids":["start-node"]},"n0-n2":{"ids":["start-node"]}}}},"tasks":[{"template":{"id":{"resourceType":"TASK","name":"core.control_flow.conditions.double"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"n":{"type":{"simple":"FLOAT"},"description":"name of the parameter for the task is derived from the name of the input variable\nand the type is mapped to ``Types.Integer``"}}},"outputs":{"variables":{"o0":{"type":{"simple":"FLOAT"},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.control_flow.conditions","task-name","double"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},{"template":{"id":{"resourceType":"TASK","name":"core.control_flow.conditions.square"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"n":{"type":{"simple":"FLOAT"},"description":"name of the parameter for the task is derived from the name of the input variable, and\nthe type is automatically mapped to Types.Integer"}}},"outputs":{"variables":{"o0":{"type":{"simple":"FLOAT"},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.control_flow.conditions","task-name","square"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}}]} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/039_core.control_flow.conditions.nested_conditions_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/039_core.control_flow.conditions.nested_conditions_2_wf.yaml new file mode 100755 index 0000000000..a61d08e8c5 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/039_core.control_flow.conditions.nested_conditions_2_wf.yaml @@ -0,0 +1,403 @@ +primary: + connections: + downstream: + n0: + ids: + - end-node + start-node: + ids: + - n0 + upstream: + end-node: + ids: + - n0 + n0: + ids: + - start-node + n0-n0: + ids: + - start-node + n0-n0-n0: + ids: + - start-node + n0-n0-n1: + ids: + - start-node + n0-n1: + ids: + - start-node + n0-n2: + ids: + - start-node + template: + id: + name: core.control_flow.conditions.nested_conditions + resource_type: 2 + interface: + inputs: + variables: + my_input: + description: my_input + type: + Type: + Simple: 2 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 2 + metadata: {} + metadata_defaults: {} + nodes: + - Target: null + id: start-node + - Target: null + id: end-node + inputs: + - binding: + Value: + Promise: + node_id: n0 + var: o0 + var: o0 + - Target: + BranchNode: + if_else: + Default: + ElseNode: + Target: + TaskNode: + Reference: + ReferenceId: + name: core.control_flow.conditions.double + resource_type: 1 + overrides: {} + id: n0-n2 + inputs: + - binding: + Value: + Promise: + node_id: start-node + var: my_input + var: "n" + metadata: + InterruptibleValue: null + name: double + retries: {} + case: + condition: + Expr: + Conjunction: + left_expression: + Expr: + Comparison: + left_value: + Val: + Var: .my_input + operator: 2 + right_value: + Val: + Primitive: + Value: + FloatValue: 0.1 + right_expression: + Expr: + Comparison: + left_value: + Val: + Var: .my_input + operator: 4 + right_value: + Val: + Primitive: + Value: + FloatValue: 1 + then_node: + Target: + BranchNode: + if_else: + Default: + Error: + failed_node_id: inner_fractions + message: Only <0.7 allowed + case: + condition: + Expr: + Comparison: + left_value: + Val: + Var: .my_input + operator: 4 + right_value: + Val: + Primitive: + Value: + FloatValue: 0.5 + then_node: + Target: + TaskNode: + Reference: + ReferenceId: + name: core.control_flow.conditions.double + resource_type: 1 + overrides: {} + id: n0-n0-n0 + inputs: + - binding: + Value: + Promise: + node_id: start-node + var: my_input + var: "n" + metadata: + InterruptibleValue: null + name: double + retries: {} + other: + - condition: + Expr: + Conjunction: + left_expression: + Expr: + Comparison: + left_value: + Val: + Var: .my_input + operator: 2 + right_value: + Val: + Primitive: + Value: + FloatValue: 0.5 + right_expression: + Expr: + Comparison: + left_value: + Val: + Var: .my_input + operator: 4 + right_value: + Val: + Primitive: + Value: + FloatValue: 0.7 + then_node: + Target: + TaskNode: + Reference: + ReferenceId: + name: core.control_flow.conditions.square + resource_type: 1 + overrides: {} + id: n0-n0-n1 + inputs: + - binding: + Value: + Promise: + node_id: start-node + var: my_input + var: "n" + metadata: + InterruptibleValue: null + name: square + retries: {} + id: n0-n0 + inputs: + - binding: + Value: + Promise: + node_id: start-node + var: my_input + var: .my_input + metadata: + InterruptibleValue: null + name: inner_fractions + retries: {} + other: + - condition: + Expr: + Conjunction: + left_expression: + Expr: + Comparison: + left_value: + Val: + Var: .my_input + operator: 2 + right_value: + Val: + Primitive: + Value: + FloatValue: 1 + right_expression: + Expr: + Comparison: + left_value: + Val: + Var: .my_input + operator: 4 + right_value: + Val: + Primitive: + Value: + FloatValue: 10 + then_node: + Target: + TaskNode: + Reference: + ReferenceId: + name: core.control_flow.conditions.square + resource_type: 1 + overrides: {} + id: n0-n1 + inputs: + - binding: + Value: + Promise: + node_id: start-node + var: my_input + var: "n" + metadata: + InterruptibleValue: null + name: square + retries: {} + id: n0 + inputs: + - binding: + Value: + Promise: + node_id: start-node + var: my_input + var: .my_input + metadata: + InterruptibleValue: null + name: fractions + retries: {} + outputs: + - binding: + Value: + Promise: + node_id: n0 + var: o0 + var: o0 +tasks: +- template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.control_flow.conditions + - task-name + - double + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.control_flow.conditions.double + resource_type: 1 + interface: + inputs: + variables: + "n": + description: |- + name of the parameter for the task is derived from the name of the input variable + and the type is mapped to ``Types.Integer`` + type: + Type: + Simple: 2 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 2 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task +- template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.control_flow.conditions + - task-name + - square + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.control_flow.conditions.square + resource_type: 1 + interface: + inputs: + variables: + "n": + description: |- + name of the parameter for the task is derived from the name of the input variable, and + the type is automatically mapped to Types.Integer + type: + Type: + Simple: 2 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 2 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/041_core.control_flow.conditions.calc_sum_1_task.json b/pkg/compiler/test/testdata/snacks-core/compiled/041_core.control_flow.conditions.calc_sum_1_task.json new file mode 100755 index 0000000000..cb3eb36274 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/041_core.control_flow.conditions.calc_sum_1_task.json @@ -0,0 +1 @@ +{"template":{"id":{"resourceType":"TASK","name":"core.control_flow.conditions.calc_sum"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"a":{"type":{"simple":"FLOAT"},"description":"a"},"b":{"type":{"simple":"FLOAT"},"description":"b"}}},"outputs":{"variables":{"o0":{"type":{"simple":"FLOAT"},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.control_flow.conditions","task-name","calc_sum"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/041_core.control_flow.conditions.calc_sum_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/041_core.control_flow.conditions.calc_sum_1_task.yaml new file mode 100755 index 0000000000..87bdc4a87f --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/041_core.control_flow.conditions.calc_sum_1_task.yaml @@ -0,0 +1,62 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.control_flow.conditions + - task-name + - calc_sum + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.control_flow.conditions.calc_sum + resource_type: 1 + interface: + inputs: + variables: + a: + description: a + type: + Type: + Simple: 2 + b: + description: b + type: + Type: + Simple: 2 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 2 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/042_core.control_flow.conditions.consume_outputs_2_wf.json b/pkg/compiler/test/testdata/snacks-core/compiled/042_core.control_flow.conditions.consume_outputs_2_wf.json new file mode 100755 index 0000000000..c47000978d --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/042_core.control_flow.conditions.consume_outputs_2_wf.json @@ -0,0 +1 @@ +{"primary":{"template":{"id":{"resourceType":"WORKFLOW","name":"core.control_flow.conditions.consume_outputs"},"metadata":{},"interface":{"inputs":{"variables":{"my_input":{"type":{"simple":"FLOAT"},"description":"my_input"},"seed":{"type":{"simple":"INTEGER"},"description":"seed"}}},"outputs":{"variables":{"o0":{"type":{"simple":"FLOAT"},"description":"o0"}}}},"nodes":[{"id":"start-node"},{"id":"end-node","inputs":[{"var":"o0","binding":{"promise":{"nodeId":"n2","var":"o0"}}}]},{"id":"n0","metadata":{"name":"coin_toss","retries":{}},"inputs":[{"var":"seed","binding":{"promise":{"nodeId":"start-node","var":"seed"}}}],"taskNode":{"referenceId":{"resourceType":"TASK","name":"core.control_flow.conditions.coin_toss"},"overrides":{}}},{"id":"n1","metadata":{"name":"double_or_square","retries":{}},"inputs":[{"var":"n0.o0","binding":{"promise":{"nodeId":"n0","var":"o0"}}}],"upstreamNodeIds":["n0"],"branchNode":{"ifElse":{"case":{"condition":{"comparison":{"leftValue":{"var":"n0.o0"},"rightValue":{"primitive":{"boolean":true}}}},"thenNode":{"id":"n1-n0","metadata":{"name":"square","retries":{}},"inputs":[{"var":"n","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"taskNode":{"referenceId":{"resourceType":"TASK","name":"core.control_flow.conditions.square"},"overrides":{}}}},"elseNode":{"id":"n1-n1","metadata":{"name":"calc_sum","retries":{}},"inputs":[{"var":"a","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}},{"var":"b","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"taskNode":{"referenceId":{"resourceType":"TASK","name":"core.control_flow.conditions.calc_sum"},"overrides":{}}}}}},{"id":"n2","metadata":{"name":"double","retries":{}},"inputs":[{"var":"n","binding":{"promise":{"nodeId":"n1","var":"o0"}}}],"upstreamNodeIds":["n1"],"taskNode":{"referenceId":{"resourceType":"TASK","name":"core.control_flow.conditions.double"},"overrides":{}}}],"outputs":[{"var":"o0","binding":{"promise":{"nodeId":"n2","var":"o0"}}}],"metadataDefaults":{}},"connections":{"downstream":{"n0":{"ids":["n1"]},"n1":{"ids":["n2"]},"n2":{"ids":["end-node"]},"start-node":{"ids":["n0"]}},"upstream":{"end-node":{"ids":["n2"]},"n0":{"ids":["start-node"]},"n1":{"ids":["n0"]},"n1-n0":{"ids":["start-node"]},"n1-n1":{"ids":["start-node"]},"n2":{"ids":["n1"]}}}},"tasks":[{"template":{"id":{"resourceType":"TASK","name":"core.control_flow.conditions.calc_sum"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"a":{"type":{"simple":"FLOAT"},"description":"a"},"b":{"type":{"simple":"FLOAT"},"description":"b"}}},"outputs":{"variables":{"o0":{"type":{"simple":"FLOAT"},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.control_flow.conditions","task-name","calc_sum"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},{"template":{"id":{"resourceType":"TASK","name":"core.control_flow.conditions.coin_toss"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"seed":{"type":{"simple":"INTEGER"},"description":"seed"}}},"outputs":{"variables":{"o0":{"type":{"simple":"BOOLEAN"},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.control_flow.conditions","task-name","coin_toss"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},{"template":{"id":{"resourceType":"TASK","name":"core.control_flow.conditions.double"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"n":{"type":{"simple":"FLOAT"},"description":"name of the parameter for the task is derived from the name of the input variable\nand the type is mapped to ``Types.Integer``"}}},"outputs":{"variables":{"o0":{"type":{"simple":"FLOAT"},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.control_flow.conditions","task-name","double"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},{"template":{"id":{"resourceType":"TASK","name":"core.control_flow.conditions.square"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"n":{"type":{"simple":"FLOAT"},"description":"name of the parameter for the task is derived from the name of the input variable, and\nthe type is automatically mapped to Types.Integer"}}},"outputs":{"variables":{"o0":{"type":{"simple":"FLOAT"},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.control_flow.conditions","task-name","square"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}}]} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/042_core.control_flow.conditions.consume_outputs_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/042_core.control_flow.conditions.consume_outputs_2_wf.yaml new file mode 100755 index 0000000000..20926d2160 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/042_core.control_flow.conditions.consume_outputs_2_wf.yaml @@ -0,0 +1,433 @@ +primary: + connections: + downstream: + n0: + ids: + - n1 + n1: + ids: + - n2 + n2: + ids: + - end-node + start-node: + ids: + - n0 + upstream: + end-node: + ids: + - n2 + n0: + ids: + - start-node + n1: + ids: + - n0 + n1-n0: + ids: + - start-node + n1-n1: + ids: + - start-node + n2: + ids: + - n1 + template: + id: + name: core.control_flow.conditions.consume_outputs + resource_type: 2 + interface: + inputs: + variables: + my_input: + description: my_input + type: + Type: + Simple: 2 + seed: + description: seed + type: + Type: + Simple: 1 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 2 + metadata: {} + metadata_defaults: {} + nodes: + - Target: null + id: start-node + - Target: null + id: end-node + inputs: + - binding: + Value: + Promise: + node_id: n2 + var: o0 + var: o0 + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.control_flow.conditions.coin_toss + resource_type: 1 + overrides: {} + id: n0 + inputs: + - binding: + Value: + Promise: + node_id: start-node + var: seed + var: seed + metadata: + InterruptibleValue: null + name: coin_toss + retries: {} + - Target: + BranchNode: + if_else: + Default: + ElseNode: + Target: + TaskNode: + Reference: + ReferenceId: + name: core.control_flow.conditions.calc_sum + resource_type: 1 + overrides: {} + id: n1-n1 + inputs: + - binding: + Value: + Promise: + node_id: start-node + var: my_input + var: a + - binding: + Value: + Promise: + node_id: start-node + var: my_input + var: b + metadata: + InterruptibleValue: null + name: calc_sum + retries: {} + case: + condition: + Expr: + Comparison: + left_value: + Val: + Var: n0.o0 + right_value: + Val: + Primitive: + Value: + Boolean: true + then_node: + Target: + TaskNode: + Reference: + ReferenceId: + name: core.control_flow.conditions.square + resource_type: 1 + overrides: {} + id: n1-n0 + inputs: + - binding: + Value: + Promise: + node_id: start-node + var: my_input + var: "n" + metadata: + InterruptibleValue: null + name: square + retries: {} + id: n1 + inputs: + - binding: + Value: + Promise: + node_id: n0 + var: o0 + var: n0.o0 + metadata: + InterruptibleValue: null + name: double_or_square + retries: {} + upstream_node_ids: + - n0 + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.control_flow.conditions.double + resource_type: 1 + overrides: {} + id: n2 + inputs: + - binding: + Value: + Promise: + node_id: n1 + var: o0 + var: "n" + metadata: + InterruptibleValue: null + name: double + retries: {} + upstream_node_ids: + - n1 + outputs: + - binding: + Value: + Promise: + node_id: n2 + var: o0 + var: o0 +tasks: +- template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.control_flow.conditions + - task-name + - calc_sum + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.control_flow.conditions.calc_sum + resource_type: 1 + interface: + inputs: + variables: + a: + description: a + type: + Type: + Simple: 2 + b: + description: b + type: + Type: + Simple: 2 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 2 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task +- template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.control_flow.conditions + - task-name + - coin_toss + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.control_flow.conditions.coin_toss + resource_type: 1 + interface: + inputs: + variables: + seed: + description: seed + type: + Type: + Simple: 1 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 4 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task +- template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.control_flow.conditions + - task-name + - double + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.control_flow.conditions.double + resource_type: 1 + interface: + inputs: + variables: + "n": + description: |- + name of the parameter for the task is derived from the name of the input variable + and the type is mapped to ``Types.Integer`` + type: + Type: + Simple: 2 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 2 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task +- template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.control_flow.conditions + - task-name + - square + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.control_flow.conditions.square + resource_type: 1 + interface: + inputs: + variables: + "n": + description: |- + name of the parameter for the task is derived from the name of the input variable, and + the type is automatically mapped to Types.Integer + type: + Type: + Simple: 2 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 2 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/044_core.control_flow.dynamics.return_index_1_task.json b/pkg/compiler/test/testdata/snacks-core/compiled/044_core.control_flow.dynamics.return_index_1_task.json new file mode 100755 index 0000000000..f54422e822 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/044_core.control_flow.dynamics.return_index_1_task.json @@ -0,0 +1 @@ +{"template":{"id":{"resourceType":"TASK","name":"core.control_flow.dynamics.return_index"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"character":{"type":{"simple":"STRING"},"description":"character"}}},"outputs":{"variables":{"o0":{"type":{"simple":"INTEGER"},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.control_flow.dynamics","task-name","return_index"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/044_core.control_flow.dynamics.return_index_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/044_core.control_flow.dynamics.return_index_1_task.yaml new file mode 100755 index 0000000000..0a35877307 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/044_core.control_flow.dynamics.return_index_1_task.yaml @@ -0,0 +1,57 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.control_flow.dynamics + - task-name + - return_index + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.control_flow.dynamics.return_index + resource_type: 1 + interface: + inputs: + variables: + character: + description: character + type: + Type: + Simple: 3 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 1 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/045_core.control_flow.dynamics.update_list_1_task.json b/pkg/compiler/test/testdata/snacks-core/compiled/045_core.control_flow.dynamics.update_list_1_task.json new file mode 100755 index 0000000000..789dda8426 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/045_core.control_flow.dynamics.update_list_1_task.json @@ -0,0 +1 @@ +{"template":{"id":{"resourceType":"TASK","name":"core.control_flow.dynamics.update_list"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"freq_list":{"type":{"collectionType":{"simple":"INTEGER"}},"description":"freq_list"},"list_index":{"type":{"simple":"INTEGER"},"description":"list_index"}}},"outputs":{"variables":{"o0":{"type":{"collectionType":{"simple":"INTEGER"}},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.control_flow.dynamics","task-name","update_list"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/045_core.control_flow.dynamics.update_list_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/045_core.control_flow.dynamics.update_list_1_task.yaml new file mode 100755 index 0000000000..555f8dddba --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/045_core.control_flow.dynamics.update_list_1_task.yaml @@ -0,0 +1,66 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.control_flow.dynamics + - task-name + - update_list + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.control_flow.dynamics.update_list + resource_type: 1 + interface: + inputs: + variables: + freq_list: + description: freq_list + type: + Type: + CollectionType: + Type: + Simple: 1 + list_index: + description: list_index + type: + Type: + Simple: 1 + outputs: + variables: + o0: + description: o0 + type: + Type: + CollectionType: + Type: + Simple: 1 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/046_core.control_flow.dynamics.derive_count_1_task.json b/pkg/compiler/test/testdata/snacks-core/compiled/046_core.control_flow.dynamics.derive_count_1_task.json new file mode 100755 index 0000000000..efbe3e4f33 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/046_core.control_flow.dynamics.derive_count_1_task.json @@ -0,0 +1 @@ +{"template":{"id":{"resourceType":"TASK","name":"core.control_flow.dynamics.derive_count"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"freq1":{"type":{"collectionType":{"simple":"INTEGER"}},"description":"freq1"},"freq2":{"type":{"collectionType":{"simple":"INTEGER"}},"description":"freq2"}}},"outputs":{"variables":{"o0":{"type":{"simple":"INTEGER"},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.control_flow.dynamics","task-name","derive_count"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/046_core.control_flow.dynamics.derive_count_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/046_core.control_flow.dynamics.derive_count_1_task.yaml new file mode 100755 index 0000000000..9e924e6004 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/046_core.control_flow.dynamics.derive_count_1_task.yaml @@ -0,0 +1,66 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.control_flow.dynamics + - task-name + - derive_count + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.control_flow.dynamics.derive_count + resource_type: 1 + interface: + inputs: + variables: + freq1: + description: freq1 + type: + Type: + CollectionType: + Type: + Simple: 1 + freq2: + description: freq2 + type: + Type: + CollectionType: + Type: + Simple: 1 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 1 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/047_core.control_flow.dynamics.count_characters_1_task.json b/pkg/compiler/test/testdata/snacks-core/compiled/047_core.control_flow.dynamics.count_characters_1_task.json new file mode 100755 index 0000000000..a37b564535 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/047_core.control_flow.dynamics.count_characters_1_task.json @@ -0,0 +1 @@ +{"template":{"id":{"resourceType":"TASK","name":"core.control_flow.dynamics.count_characters"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"s1":{"type":{"simple":"STRING"},"description":"s1"},"s2":{"type":{"simple":"STRING"},"description":"s2"}}},"outputs":{"variables":{"o0":{"type":{"simple":"INTEGER"},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.control_flow.dynamics","task-name","count_characters"],"resources":{},"env":[{"key":"_F_SS_C","value":"H4sIAAAAAAAC/8VRXW7DIAy+ysTzWtKtUuiuMk2RAZN6pZCBE6mrevcBXdXuBHsIsr8f85mcBR1hxMHE4GgUb09nYdHB7HloREMCHGtxY8Tzk3BfoSLj3qQ1Ren8iTGm8VqYGA+6fFXIUKcKExOulFa4AbXdmR1YZ0CpbruDrt/qTWfBvvTK9V3X9+JSjO36XLzv/xHgo0aYUvxEw8UeZu8LYOMRKNz7BVOm+ABgWO7NdOJ9DAMFxjQlLGcNIuPEcilCqSnIq+a1bVSTH4iHhRLP4ItkSDHyX5MnfTOtlczEuJrAHOpbyduENg0yDxkTgadv4JKydMwUxtz+KQbQHm2pHfiMdTnMhb5KLaWHrSlzIj03xkfTJL/05fIDQ+X0kUICAAA="}],"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/047_core.control_flow.dynamics.count_characters_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/047_core.control_flow.dynamics.count_characters_1_task.yaml new file mode 100755 index 0000000000..0605db6c29 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/047_core.control_flow.dynamics.count_characters_1_task.yaml @@ -0,0 +1,65 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.control_flow.dynamics + - task-name + - count_characters + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + env: + - key: _F_SS_C + value: H4sIAAAAAAAC/8VRXW7DIAy+ysTzWtKtUuiuMk2RAZN6pZCBE6mrevcBXdXuBHsIsr8f85mcBR1hxMHE4GgUb09nYdHB7HloREMCHGtxY8Tzk3BfoSLj3qQ1Ren8iTGm8VqYGA+6fFXIUKcKExOulFa4AbXdmR1YZ0CpbruDrt/qTWfBvvTK9V3X9+JSjO36XLzv/xHgo0aYUvxEw8UeZu8LYOMRKNz7BVOm+ABgWO7NdOJ9DAMFxjQlLGcNIuPEcilCqSnIq+a1bVSTH4iHhRLP4ItkSDHyX5MnfTOtlczEuJrAHOpbyduENg0yDxkTgadv4JKydMwUxtz+KQbQHm2pHfiMdTnMhb5KLaWHrSlzIj03xkfTJL/05fIDQ+X0kUICAAA= + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.control_flow.dynamics.count_characters + resource_type: 1 + interface: + inputs: + variables: + s1: + description: s1 + type: + Type: + Simple: 3 + s2: + description: s2 + type: + Type: + Simple: 3 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 1 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/048_core.control_flow.dynamics.wf_2_wf.json b/pkg/compiler/test/testdata/snacks-core/compiled/048_core.control_flow.dynamics.wf_2_wf.json new file mode 100755 index 0000000000..3da3788538 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/048_core.control_flow.dynamics.wf_2_wf.json @@ -0,0 +1 @@ +{"primary":{"template":{"id":{"resourceType":"WORKFLOW","name":"core.control_flow.dynamics.wf"},"metadata":{},"interface":{"inputs":{"variables":{"s1":{"type":{"simple":"STRING"},"description":"s1"},"s2":{"type":{"simple":"STRING"},"description":"s2"}}},"outputs":{"variables":{"o0":{"type":{"simple":"INTEGER"},"description":"o0"}}}},"nodes":[{"id":"start-node"},{"id":"end-node","inputs":[{"var":"o0","binding":{"promise":{"nodeId":"n0","var":"o0"}}}]},{"id":"n0","metadata":{"name":"count_characters","retries":{}},"inputs":[{"var":"s1","binding":{"promise":{"nodeId":"start-node","var":"s1"}}},{"var":"s2","binding":{"promise":{"nodeId":"start-node","var":"s2"}}}],"taskNode":{"referenceId":{"resourceType":"TASK","name":"core.control_flow.dynamics.count_characters"},"overrides":{}}}],"outputs":[{"var":"o0","binding":{"promise":{"nodeId":"n0","var":"o0"}}}],"metadataDefaults":{}},"connections":{"downstream":{"n0":{"ids":["end-node"]},"start-node":{"ids":["n0"]}},"upstream":{"end-node":{"ids":["n0"]},"n0":{"ids":["start-node"]}}}},"tasks":[{"template":{"id":{"resourceType":"TASK","name":"core.control_flow.dynamics.count_characters"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"s1":{"type":{"simple":"STRING"},"description":"s1"},"s2":{"type":{"simple":"STRING"},"description":"s2"}}},"outputs":{"variables":{"o0":{"type":{"simple":"INTEGER"},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.control_flow.dynamics","task-name","count_characters"],"resources":{},"env":[{"key":"_F_SS_C","value":"H4sIAAAAAAAC/8VRXW7DIAy+ysTzWtKtUuiuMk2RAZN6pZCBE6mrevcBXdXuBHsIsr8f85mcBR1hxMHE4GgUb09nYdHB7HloREMCHGtxY8Tzk3BfoSLj3qQ1Ren8iTGm8VqYGA+6fFXIUKcKExOulFa4AbXdmR1YZ0CpbruDrt/qTWfBvvTK9V3X9+JSjO36XLzv/xHgo0aYUvxEw8UeZu8LYOMRKNz7BVOm+ABgWO7NdOJ9DAMFxjQlLGcNIuPEcilCqSnIq+a1bVSTH4iHhRLP4ItkSDHyX5MnfTOtlczEuJrAHOpbyduENg0yDxkTgadv4JKydMwUxtz+KQbQHm2pHfiMdTnMhb5KLaWHrSlzIj03xkfTJL/05fIDQ+X0kUICAAA="}],"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}}]} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/048_core.control_flow.dynamics.wf_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/048_core.control_flow.dynamics.wf_2_wf.yaml new file mode 100755 index 0000000000..cc80b0308d --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/048_core.control_flow.dynamics.wf_2_wf.yaml @@ -0,0 +1,152 @@ +primary: + connections: + downstream: + n0: + ids: + - end-node + start-node: + ids: + - n0 + upstream: + end-node: + ids: + - n0 + n0: + ids: + - start-node + template: + id: + name: core.control_flow.dynamics.wf + resource_type: 2 + interface: + inputs: + variables: + s1: + description: s1 + type: + Type: + Simple: 3 + s2: + description: s2 + type: + Type: + Simple: 3 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 1 + metadata: {} + metadata_defaults: {} + nodes: + - Target: null + id: start-node + - Target: null + id: end-node + inputs: + - binding: + Value: + Promise: + node_id: n0 + var: o0 + var: o0 + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.control_flow.dynamics.count_characters + resource_type: 1 + overrides: {} + id: n0 + inputs: + - binding: + Value: + Promise: + node_id: start-node + var: s1 + var: s1 + - binding: + Value: + Promise: + node_id: start-node + var: s2 + var: s2 + metadata: + InterruptibleValue: null + name: count_characters + retries: {} + outputs: + - binding: + Value: + Promise: + node_id: n0 + var: o0 + var: o0 +tasks: +- template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.control_flow.dynamics + - task-name + - count_characters + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + env: + - key: _F_SS_C + value: H4sIAAAAAAAC/8VRXW7DIAy+ysTzWtKtUuiuMk2RAZN6pZCBE6mrevcBXdXuBHsIsr8f85mcBR1hxMHE4GgUb09nYdHB7HloREMCHGtxY8Tzk3BfoSLj3qQ1Ren8iTGm8VqYGA+6fFXIUKcKExOulFa4AbXdmR1YZ0CpbruDrt/qTWfBvvTK9V3X9+JSjO36XLzv/xHgo0aYUvxEw8UeZu8LYOMRKNz7BVOm+ABgWO7NdOJ9DAMFxjQlLGcNIuPEcilCqSnIq+a1bVSTH4iHhRLP4ItkSDHyX5MnfTOtlczEuJrAHOpbyduENg0yDxkTgadv4JKydMwUxtz+KQbQHm2pHfiMdTnMhb5KLaWHrSlzIj03xkfTJL/05fIDQ+X0kUICAAA= + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.control_flow.dynamics.count_characters + resource_type: 1 + interface: + inputs: + variables: + s1: + description: s1 + type: + Type: + Simple: 3 + s2: + description: s2 + type: + Type: + Simple: 3 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 1 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/050_core.control_flow.map_task.a_mappable_task_1_task.json b/pkg/compiler/test/testdata/snacks-core/compiled/050_core.control_flow.map_task.a_mappable_task_1_task.json new file mode 100755 index 0000000000..85649eb4c7 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/050_core.control_flow.map_task.a_mappable_task_1_task.json @@ -0,0 +1 @@ +{"template":{"id":{"resourceType":"TASK","name":"core.control_flow.map_task.a_mappable_task"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"a":{"type":{"simple":"INTEGER"},"description":"a"}}},"outputs":{"variables":{"o0":{"type":{"simple":"STRING"},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.control_flow.map_task","task-name","a_mappable_task"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/050_core.control_flow.map_task.a_mappable_task_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/050_core.control_flow.map_task.a_mappable_task_1_task.yaml new file mode 100755 index 0000000000..2cc969bce7 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/050_core.control_flow.map_task.a_mappable_task_1_task.yaml @@ -0,0 +1,57 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.control_flow.map_task + - task-name + - a_mappable_task + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.control_flow.map_task.a_mappable_task + resource_type: 1 + interface: + inputs: + variables: + a: + description: a + type: + Type: + Simple: 1 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 3 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/051_core.control_flow.map_task.coalesce_1_task.json b/pkg/compiler/test/testdata/snacks-core/compiled/051_core.control_flow.map_task.coalesce_1_task.json new file mode 100755 index 0000000000..099f10d1d8 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/051_core.control_flow.map_task.coalesce_1_task.json @@ -0,0 +1 @@ +{"template":{"id":{"resourceType":"TASK","name":"core.control_flow.map_task.coalesce"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"b":{"type":{"collectionType":{"simple":"STRING"}},"description":"b"}}},"outputs":{"variables":{"o0":{"type":{"simple":"STRING"},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.control_flow.map_task","task-name","coalesce"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/051_core.control_flow.map_task.coalesce_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/051_core.control_flow.map_task.coalesce_1_task.yaml new file mode 100755 index 0000000000..c1add64c9d --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/051_core.control_flow.map_task.coalesce_1_task.yaml @@ -0,0 +1,59 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.control_flow.map_task + - task-name + - coalesce + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.control_flow.map_task.coalesce + resource_type: 1 + interface: + inputs: + variables: + b: + description: b + type: + Type: + CollectionType: + Type: + Simple: 3 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 3 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/052_core.control_flow.map_task.mapper_a_mappable_task_0_1_task.json b/pkg/compiler/test/testdata/snacks-core/compiled/052_core.control_flow.map_task.mapper_a_mappable_task_0_1_task.json new file mode 100755 index 0000000000..1424cf4937 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/052_core.control_flow.map_task.mapper_a_mappable_task_0_1_task.json @@ -0,0 +1 @@ +{"template":{"id":{"resourceType":"TASK","name":"core.control_flow.map_task.mapper_a_mappable_task_0"},"type":"container_array","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"a":{"type":{"collectionType":{"simple":"INTEGER"}},"description":"a"}}},"outputs":{"variables":{"o0":{"type":{"collectionType":{"simple":"STRING"}},"description":"o0"}}}},"custom":{"minSuccessRatio":1},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-map-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.control_flow.map_task","task-name","a_mappable_task"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]},"taskTypeVersion":1}} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/052_core.control_flow.map_task.mapper_a_mappable_task_0_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/052_core.control_flow.map_task.mapper_a_mappable_task_0_1_task.yaml new file mode 100755 index 0000000000..955ac744ce --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/052_core.control_flow.map_task.mapper_a_mappable_task_0_1_task.yaml @@ -0,0 +1,64 @@ +template: + Target: + Container: + args: + - pyflyte-map-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.control_flow.map_task + - task-name + - a_mappable_task + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + custom: + minSuccessRatio: 1 + id: + name: core.control_flow.map_task.mapper_a_mappable_task_0 + resource_type: 1 + interface: + inputs: + variables: + a: + description: a + type: + Type: + CollectionType: + Type: + Simple: 1 + outputs: + variables: + o0: + description: o0 + type: + Type: + CollectionType: + Type: + Simple: 3 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + task_type_version: 1 + type: container_array diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/053_core.control_flow.map_task.my_map_workflow_2_wf.json b/pkg/compiler/test/testdata/snacks-core/compiled/053_core.control_flow.map_task.my_map_workflow_2_wf.json new file mode 100755 index 0000000000..b6ec606f23 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/053_core.control_flow.map_task.my_map_workflow_2_wf.json @@ -0,0 +1 @@ +{"primary":{"template":{"id":{"resourceType":"WORKFLOW","name":"core.control_flow.map_task.my_map_workflow"},"metadata":{},"interface":{"inputs":{"variables":{"a":{"type":{"collectionType":{"simple":"INTEGER"}},"description":"a"}}},"outputs":{"variables":{"o0":{"type":{"simple":"STRING"},"description":"o0"}}}},"nodes":[{"id":"start-node"},{"id":"end-node","inputs":[{"var":"o0","binding":{"promise":{"nodeId":"n1","var":"o0"}}}]},{"id":"n0","metadata":{"name":"mapper_a_mappable_task_0","retries":{"retries":1}},"inputs":[{"var":"a","binding":{"promise":{"nodeId":"start-node","var":"a"}}}],"taskNode":{"referenceId":{"resourceType":"TASK","name":"core.control_flow.map_task.mapper_a_mappable_task_0"},"overrides":{"resources":{"requests":[{"name":"MEMORY","value":"300Mi"}],"limits":[{"name":"MEMORY","value":"500Mi"}]}}}},{"id":"n1","metadata":{"name":"coalesce","retries":{}},"inputs":[{"var":"b","binding":{"promise":{"nodeId":"n0","var":"o0"}}}],"upstreamNodeIds":["n0"],"taskNode":{"referenceId":{"resourceType":"TASK","name":"core.control_flow.map_task.coalesce"},"overrides":{}}}],"outputs":[{"var":"o0","binding":{"promise":{"nodeId":"n1","var":"o0"}}}],"metadataDefaults":{}},"connections":{"downstream":{"n0":{"ids":["n1"]},"n1":{"ids":["end-node"]},"start-node":{"ids":["n0"]}},"upstream":{"end-node":{"ids":["n1"]},"n0":{"ids":["start-node"]},"n1":{"ids":["n0"]}}}},"tasks":[{"template":{"id":{"resourceType":"TASK","name":"core.control_flow.map_task.coalesce"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"b":{"type":{"collectionType":{"simple":"STRING"}},"description":"b"}}},"outputs":{"variables":{"o0":{"type":{"simple":"STRING"},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.control_flow.map_task","task-name","coalesce"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},{"template":{"id":{"resourceType":"TASK","name":"core.control_flow.map_task.mapper_a_mappable_task_0"},"type":"container_array","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"a":{"type":{"collectionType":{"simple":"INTEGER"}},"description":"a"}}},"outputs":{"variables":{"o0":{"type":{"collectionType":{"simple":"STRING"}},"description":"o0"}}}},"custom":{"minSuccessRatio":1},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-map-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.control_flow.map_task","task-name","a_mappable_task"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]},"taskTypeVersion":1}}]} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/053_core.control_flow.map_task.my_map_workflow_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/053_core.control_flow.map_task.my_map_workflow_2_wf.yaml new file mode 100755 index 0000000000..42c7da57a4 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/053_core.control_flow.map_task.my_map_workflow_2_wf.yaml @@ -0,0 +1,236 @@ +primary: + connections: + downstream: + n0: + ids: + - n1 + n1: + ids: + - end-node + start-node: + ids: + - n0 + upstream: + end-node: + ids: + - n1 + n0: + ids: + - start-node + n1: + ids: + - n0 + template: + id: + name: core.control_flow.map_task.my_map_workflow + resource_type: 2 + interface: + inputs: + variables: + a: + description: a + type: + Type: + CollectionType: + Type: + Simple: 1 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 3 + metadata: {} + metadata_defaults: {} + nodes: + - Target: null + id: start-node + - Target: null + id: end-node + inputs: + - binding: + Value: + Promise: + node_id: n1 + var: o0 + var: o0 + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.control_flow.map_task.mapper_a_mappable_task_0 + resource_type: 1 + overrides: + resources: + limits: + - name: 3 + value: 500Mi + requests: + - name: 3 + value: 300Mi + id: n0 + inputs: + - binding: + Value: + Promise: + node_id: start-node + var: a + var: a + metadata: + InterruptibleValue: null + name: mapper_a_mappable_task_0 + retries: + retries: 1 + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.control_flow.map_task.coalesce + resource_type: 1 + overrides: {} + id: n1 + inputs: + - binding: + Value: + Promise: + node_id: n0 + var: o0 + var: b + metadata: + InterruptibleValue: null + name: coalesce + retries: {} + upstream_node_ids: + - n0 + outputs: + - binding: + Value: + Promise: + node_id: n1 + var: o0 + var: o0 +tasks: +- template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.control_flow.map_task + - task-name + - coalesce + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.control_flow.map_task.coalesce + resource_type: 1 + interface: + inputs: + variables: + b: + description: b + type: + Type: + CollectionType: + Type: + Simple: 3 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 3 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task +- template: + Target: + Container: + args: + - pyflyte-map-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.control_flow.map_task + - task-name + - a_mappable_task + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + custom: + minSuccessRatio: 1 + id: + name: core.control_flow.map_task.mapper_a_mappable_task_0 + resource_type: 1 + interface: + inputs: + variables: + a: + description: a + type: + Type: + CollectionType: + Type: + Simple: 1 + outputs: + variables: + o0: + description: o0 + type: + Type: + CollectionType: + Type: + Simple: 3 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + task_type_version: 1 + type: container_array diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/055_core.control_flow.merge_sort.split_1_task.json b/pkg/compiler/test/testdata/snacks-core/compiled/055_core.control_flow.merge_sort.split_1_task.json new file mode 100755 index 0000000000..2a28024dea --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/055_core.control_flow.merge_sort.split_1_task.json @@ -0,0 +1 @@ +{"template":{"id":{"resourceType":"TASK","name":"core.control_flow.merge_sort.split"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"numbers":{"type":{"collectionType":{"simple":"INTEGER"}},"description":"numbers"}}},"outputs":{"variables":{"o0":{"type":{"collectionType":{"simple":"INTEGER"}},"description":"o0"},"o1":{"type":{"collectionType":{"simple":"INTEGER"}},"description":"o1"},"o2":{"type":{"simple":"INTEGER"},"description":"o2"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.control_flow.merge_sort","task-name","split"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/055_core.control_flow.merge_sort.split_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/055_core.control_flow.merge_sort.split_1_task.yaml new file mode 100755 index 0000000000..46815cb6ef --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/055_core.control_flow.merge_sort.split_1_task.yaml @@ -0,0 +1,73 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.control_flow.merge_sort + - task-name + - split + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.control_flow.merge_sort.split + resource_type: 1 + interface: + inputs: + variables: + numbers: + description: numbers + type: + Type: + CollectionType: + Type: + Simple: 1 + outputs: + variables: + o0: + description: o0 + type: + Type: + CollectionType: + Type: + Simple: 1 + o1: + description: o1 + type: + Type: + CollectionType: + Type: + Simple: 1 + o2: + description: o2 + type: + Type: + Simple: 1 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/056_core.control_flow.merge_sort.merge_1_task.json b/pkg/compiler/test/testdata/snacks-core/compiled/056_core.control_flow.merge_sort.merge_1_task.json new file mode 100755 index 0000000000..b0713fe152 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/056_core.control_flow.merge_sort.merge_1_task.json @@ -0,0 +1 @@ +{"template":{"id":{"resourceType":"TASK","name":"core.control_flow.merge_sort.merge"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"sorted_list1":{"type":{"collectionType":{"simple":"INTEGER"}},"description":"sorted_list1"},"sorted_list2":{"type":{"collectionType":{"simple":"INTEGER"}},"description":"sorted_list2"}}},"outputs":{"variables":{"o0":{"type":{"collectionType":{"simple":"INTEGER"}},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.control_flow.merge_sort","task-name","merge"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/056_core.control_flow.merge_sort.merge_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/056_core.control_flow.merge_sort.merge_1_task.yaml new file mode 100755 index 0000000000..bb3a3dbbcc --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/056_core.control_flow.merge_sort.merge_1_task.yaml @@ -0,0 +1,68 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.control_flow.merge_sort + - task-name + - merge + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.control_flow.merge_sort.merge + resource_type: 1 + interface: + inputs: + variables: + sorted_list1: + description: sorted_list1 + type: + Type: + CollectionType: + Type: + Simple: 1 + sorted_list2: + description: sorted_list2 + type: + Type: + CollectionType: + Type: + Simple: 1 + outputs: + variables: + o0: + description: o0 + type: + Type: + CollectionType: + Type: + Simple: 1 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/057_core.control_flow.merge_sort.sort_locally_1_task.json b/pkg/compiler/test/testdata/snacks-core/compiled/057_core.control_flow.merge_sort.sort_locally_1_task.json new file mode 100755 index 0000000000..5f11485540 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/057_core.control_flow.merge_sort.sort_locally_1_task.json @@ -0,0 +1 @@ +{"template":{"id":{"resourceType":"TASK","name":"core.control_flow.merge_sort.sort_locally"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"numbers":{"type":{"collectionType":{"simple":"INTEGER"}},"description":"numbers"}}},"outputs":{"variables":{"o0":{"type":{"collectionType":{"simple":"INTEGER"}},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.control_flow.merge_sort","task-name","sort_locally"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/057_core.control_flow.merge_sort.sort_locally_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/057_core.control_flow.merge_sort.sort_locally_1_task.yaml new file mode 100755 index 0000000000..3741ee770b --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/057_core.control_flow.merge_sort.sort_locally_1_task.yaml @@ -0,0 +1,61 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.control_flow.merge_sort + - task-name + - sort_locally + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.control_flow.merge_sort.sort_locally + resource_type: 1 + interface: + inputs: + variables: + numbers: + description: numbers + type: + Type: + CollectionType: + Type: + Simple: 1 + outputs: + variables: + o0: + description: o0 + type: + Type: + CollectionType: + Type: + Simple: 1 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/058_core.control_flow.merge_sort.merge_sort_remotely_1_task.json b/pkg/compiler/test/testdata/snacks-core/compiled/058_core.control_flow.merge_sort.merge_sort_remotely_1_task.json new file mode 100755 index 0000000000..2a8b4c8d07 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/058_core.control_flow.merge_sort.merge_sort_remotely_1_task.json @@ -0,0 +1 @@ +{"template":{"id":{"resourceType":"TASK","name":"core.control_flow.merge_sort.merge_sort_remotely"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"numbers":{"type":{"collectionType":{"simple":"INTEGER"}},"description":"numbers"},"run_local_at_count":{"type":{"simple":"INTEGER"},"description":"run_local_at_count"}}},"outputs":{"variables":{"o0":{"type":{"collectionType":{"simple":"INTEGER"}},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.control_flow.merge_sort","task-name","merge_sort_remotely"],"resources":{},"env":[{"key":"_F_SS_C","value":"H4sIAAAAAAAC/8VRXW7DIAy+ysTzWtKtUuiuMk2RAZN6pZCBE6mrevcBXdXuBHsIsr8f85mcBR1hxMHE4GgUb09nYdHB7HloREMCHGtxY8Tzk3BfoSLj3qQ1Ren8iTGm8VqYGA+6fFXIUKcKExOulFa4AbXdmR1YZ0CpbruDrt/qTWfBvvTK9V3X9+JSjO36XLzv/xHgo0aYUvxEw8UeZu8LYOMRKNz7BVOm+ABgWO7NdOJ9DAMFxjQlLGcNIuPEcilCqSnIq+a1bVSTH4iHhRLP4ItkSDHyX5MnfTOtlczEuJrAHOpbyduENg0yDxkTgadv4JKydMwUxtz+KQbQHm2pHfiMdTnMhb5KLaWHrSlzIj03xkfTJL/05fIDQ+X0kUICAAA="}],"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/058_core.control_flow.merge_sort.merge_sort_remotely_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/058_core.control_flow.merge_sort.merge_sort_remotely_1_task.yaml new file mode 100755 index 0000000000..f61183d9c1 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/058_core.control_flow.merge_sort.merge_sort_remotely_1_task.yaml @@ -0,0 +1,69 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.control_flow.merge_sort + - task-name + - merge_sort_remotely + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + env: + - key: _F_SS_C + value: H4sIAAAAAAAC/8VRXW7DIAy+ysTzWtKtUuiuMk2RAZN6pZCBE6mrevcBXdXuBHsIsr8f85mcBR1hxMHE4GgUb09nYdHB7HloREMCHGtxY8Tzk3BfoSLj3qQ1Ren8iTGm8VqYGA+6fFXIUKcKExOulFa4AbXdmR1YZ0CpbruDrt/qTWfBvvTK9V3X9+JSjO36XLzv/xHgo0aYUvxEw8UeZu8LYOMRKNz7BVOm+ABgWO7NdOJ9DAMFxjQlLGcNIuPEcilCqSnIq+a1bVSTH4iHhRLP4ItkSDHyX5MnfTOtlczEuJrAHOpbyduENg0yDxkTgadv4JKydMwUxtz+KQbQHm2pHfiMdTnMhb5KLaWHrSlzIj03xkfTJL/05fIDQ+X0kUICAAA= + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.control_flow.merge_sort.merge_sort_remotely + resource_type: 1 + interface: + inputs: + variables: + numbers: + description: numbers + type: + Type: + CollectionType: + Type: + Simple: 1 + run_local_at_count: + description: run_local_at_count + type: + Type: + Simple: 1 + outputs: + variables: + o0: + description: o0 + type: + Type: + CollectionType: + Type: + Simple: 1 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/059_core.control_flow.merge_sort.merge_sort_2_wf.json b/pkg/compiler/test/testdata/snacks-core/compiled/059_core.control_flow.merge_sort.merge_sort_2_wf.json new file mode 100755 index 0000000000..587f08f922 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/059_core.control_flow.merge_sort.merge_sort_2_wf.json @@ -0,0 +1 @@ +{"primary":{"template":{"id":{"resourceType":"WORKFLOW","name":"core.control_flow.merge_sort.merge_sort"},"metadata":{},"interface":{"inputs":{"variables":{"numbers":{"type":{"collectionType":{"simple":"INTEGER"}},"description":"numbers"},"numbers_count":{"type":{"simple":"INTEGER"},"description":"numbers_count"},"run_local_at_count":{"type":{"simple":"INTEGER"},"description":"run_local_at_count"}}},"outputs":{"variables":{"o0":{"type":{"collectionType":{"simple":"INTEGER"}},"description":"o0"}}}},"nodes":[{"id":"start-node"},{"id":"end-node","inputs":[{"var":"o0","binding":{"promise":{"nodeId":"n0","var":"o0"}}}]},{"id":"n0","metadata":{"name":"terminal_case","retries":{}},"inputs":[{"var":".numbers_count","binding":{"promise":{"nodeId":"start-node","var":"numbers_count"}}},{"var":".run_local_at_count","binding":{"promise":{"nodeId":"start-node","var":"run_local_at_count"}}}],"branchNode":{"ifElse":{"case":{"condition":{"comparison":{"operator":"LTE","leftValue":{"var":".numbers_count"},"rightValue":{"var":".run_local_at_count"}}},"thenNode":{"id":"n0-n0","metadata":{"name":"sort_locally","retries":{}},"inputs":[{"var":"numbers","binding":{"promise":{"nodeId":"start-node","var":"numbers"}}}],"taskNode":{"referenceId":{"resourceType":"TASK","name":"core.control_flow.merge_sort.sort_locally"},"overrides":{}}}},"elseNode":{"id":"n0-n1","metadata":{"name":"merge_sort_remotely","retries":{}},"inputs":[{"var":"numbers","binding":{"promise":{"nodeId":"start-node","var":"numbers"}}},{"var":"run_local_at_count","binding":{"promise":{"nodeId":"start-node","var":"run_local_at_count"}}}],"taskNode":{"referenceId":{"resourceType":"TASK","name":"core.control_flow.merge_sort.merge_sort_remotely"},"overrides":{}}}}}}],"outputs":[{"var":"o0","binding":{"promise":{"nodeId":"n0","var":"o0"}}}],"metadataDefaults":{}},"connections":{"downstream":{"n0":{"ids":["end-node"]},"start-node":{"ids":["n0"]}},"upstream":{"end-node":{"ids":["n0"]},"n0":{"ids":["start-node"]},"n0-n0":{"ids":["start-node"]},"n0-n1":{"ids":["start-node"]}}}},"tasks":[{"template":{"id":{"resourceType":"TASK","name":"core.control_flow.merge_sort.merge_sort_remotely"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"numbers":{"type":{"collectionType":{"simple":"INTEGER"}},"description":"numbers"},"run_local_at_count":{"type":{"simple":"INTEGER"},"description":"run_local_at_count"}}},"outputs":{"variables":{"o0":{"type":{"collectionType":{"simple":"INTEGER"}},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.control_flow.merge_sort","task-name","merge_sort_remotely"],"resources":{},"env":[{"key":"_F_SS_C","value":"H4sIAAAAAAAC/8VRXW7DIAy+ysTzWtKtUuiuMk2RAZN6pZCBE6mrevcBXdXuBHsIsr8f85mcBR1hxMHE4GgUb09nYdHB7HloREMCHGtxY8Tzk3BfoSLj3qQ1Ren8iTGm8VqYGA+6fFXIUKcKExOulFa4AbXdmR1YZ0CpbruDrt/qTWfBvvTK9V3X9+JSjO36XLzv/xHgo0aYUvxEw8UeZu8LYOMRKNz7BVOm+ABgWO7NdOJ9DAMFxjQlLGcNIuPEcilCqSnIq+a1bVSTH4iHhRLP4ItkSDHyX5MnfTOtlczEuJrAHOpbyduENg0yDxkTgadv4JKydMwUxtz+KQbQHm2pHfiMdTnMhb5KLaWHrSlzIj03xkfTJL/05fIDQ+X0kUICAAA="}],"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},{"template":{"id":{"resourceType":"TASK","name":"core.control_flow.merge_sort.sort_locally"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"numbers":{"type":{"collectionType":{"simple":"INTEGER"}},"description":"numbers"}}},"outputs":{"variables":{"o0":{"type":{"collectionType":{"simple":"INTEGER"}},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.control_flow.merge_sort","task-name","sort_locally"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}}]} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/059_core.control_flow.merge_sort.merge_sort_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/059_core.control_flow.merge_sort.merge_sort_2_wf.yaml new file mode 100755 index 0000000000..217bfa1a0b --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/059_core.control_flow.merge_sort.merge_sort_2_wf.yaml @@ -0,0 +1,286 @@ +primary: + connections: + downstream: + n0: + ids: + - end-node + start-node: + ids: + - n0 + upstream: + end-node: + ids: + - n0 + n0: + ids: + - start-node + n0-n0: + ids: + - start-node + n0-n1: + ids: + - start-node + template: + id: + name: core.control_flow.merge_sort.merge_sort + resource_type: 2 + interface: + inputs: + variables: + numbers: + description: numbers + type: + Type: + CollectionType: + Type: + Simple: 1 + numbers_count: + description: numbers_count + type: + Type: + Simple: 1 + run_local_at_count: + description: run_local_at_count + type: + Type: + Simple: 1 + outputs: + variables: + o0: + description: o0 + type: + Type: + CollectionType: + Type: + Simple: 1 + metadata: {} + metadata_defaults: {} + nodes: + - Target: null + id: start-node + - Target: null + id: end-node + inputs: + - binding: + Value: + Promise: + node_id: n0 + var: o0 + var: o0 + - Target: + BranchNode: + if_else: + Default: + ElseNode: + Target: + TaskNode: + Reference: + ReferenceId: + name: core.control_flow.merge_sort.merge_sort_remotely + resource_type: 1 + overrides: {} + id: n0-n1 + inputs: + - binding: + Value: + Promise: + node_id: start-node + var: numbers + var: numbers + - binding: + Value: + Promise: + node_id: start-node + var: run_local_at_count + var: run_local_at_count + metadata: + InterruptibleValue: null + name: merge_sort_remotely + retries: {} + case: + condition: + Expr: + Comparison: + left_value: + Val: + Var: .numbers_count + operator: 5 + right_value: + Val: + Var: .run_local_at_count + then_node: + Target: + TaskNode: + Reference: + ReferenceId: + name: core.control_flow.merge_sort.sort_locally + resource_type: 1 + overrides: {} + id: n0-n0 + inputs: + - binding: + Value: + Promise: + node_id: start-node + var: numbers + var: numbers + metadata: + InterruptibleValue: null + name: sort_locally + retries: {} + id: n0 + inputs: + - binding: + Value: + Promise: + node_id: start-node + var: numbers_count + var: .numbers_count + - binding: + Value: + Promise: + node_id: start-node + var: run_local_at_count + var: .run_local_at_count + metadata: + InterruptibleValue: null + name: terminal_case + retries: {} + outputs: + - binding: + Value: + Promise: + node_id: n0 + var: o0 + var: o0 +tasks: +- template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.control_flow.merge_sort + - task-name + - merge_sort_remotely + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + env: + - key: _F_SS_C + value: H4sIAAAAAAAC/8VRXW7DIAy+ysTzWtKtUuiuMk2RAZN6pZCBE6mrevcBXdXuBHsIsr8f85mcBR1hxMHE4GgUb09nYdHB7HloREMCHGtxY8Tzk3BfoSLj3qQ1Ren8iTGm8VqYGA+6fFXIUKcKExOulFa4AbXdmR1YZ0CpbruDrt/qTWfBvvTK9V3X9+JSjO36XLzv/xHgo0aYUvxEw8UeZu8LYOMRKNz7BVOm+ABgWO7NdOJ9DAMFxjQlLGcNIuPEcilCqSnIq+a1bVSTH4iHhRLP4ItkSDHyX5MnfTOtlczEuJrAHOpbyduENg0yDxkTgadv4JKydMwUxtz+KQbQHm2pHfiMdTnMhb5KLaWHrSlzIj03xkfTJL/05fIDQ+X0kUICAAA= + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.control_flow.merge_sort.merge_sort_remotely + resource_type: 1 + interface: + inputs: + variables: + numbers: + description: numbers + type: + Type: + CollectionType: + Type: + Simple: 1 + run_local_at_count: + description: run_local_at_count + type: + Type: + Simple: 1 + outputs: + variables: + o0: + description: o0 + type: + Type: + CollectionType: + Type: + Simple: 1 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task +- template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.control_flow.merge_sort + - task-name + - sort_locally + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.control_flow.merge_sort.sort_locally + resource_type: 1 + interface: + inputs: + variables: + numbers: + description: numbers + type: + Type: + CollectionType: + Type: + Simple: 1 + outputs: + variables: + o0: + description: o0 + type: + Type: + CollectionType: + Type: + Simple: 1 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/061_core.control_flow.subworkflows.t1_1_task.json b/pkg/compiler/test/testdata/snacks-core/compiled/061_core.control_flow.subworkflows.t1_1_task.json new file mode 100755 index 0000000000..46608e6bfe --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/061_core.control_flow.subworkflows.t1_1_task.json @@ -0,0 +1 @@ +{"template":{"id":{"resourceType":"TASK","name":"core.control_flow.subworkflows.t1"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"a":{"type":{"simple":"INTEGER"},"description":"a"}}},"outputs":{"variables":{"c":{"type":{"simple":"STRING"},"description":"c"},"t1_int_output":{"type":{"simple":"INTEGER"},"description":"t1_int_output"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.control_flow.subworkflows","task-name","t1"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/061_core.control_flow.subworkflows.t1_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/061_core.control_flow.subworkflows.t1_1_task.yaml new file mode 100755 index 0000000000..c6d2005f44 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/061_core.control_flow.subworkflows.t1_1_task.yaml @@ -0,0 +1,62 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.control_flow.subworkflows + - task-name + - t1 + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.control_flow.subworkflows.t1 + resource_type: 1 + interface: + inputs: + variables: + a: + description: a + type: + Type: + Simple: 1 + outputs: + variables: + c: + description: c + type: + Type: + Simple: 3 + t1_int_output: + description: t1_int_output + type: + Type: + Simple: 1 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/062_core.control_flow.subworkflows.my_subwf_2_wf.json b/pkg/compiler/test/testdata/snacks-core/compiled/062_core.control_flow.subworkflows.my_subwf_2_wf.json new file mode 100755 index 0000000000..414f81f1c4 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/062_core.control_flow.subworkflows.my_subwf_2_wf.json @@ -0,0 +1 @@ +{"primary":{"template":{"id":{"resourceType":"WORKFLOW","name":"core.control_flow.subworkflows.my_subwf"},"metadata":{},"interface":{"inputs":{"variables":{"a":{"type":{"simple":"INTEGER"},"description":"a"}}},"outputs":{"variables":{"o0":{"type":{"simple":"STRING"},"description":"o0"},"o1":{"type":{"simple":"STRING"},"description":"o1"}}}},"nodes":[{"id":"start-node"},{"id":"end-node","inputs":[{"var":"o0","binding":{"promise":{"nodeId":"n0","var":"c"}}},{"var":"o1","binding":{"promise":{"nodeId":"n1","var":"c"}}}]},{"id":"n0","metadata":{"name":"t1","retries":{}},"inputs":[{"var":"a","binding":{"promise":{"nodeId":"start-node","var":"a"}}}],"taskNode":{"referenceId":{"resourceType":"TASK","name":"core.control_flow.subworkflows.t1"},"overrides":{}}},{"id":"n1","metadata":{"name":"t1","retries":{}},"inputs":[{"var":"a","binding":{"promise":{"nodeId":"n0","var":"t1_int_output"}}}],"upstreamNodeIds":["n0"],"taskNode":{"referenceId":{"resourceType":"TASK","name":"core.control_flow.subworkflows.t1"},"overrides":{}}}],"outputs":[{"var":"o0","binding":{"promise":{"nodeId":"n0","var":"c"}}},{"var":"o1","binding":{"promise":{"nodeId":"n1","var":"c"}}}],"metadataDefaults":{}},"connections":{"downstream":{"n0":{"ids":["end-node","n1"]},"n1":{"ids":["end-node"]},"start-node":{"ids":["n0"]}},"upstream":{"end-node":{"ids":["n0","n1"]},"n0":{"ids":["start-node"]},"n1":{"ids":["n0"]}}}},"tasks":[{"template":{"id":{"resourceType":"TASK","name":"core.control_flow.subworkflows.t1"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"a":{"type":{"simple":"INTEGER"},"description":"a"}}},"outputs":{"variables":{"c":{"type":{"simple":"STRING"},"description":"c"},"t1_int_output":{"type":{"simple":"INTEGER"},"description":"t1_int_output"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.control_flow.subworkflows","task-name","t1"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}}]} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/062_core.control_flow.subworkflows.my_subwf_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/062_core.control_flow.subworkflows.my_subwf_2_wf.yaml new file mode 100755 index 0000000000..1cefa24edc --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/062_core.control_flow.subworkflows.my_subwf_2_wf.yaml @@ -0,0 +1,184 @@ +primary: + connections: + downstream: + n0: + ids: + - end-node + - n1 + n1: + ids: + - end-node + start-node: + ids: + - n0 + upstream: + end-node: + ids: + - n0 + - n1 + n0: + ids: + - start-node + n1: + ids: + - n0 + template: + id: + name: core.control_flow.subworkflows.my_subwf + resource_type: 2 + interface: + inputs: + variables: + a: + description: a + type: + Type: + Simple: 1 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 3 + o1: + description: o1 + type: + Type: + Simple: 3 + metadata: {} + metadata_defaults: {} + nodes: + - Target: null + id: start-node + - Target: null + id: end-node + inputs: + - binding: + Value: + Promise: + node_id: n0 + var: c + var: o0 + - binding: + Value: + Promise: + node_id: n1 + var: c + var: o1 + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.control_flow.subworkflows.t1 + resource_type: 1 + overrides: {} + id: n0 + inputs: + - binding: + Value: + Promise: + node_id: start-node + var: a + var: a + metadata: + InterruptibleValue: null + name: t1 + retries: {} + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.control_flow.subworkflows.t1 + resource_type: 1 + overrides: {} + id: n1 + inputs: + - binding: + Value: + Promise: + node_id: n0 + var: t1_int_output + var: a + metadata: + InterruptibleValue: null + name: t1 + retries: {} + upstream_node_ids: + - n0 + outputs: + - binding: + Value: + Promise: + node_id: n0 + var: c + var: o0 + - binding: + Value: + Promise: + node_id: n1 + var: c + var: o1 +tasks: +- template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.control_flow.subworkflows + - task-name + - t1 + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.control_flow.subworkflows.t1 + resource_type: 1 + interface: + inputs: + variables: + a: + description: a + type: + Type: + Simple: 1 + outputs: + variables: + c: + description: c + type: + Type: + Simple: 3 + t1_int_output: + description: t1_int_output + type: + Type: + Simple: 1 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/068_core.control_flow.subworkflows.count_freq_words_1_task.json b/pkg/compiler/test/testdata/snacks-core/compiled/068_core.control_flow.subworkflows.count_freq_words_1_task.json new file mode 100755 index 0000000000..dd697ffdbd --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/068_core.control_flow.subworkflows.count_freq_words_1_task.json @@ -0,0 +1 @@ +{"template":{"id":{"resourceType":"TASK","name":"core.control_flow.subworkflows.count_freq_words"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"input_string1":{"type":{"simple":"STRING"},"description":"input_string1"}}},"outputs":{"variables":{"o0":{"type":{"simple":"STRUCT"},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.control_flow.subworkflows","task-name","count_freq_words"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/068_core.control_flow.subworkflows.count_freq_words_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/068_core.control_flow.subworkflows.count_freq_words_1_task.yaml new file mode 100755 index 0000000000..a57956cc71 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/068_core.control_flow.subworkflows.count_freq_words_1_task.yaml @@ -0,0 +1,57 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.control_flow.subworkflows + - task-name + - count_freq_words + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.control_flow.subworkflows.count_freq_words + resource_type: 1 + interface: + inputs: + variables: + input_string1: + description: input_string1 + type: + Type: + Simple: 3 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 9 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/069_core.control_flow.subworkflows.ext_workflow_2_wf.json b/pkg/compiler/test/testdata/snacks-core/compiled/069_core.control_flow.subworkflows.ext_workflow_2_wf.json new file mode 100755 index 0000000000..ad2477c222 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/069_core.control_flow.subworkflows.ext_workflow_2_wf.json @@ -0,0 +1 @@ +{"primary":{"template":{"id":{"resourceType":"WORKFLOW","name":"core.control_flow.subworkflows.ext_workflow"},"metadata":{},"interface":{"inputs":{"variables":{"my_input":{"type":{"simple":"STRING"},"description":"my_input"}}},"outputs":{"variables":{"o0":{"type":{"simple":"STRUCT"},"description":"o0"}}}},"nodes":[{"id":"start-node"},{"id":"end-node","inputs":[{"var":"o0","binding":{"promise":{"nodeId":"n0","var":"o0"}}}]},{"id":"n0","metadata":{"name":"count_freq_words","retries":{}},"inputs":[{"var":"input_string1","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"taskNode":{"referenceId":{"resourceType":"TASK","name":"core.control_flow.subworkflows.count_freq_words"},"overrides":{}}}],"outputs":[{"var":"o0","binding":{"promise":{"nodeId":"n0","var":"o0"}}}],"metadataDefaults":{}},"connections":{"downstream":{"n0":{"ids":["end-node"]},"start-node":{"ids":["n0"]}},"upstream":{"end-node":{"ids":["n0"]},"n0":{"ids":["start-node"]}}}},"tasks":[{"template":{"id":{"resourceType":"TASK","name":"core.control_flow.subworkflows.count_freq_words"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"input_string1":{"type":{"simple":"STRING"},"description":"input_string1"}}},"outputs":{"variables":{"o0":{"type":{"simple":"STRUCT"},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.control_flow.subworkflows","task-name","count_freq_words"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}}]} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/069_core.control_flow.subworkflows.ext_workflow_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/069_core.control_flow.subworkflows.ext_workflow_2_wf.yaml new file mode 100755 index 0000000000..a76deaaece --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/069_core.control_flow.subworkflows.ext_workflow_2_wf.yaml @@ -0,0 +1,133 @@ +primary: + connections: + downstream: + n0: + ids: + - end-node + start-node: + ids: + - n0 + upstream: + end-node: + ids: + - n0 + n0: + ids: + - start-node + template: + id: + name: core.control_flow.subworkflows.ext_workflow + resource_type: 2 + interface: + inputs: + variables: + my_input: + description: my_input + type: + Type: + Simple: 3 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 9 + metadata: {} + metadata_defaults: {} + nodes: + - Target: null + id: start-node + - Target: null + id: end-node + inputs: + - binding: + Value: + Promise: + node_id: n0 + var: o0 + var: o0 + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.control_flow.subworkflows.count_freq_words + resource_type: 1 + overrides: {} + id: n0 + inputs: + - binding: + Value: + Promise: + node_id: start-node + var: my_input + var: input_string1 + metadata: + InterruptibleValue: null + name: count_freq_words + retries: {} + outputs: + - binding: + Value: + Promise: + node_id: n0 + var: o0 + var: o0 +tasks: +- template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.control_flow.subworkflows + - task-name + - count_freq_words + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.control_flow.subworkflows.count_freq_words + resource_type: 1 + interface: + inputs: + variables: + input_string1: + description: input_string1 + type: + Type: + Simple: 3 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 9 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/072_core.control_flow.subworkflows.count_repetitive_words_1_task.json b/pkg/compiler/test/testdata/snacks-core/compiled/072_core.control_flow.subworkflows.count_repetitive_words_1_task.json new file mode 100755 index 0000000000..76d88d9c01 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/072_core.control_flow.subworkflows.count_repetitive_words_1_task.json @@ -0,0 +1 @@ +{"template":{"id":{"resourceType":"TASK","name":"core.control_flow.subworkflows.count_repetitive_words"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"word_counter":{"type":{"simple":"STRUCT"},"description":"word_counter"}}},"outputs":{"variables":{"o0":{"type":{"collectionType":{"simple":"STRING"}},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.control_flow.subworkflows","task-name","count_repetitive_words"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/072_core.control_flow.subworkflows.count_repetitive_words_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/072_core.control_flow.subworkflows.count_repetitive_words_1_task.yaml new file mode 100755 index 0000000000..c960ed3fdb --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/072_core.control_flow.subworkflows.count_repetitive_words_1_task.yaml @@ -0,0 +1,59 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.control_flow.subworkflows + - task-name + - count_repetitive_words + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.control_flow.subworkflows.count_repetitive_words + resource_type: 1 + interface: + inputs: + variables: + word_counter: + description: word_counter + type: + Type: + Simple: 9 + outputs: + variables: + o0: + description: o0 + type: + Type: + CollectionType: + Type: + Simple: 3 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/075_my-objectstore-sensor_1_task.json b/pkg/compiler/test/testdata/snacks-core/compiled/075_my-objectstore-sensor_1_task.json new file mode 100755 index 0000000000..8cacfaff08 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/075_my-objectstore-sensor_1_task.json @@ -0,0 +1 @@ +{"template":{"id":{"resourceType":"TASK","name":"my-objectstore-sensor"},"type":"object-store-sensor","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"timeout":"1200s","retries":{"retries":10}},"interface":{"inputs":{"variables":{"path":{"type":{"simple":"STRING"},"description":"path"}}},"outputs":{"variables":{"path":{"type":{"simple":"STRING"},"description":"path"}}}}}} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/075_my-objectstore-sensor_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/075_my-objectstore-sensor_1_task.yaml new file mode 100755 index 0000000000..02c6d654ba --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/075_my-objectstore-sensor_1_task.yaml @@ -0,0 +1,31 @@ +template: + Target: null + id: + name: my-objectstore-sensor + resource_type: 1 + interface: + inputs: + variables: + path: + description: path + type: + Type: + Simple: 3 + outputs: + variables: + path: + description: path + type: + Type: + Simple: 3 + metadata: + InterruptibleValue: null + retries: + retries: 10 + runtime: + flavor: python + type: 1 + version: 0.32.6 + timeout: + seconds: 1200 + type: object-store-sensor diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/076_core.extend_flyte.custom_task_plugin.print_file_1_task.json b/pkg/compiler/test/testdata/snacks-core/compiled/076_core.extend_flyte.custom_task_plugin.print_file_1_task.json new file mode 100755 index 0000000000..11b6fde3c1 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/076_core.extend_flyte.custom_task_plugin.print_file_1_task.json @@ -0,0 +1 @@ +{"template":{"id":{"resourceType":"TASK","name":"core.extend_flyte.custom_task_plugin.print_file"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"path":{"type":{"simple":"STRING"},"description":"path"}}},"outputs":{"variables":{"o0":{"type":{"simple":"STRING"},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.extend_flyte.custom_task_plugin","task-name","print_file"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/076_core.extend_flyte.custom_task_plugin.print_file_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/076_core.extend_flyte.custom_task_plugin.print_file_1_task.yaml new file mode 100755 index 0000000000..22abe612cf --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/076_core.extend_flyte.custom_task_plugin.print_file_1_task.yaml @@ -0,0 +1,57 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.extend_flyte.custom_task_plugin + - task-name + - print_file + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.extend_flyte.custom_task_plugin.print_file + resource_type: 1 + interface: + inputs: + variables: + path: + description: path + type: + Type: + Simple: 3 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 3 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/077_core.extend_flyte.custom_task_plugin.my_workflow_2_wf.json b/pkg/compiler/test/testdata/snacks-core/compiled/077_core.extend_flyte.custom_task_plugin.my_workflow_2_wf.json new file mode 100755 index 0000000000..b2cb139860 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/077_core.extend_flyte.custom_task_plugin.my_workflow_2_wf.json @@ -0,0 +1 @@ +{"primary":{"template":{"id":{"resourceType":"WORKFLOW","name":"core.extend_flyte.custom_task_plugin.my_workflow"},"metadata":{},"interface":{"inputs":{"variables":{"path":{"type":{"simple":"STRING"},"description":"path"}}},"outputs":{"variables":{"o0":{"type":{"simple":"STRING"},"description":"o0"}}}},"nodes":[{"id":"start-node"},{"id":"end-node","inputs":[{"var":"o0","binding":{"promise":{"nodeId":"n1","var":"o0"}}}]},{"id":"n0","metadata":{"name":"my-objectstore-sensor","timeout":"1200s","retries":{"retries":10}},"inputs":[{"var":"path","binding":{"promise":{"nodeId":"start-node","var":"path"}}}],"taskNode":{"referenceId":{"resourceType":"TASK","name":"my-objectstore-sensor"},"overrides":{}}},{"id":"n1","metadata":{"name":"print_file","retries":{}},"inputs":[{"var":"path","binding":{"promise":{"nodeId":"n0","var":"path"}}}],"upstreamNodeIds":["n0"],"taskNode":{"referenceId":{"resourceType":"TASK","name":"core.extend_flyte.custom_task_plugin.print_file"},"overrides":{}}}],"outputs":[{"var":"o0","binding":{"promise":{"nodeId":"n1","var":"o0"}}}],"metadataDefaults":{}},"connections":{"downstream":{"n0":{"ids":["n1"]},"n1":{"ids":["end-node"]},"start-node":{"ids":["n0"]}},"upstream":{"end-node":{"ids":["n1"]},"n0":{"ids":["start-node"]},"n1":{"ids":["n0"]}}}},"tasks":[{"template":{"id":{"resourceType":"TASK","name":"core.extend_flyte.custom_task_plugin.print_file"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"path":{"type":{"simple":"STRING"},"description":"path"}}},"outputs":{"variables":{"o0":{"type":{"simple":"STRING"},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.extend_flyte.custom_task_plugin","task-name","print_file"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},{"template":{"id":{"resourceType":"TASK","name":"my-objectstore-sensor"},"type":"object-store-sensor","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"timeout":"1200s","retries":{"retries":10}},"interface":{"inputs":{"variables":{"path":{"type":{"simple":"STRING"},"description":"path"}}},"outputs":{"variables":{"path":{"type":{"simple":"STRING"},"description":"path"}}}}}}]} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/077_core.extend_flyte.custom_task_plugin.my_workflow_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/077_core.extend_flyte.custom_task_plugin.my_workflow_2_wf.yaml new file mode 100755 index 0000000000..402fd6cf3a --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/077_core.extend_flyte.custom_task_plugin.my_workflow_2_wf.yaml @@ -0,0 +1,194 @@ +primary: + connections: + downstream: + n0: + ids: + - n1 + n1: + ids: + - end-node + start-node: + ids: + - n0 + upstream: + end-node: + ids: + - n1 + n0: + ids: + - start-node + n1: + ids: + - n0 + template: + id: + name: core.extend_flyte.custom_task_plugin.my_workflow + resource_type: 2 + interface: + inputs: + variables: + path: + description: path + type: + Type: + Simple: 3 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 3 + metadata: {} + metadata_defaults: {} + nodes: + - Target: null + id: start-node + - Target: null + id: end-node + inputs: + - binding: + Value: + Promise: + node_id: n1 + var: o0 + var: o0 + - Target: + TaskNode: + Reference: + ReferenceId: + name: my-objectstore-sensor + resource_type: 1 + overrides: {} + id: n0 + inputs: + - binding: + Value: + Promise: + node_id: start-node + var: path + var: path + metadata: + InterruptibleValue: null + name: my-objectstore-sensor + retries: + retries: 10 + timeout: + seconds: 1200 + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.extend_flyte.custom_task_plugin.print_file + resource_type: 1 + overrides: {} + id: n1 + inputs: + - binding: + Value: + Promise: + node_id: n0 + var: path + var: path + metadata: + InterruptibleValue: null + name: print_file + retries: {} + upstream_node_ids: + - n0 + outputs: + - binding: + Value: + Promise: + node_id: n1 + var: o0 + var: o0 +tasks: +- template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.extend_flyte.custom_task_plugin + - task-name + - print_file + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.extend_flyte.custom_task_plugin.print_file + resource_type: 1 + interface: + inputs: + variables: + path: + description: path + type: + Type: + Simple: 3 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 3 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task +- template: + Target: null + id: + name: my-objectstore-sensor + resource_type: 1 + interface: + inputs: + variables: + path: + description: path + type: + Type: + Simple: 3 + outputs: + variables: + path: + description: path + type: + Type: + Simple: 3 + metadata: + InterruptibleValue: null + retries: + retries: 10 + runtime: + flavor: python + type: 1 + version: 0.32.6 + timeout: + seconds: 1200 + type: object-store-sensor diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/079_core.extend_flyte.custom_types.generate_1_task.json b/pkg/compiler/test/testdata/snacks-core/compiled/079_core.extend_flyte.custom_types.generate_1_task.json new file mode 100755 index 0000000000..1c4fb75ba9 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/079_core.extend_flyte.custom_types.generate_1_task.json @@ -0,0 +1 @@ +{"template":{"id":{"resourceType":"TASK","name":"core.extend_flyte.custom_types.generate"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{},"outputs":{"variables":{"o0":{"type":{"blob":{"format":"binary","dimensionality":"MULTIPART"}},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.extend_flyte.custom_types","task-name","generate"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/079_core.extend_flyte.custom_types.generate_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/079_core.extend_flyte.custom_types.generate_1_task.yaml new file mode 100755 index 0000000000..09de82f8df --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/079_core.extend_flyte.custom_types.generate_1_task.yaml @@ -0,0 +1,53 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.extend_flyte.custom_types + - task-name + - generate + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.extend_flyte.custom_types.generate + resource_type: 1 + interface: + inputs: {} + outputs: + variables: + o0: + description: o0 + type: + Type: + Blob: + dimensionality: 1 + format: binary + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/080_core.extend_flyte.custom_types.consume_1_task.json b/pkg/compiler/test/testdata/snacks-core/compiled/080_core.extend_flyte.custom_types.consume_1_task.json new file mode 100755 index 0000000000..facb1b5801 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/080_core.extend_flyte.custom_types.consume_1_task.json @@ -0,0 +1 @@ +{"template":{"id":{"resourceType":"TASK","name":"core.extend_flyte.custom_types.consume"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"d":{"type":{"blob":{"format":"binary","dimensionality":"MULTIPART"}},"description":"d"}}},"outputs":{"variables":{"o0":{"type":{"simple":"STRING"},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.extend_flyte.custom_types","task-name","consume"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/080_core.extend_flyte.custom_types.consume_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/080_core.extend_flyte.custom_types.consume_1_task.yaml new file mode 100755 index 0000000000..534a613c23 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/080_core.extend_flyte.custom_types.consume_1_task.yaml @@ -0,0 +1,59 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.extend_flyte.custom_types + - task-name + - consume + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.extend_flyte.custom_types.consume + resource_type: 1 + interface: + inputs: + variables: + d: + description: d + type: + Type: + Blob: + dimensionality: 1 + format: binary + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 3 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/081_core.extend_flyte.custom_types.wf_2_wf.json b/pkg/compiler/test/testdata/snacks-core/compiled/081_core.extend_flyte.custom_types.wf_2_wf.json new file mode 100755 index 0000000000..502dcc7bc7 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/081_core.extend_flyte.custom_types.wf_2_wf.json @@ -0,0 +1 @@ +{"primary":{"template":{"id":{"resourceType":"WORKFLOW","name":"core.extend_flyte.custom_types.wf"},"metadata":{},"interface":{"inputs":{},"outputs":{"variables":{"o0":{"type":{"simple":"STRING"},"description":"o0"}}}},"nodes":[{"id":"start-node"},{"id":"end-node","inputs":[{"var":"o0","binding":{"promise":{"nodeId":"n1","var":"o0"}}}]},{"id":"n0","metadata":{"name":"generate","retries":{}},"taskNode":{"referenceId":{"resourceType":"TASK","name":"core.extend_flyte.custom_types.generate"},"overrides":{}}},{"id":"n1","metadata":{"name":"consume","retries":{}},"inputs":[{"var":"d","binding":{"promise":{"nodeId":"n0","var":"o0"}}}],"upstreamNodeIds":["n0"],"taskNode":{"referenceId":{"resourceType":"TASK","name":"core.extend_flyte.custom_types.consume"},"overrides":{}}}],"outputs":[{"var":"o0","binding":{"promise":{"nodeId":"n1","var":"o0"}}}],"metadataDefaults":{}},"connections":{"downstream":{"n0":{"ids":["n1"]},"n1":{"ids":["end-node"]},"start-node":{"ids":["n0"]}},"upstream":{"end-node":{"ids":["n1"]},"n0":{"ids":["start-node"]},"n1":{"ids":["n0"]}}}},"tasks":[{"template":{"id":{"resourceType":"TASK","name":"core.extend_flyte.custom_types.consume"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"d":{"type":{"blob":{"format":"binary","dimensionality":"MULTIPART"}},"description":"d"}}},"outputs":{"variables":{"o0":{"type":{"simple":"STRING"},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.extend_flyte.custom_types","task-name","consume"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},{"template":{"id":{"resourceType":"TASK","name":"core.extend_flyte.custom_types.generate"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{},"outputs":{"variables":{"o0":{"type":{"blob":{"format":"binary","dimensionality":"MULTIPART"}},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.extend_flyte.custom_types","task-name","generate"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}}]} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/081_core.extend_flyte.custom_types.wf_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/081_core.extend_flyte.custom_types.wf_2_wf.yaml new file mode 100755 index 0000000000..a787de1df1 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/081_core.extend_flyte.custom_types.wf_2_wf.yaml @@ -0,0 +1,202 @@ +primary: + connections: + downstream: + n0: + ids: + - n1 + n1: + ids: + - end-node + start-node: + ids: + - n0 + upstream: + end-node: + ids: + - n1 + n0: + ids: + - start-node + n1: + ids: + - n0 + template: + id: + name: core.extend_flyte.custom_types.wf + resource_type: 2 + interface: + inputs: {} + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 3 + metadata: {} + metadata_defaults: {} + nodes: + - Target: null + id: start-node + - Target: null + id: end-node + inputs: + - binding: + Value: + Promise: + node_id: n1 + var: o0 + var: o0 + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.extend_flyte.custom_types.generate + resource_type: 1 + overrides: {} + id: n0 + metadata: + InterruptibleValue: null + name: generate + retries: {} + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.extend_flyte.custom_types.consume + resource_type: 1 + overrides: {} + id: n1 + inputs: + - binding: + Value: + Promise: + node_id: n0 + var: o0 + var: d + metadata: + InterruptibleValue: null + name: consume + retries: {} + upstream_node_ids: + - n0 + outputs: + - binding: + Value: + Promise: + node_id: n1 + var: o0 + var: o0 +tasks: +- template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.extend_flyte.custom_types + - task-name + - consume + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.extend_flyte.custom_types.consume + resource_type: 1 + interface: + inputs: + variables: + d: + description: d + type: + Type: + Blob: + dimensionality: 1 + format: binary + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 3 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task +- template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.extend_flyte.custom_types + - task-name + - generate + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.extend_flyte.custom_types.generate + resource_type: 1 + interface: + inputs: {} + outputs: + variables: + o0: + description: o0 + type: + Type: + Blob: + dimensionality: 1 + format: binary + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/083_core.flyte_basics.basic_workflow.t1_1_task.json b/pkg/compiler/test/testdata/snacks-core/compiled/083_core.flyte_basics.basic_workflow.t1_1_task.json new file mode 100755 index 0000000000..ad97e5669e --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/083_core.flyte_basics.basic_workflow.t1_1_task.json @@ -0,0 +1 @@ +{"template":{"id":{"resourceType":"TASK","name":"core.flyte_basics.basic_workflow.t1"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"a":{"type":{"simple":"INTEGER"},"description":"a"}}},"outputs":{"variables":{"c":{"type":{"simple":"STRING"},"description":"c"},"t1_int_output":{"type":{"simple":"INTEGER"},"description":"t1_int_output"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.flyte_basics.basic_workflow","task-name","t1"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/083_core.flyte_basics.basic_workflow.t1_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/083_core.flyte_basics.basic_workflow.t1_1_task.yaml new file mode 100755 index 0000000000..a51c5f9305 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/083_core.flyte_basics.basic_workflow.t1_1_task.yaml @@ -0,0 +1,62 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.basic_workflow + - task-name + - t1 + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.flyte_basics.basic_workflow.t1 + resource_type: 1 + interface: + inputs: + variables: + a: + description: a + type: + Type: + Simple: 1 + outputs: + variables: + c: + description: c + type: + Type: + Simple: 3 + t1_int_output: + description: t1_int_output + type: + Type: + Simple: 1 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/084_core.flyte_basics.basic_workflow.t2_1_task.json b/pkg/compiler/test/testdata/snacks-core/compiled/084_core.flyte_basics.basic_workflow.t2_1_task.json new file mode 100755 index 0000000000..ffb21f3351 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/084_core.flyte_basics.basic_workflow.t2_1_task.json @@ -0,0 +1 @@ +{"template":{"id":{"resourceType":"TASK","name":"core.flyte_basics.basic_workflow.t2"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"a":{"type":{"simple":"STRING"},"description":"a"},"b":{"type":{"simple":"STRING"},"description":"b"}}},"outputs":{"variables":{"o0":{"type":{"simple":"STRING"},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.flyte_basics.basic_workflow","task-name","t2"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/084_core.flyte_basics.basic_workflow.t2_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/084_core.flyte_basics.basic_workflow.t2_1_task.yaml new file mode 100755 index 0000000000..cbafc25090 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/084_core.flyte_basics.basic_workflow.t2_1_task.yaml @@ -0,0 +1,62 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.basic_workflow + - task-name + - t2 + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.flyte_basics.basic_workflow.t2 + resource_type: 1 + interface: + inputs: + variables: + a: + description: a + type: + Type: + Simple: 3 + b: + description: b + type: + Type: + Simple: 3 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 3 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/085_core.flyte_basics.basic_workflow.my_wf_2_wf.json b/pkg/compiler/test/testdata/snacks-core/compiled/085_core.flyte_basics.basic_workflow.my_wf_2_wf.json new file mode 100755 index 0000000000..043a725a44 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/085_core.flyte_basics.basic_workflow.my_wf_2_wf.json @@ -0,0 +1 @@ +{"primary":{"template":{"id":{"resourceType":"WORKFLOW","name":"core.flyte_basics.basic_workflow.my_wf"},"metadata":{},"interface":{"inputs":{"variables":{"a":{"type":{"simple":"INTEGER"},"description":"a"},"b":{"type":{"simple":"STRING"},"description":"b"}}},"outputs":{"variables":{"o0":{"type":{"simple":"INTEGER"},"description":"o0"},"o1":{"type":{"simple":"STRING"},"description":"o1"}}}},"nodes":[{"id":"start-node"},{"id":"end-node","inputs":[{"var":"o0","binding":{"promise":{"nodeId":"n0","var":"t1_int_output"}}},{"var":"o1","binding":{"promise":{"nodeId":"n1","var":"o0"}}}]},{"id":"n0","metadata":{"name":"t1","retries":{}},"inputs":[{"var":"a","binding":{"promise":{"nodeId":"start-node","var":"a"}}}],"taskNode":{"referenceId":{"resourceType":"TASK","name":"core.flyte_basics.basic_workflow.t1"},"overrides":{}}},{"id":"n1","metadata":{"name":"t2","retries":{}},"inputs":[{"var":"a","binding":{"promise":{"nodeId":"n0","var":"c"}}},{"var":"b","binding":{"promise":{"nodeId":"start-node","var":"b"}}}],"upstreamNodeIds":["n0"],"taskNode":{"referenceId":{"resourceType":"TASK","name":"core.flyte_basics.basic_workflow.t2"},"overrides":{}}}],"outputs":[{"var":"o0","binding":{"promise":{"nodeId":"n0","var":"t1_int_output"}}},{"var":"o1","binding":{"promise":{"nodeId":"n1","var":"o0"}}}],"metadataDefaults":{}},"connections":{"downstream":{"n0":{"ids":["end-node","n1"]},"n1":{"ids":["end-node"]},"start-node":{"ids":["n0","n1"]}},"upstream":{"end-node":{"ids":["n0","n1"]},"n0":{"ids":["start-node"]},"n1":{"ids":["n0","start-node"]}}}},"tasks":[{"template":{"id":{"resourceType":"TASK","name":"core.flyte_basics.basic_workflow.t1"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"a":{"type":{"simple":"INTEGER"},"description":"a"}}},"outputs":{"variables":{"c":{"type":{"simple":"STRING"},"description":"c"},"t1_int_output":{"type":{"simple":"INTEGER"},"description":"t1_int_output"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.flyte_basics.basic_workflow","task-name","t1"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},{"template":{"id":{"resourceType":"TASK","name":"core.flyte_basics.basic_workflow.t2"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"a":{"type":{"simple":"STRING"},"description":"a"},"b":{"type":{"simple":"STRING"},"description":"b"}}},"outputs":{"variables":{"o0":{"type":{"simple":"STRING"},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.flyte_basics.basic_workflow","task-name","t2"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}}]} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/085_core.flyte_basics.basic_workflow.my_wf_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/085_core.flyte_basics.basic_workflow.my_wf_2_wf.yaml new file mode 100755 index 0000000000..411de9a45b --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/085_core.flyte_basics.basic_workflow.my_wf_2_wf.yaml @@ -0,0 +1,259 @@ +primary: + connections: + downstream: + n0: + ids: + - end-node + - n1 + n1: + ids: + - end-node + start-node: + ids: + - n0 + - n1 + upstream: + end-node: + ids: + - n0 + - n1 + n0: + ids: + - start-node + n1: + ids: + - n0 + - start-node + template: + id: + name: core.flyte_basics.basic_workflow.my_wf + resource_type: 2 + interface: + inputs: + variables: + a: + description: a + type: + Type: + Simple: 1 + b: + description: b + type: + Type: + Simple: 3 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 1 + o1: + description: o1 + type: + Type: + Simple: 3 + metadata: {} + metadata_defaults: {} + nodes: + - Target: null + id: start-node + - Target: null + id: end-node + inputs: + - binding: + Value: + Promise: + node_id: n0 + var: t1_int_output + var: o0 + - binding: + Value: + Promise: + node_id: n1 + var: o0 + var: o1 + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.flyte_basics.basic_workflow.t1 + resource_type: 1 + overrides: {} + id: n0 + inputs: + - binding: + Value: + Promise: + node_id: start-node + var: a + var: a + metadata: + InterruptibleValue: null + name: t1 + retries: {} + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.flyte_basics.basic_workflow.t2 + resource_type: 1 + overrides: {} + id: n1 + inputs: + - binding: + Value: + Promise: + node_id: n0 + var: c + var: a + - binding: + Value: + Promise: + node_id: start-node + var: b + var: b + metadata: + InterruptibleValue: null + name: t2 + retries: {} + upstream_node_ids: + - n0 + outputs: + - binding: + Value: + Promise: + node_id: n0 + var: t1_int_output + var: o0 + - binding: + Value: + Promise: + node_id: n1 + var: o0 + var: o1 +tasks: +- template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.basic_workflow + - task-name + - t1 + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.flyte_basics.basic_workflow.t1 + resource_type: 1 + interface: + inputs: + variables: + a: + description: a + type: + Type: + Simple: 1 + outputs: + variables: + c: + description: c + type: + Type: + Simple: 3 + t1_int_output: + description: t1_int_output + type: + Type: + Simple: 1 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task +- template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.basic_workflow + - task-name + - t2 + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.flyte_basics.basic_workflow.t2 + resource_type: 1 + interface: + inputs: + variables: + a: + description: a + type: + Type: + Simple: 3 + b: + description: b + type: + Type: + Simple: 3 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 3 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/087_core.flyte_basics.decorating_tasks.t1_1_task.json b/pkg/compiler/test/testdata/snacks-core/compiled/087_core.flyte_basics.decorating_tasks.t1_1_task.json new file mode 100755 index 0000000000..1e87c99ca2 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/087_core.flyte_basics.decorating_tasks.t1_1_task.json @@ -0,0 +1 @@ +{"template":{"id":{"resourceType":"TASK","name":"core.flyte_basics.decorating_tasks.t1"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"x":{"type":{"simple":"INTEGER"},"description":"x"}}},"outputs":{"variables":{"o0":{"type":{"simple":"INTEGER"},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.flyte_basics.decorating_tasks","task-name","t1"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/087_core.flyte_basics.decorating_tasks.t1_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/087_core.flyte_basics.decorating_tasks.t1_1_task.yaml new file mode 100755 index 0000000000..aeb251f66a --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/087_core.flyte_basics.decorating_tasks.t1_1_task.yaml @@ -0,0 +1,57 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.decorating_tasks + - task-name + - t1 + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.flyte_basics.decorating_tasks.t1 + resource_type: 1 + interface: + inputs: + variables: + x: + description: x + type: + Type: + Simple: 1 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 1 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/088_core.flyte_basics.decorating_tasks.t2_1_task.json b/pkg/compiler/test/testdata/snacks-core/compiled/088_core.flyte_basics.decorating_tasks.t2_1_task.json new file mode 100755 index 0000000000..8b14f216a7 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/088_core.flyte_basics.decorating_tasks.t2_1_task.json @@ -0,0 +1 @@ +{"template":{"id":{"resourceType":"TASK","name":"core.flyte_basics.decorating_tasks.t2"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"x":{"type":{"simple":"INTEGER"},"description":"x"}}},"outputs":{"variables":{"o0":{"type":{"simple":"INTEGER"},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.flyte_basics.decorating_tasks","task-name","t2"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/088_core.flyte_basics.decorating_tasks.t2_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/088_core.flyte_basics.decorating_tasks.t2_1_task.yaml new file mode 100755 index 0000000000..44a9b9db2d --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/088_core.flyte_basics.decorating_tasks.t2_1_task.yaml @@ -0,0 +1,57 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.decorating_tasks + - task-name + - t2 + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.flyte_basics.decorating_tasks.t2 + resource_type: 1 + interface: + inputs: + variables: + x: + description: x + type: + Type: + Simple: 1 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 1 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/089_core.flyte_basics.decorating_tasks.wf_2_wf.json b/pkg/compiler/test/testdata/snacks-core/compiled/089_core.flyte_basics.decorating_tasks.wf_2_wf.json new file mode 100755 index 0000000000..54410a1fe2 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/089_core.flyte_basics.decorating_tasks.wf_2_wf.json @@ -0,0 +1 @@ +{"primary":{"template":{"id":{"resourceType":"WORKFLOW","name":"core.flyte_basics.decorating_tasks.wf"},"metadata":{},"interface":{"inputs":{"variables":{"x":{"type":{"simple":"INTEGER"},"description":"x"}}},"outputs":{"variables":{"o0":{"type":{"simple":"INTEGER"},"description":"o0"}}}},"nodes":[{"id":"start-node"},{"id":"end-node","inputs":[{"var":"o0","binding":{"promise":{"nodeId":"n1","var":"o0"}}}]},{"id":"n0","metadata":{"name":"t1","retries":{}},"inputs":[{"var":"x","binding":{"promise":{"nodeId":"start-node","var":"x"}}}],"taskNode":{"referenceId":{"resourceType":"TASK","name":"core.flyte_basics.decorating_tasks.t1"},"overrides":{}}},{"id":"n1","metadata":{"name":"t2","retries":{}},"inputs":[{"var":"x","binding":{"promise":{"nodeId":"n0","var":"o0"}}}],"upstreamNodeIds":["n0"],"taskNode":{"referenceId":{"resourceType":"TASK","name":"core.flyte_basics.decorating_tasks.t2"},"overrides":{}}}],"outputs":[{"var":"o0","binding":{"promise":{"nodeId":"n1","var":"o0"}}}],"metadataDefaults":{}},"connections":{"downstream":{"n0":{"ids":["n1"]},"n1":{"ids":["end-node"]},"start-node":{"ids":["n0"]}},"upstream":{"end-node":{"ids":["n1"]},"n0":{"ids":["start-node"]},"n1":{"ids":["n0"]}}}},"tasks":[{"template":{"id":{"resourceType":"TASK","name":"core.flyte_basics.decorating_tasks.t1"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"x":{"type":{"simple":"INTEGER"},"description":"x"}}},"outputs":{"variables":{"o0":{"type":{"simple":"INTEGER"},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.flyte_basics.decorating_tasks","task-name","t1"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},{"template":{"id":{"resourceType":"TASK","name":"core.flyte_basics.decorating_tasks.t2"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"x":{"type":{"simple":"INTEGER"},"description":"x"}}},"outputs":{"variables":{"o0":{"type":{"simple":"INTEGER"},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.flyte_basics.decorating_tasks","task-name","t2"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}}]} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/089_core.flyte_basics.decorating_tasks.wf_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/089_core.flyte_basics.decorating_tasks.wf_2_wf.yaml new file mode 100755 index 0000000000..299e8cbe8e --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/089_core.flyte_basics.decorating_tasks.wf_2_wf.yaml @@ -0,0 +1,217 @@ +primary: + connections: + downstream: + n0: + ids: + - n1 + n1: + ids: + - end-node + start-node: + ids: + - n0 + upstream: + end-node: + ids: + - n1 + n0: + ids: + - start-node + n1: + ids: + - n0 + template: + id: + name: core.flyte_basics.decorating_tasks.wf + resource_type: 2 + interface: + inputs: + variables: + x: + description: x + type: + Type: + Simple: 1 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 1 + metadata: {} + metadata_defaults: {} + nodes: + - Target: null + id: start-node + - Target: null + id: end-node + inputs: + - binding: + Value: + Promise: + node_id: n1 + var: o0 + var: o0 + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.flyte_basics.decorating_tasks.t1 + resource_type: 1 + overrides: {} + id: n0 + inputs: + - binding: + Value: + Promise: + node_id: start-node + var: x + var: x + metadata: + InterruptibleValue: null + name: t1 + retries: {} + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.flyte_basics.decorating_tasks.t2 + resource_type: 1 + overrides: {} + id: n1 + inputs: + - binding: + Value: + Promise: + node_id: n0 + var: o0 + var: x + metadata: + InterruptibleValue: null + name: t2 + retries: {} + upstream_node_ids: + - n0 + outputs: + - binding: + Value: + Promise: + node_id: n1 + var: o0 + var: o0 +tasks: +- template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.decorating_tasks + - task-name + - t1 + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.flyte_basics.decorating_tasks.t1 + resource_type: 1 + interface: + inputs: + variables: + x: + description: x + type: + Type: + Simple: 1 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 1 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task +- template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.decorating_tasks + - task-name + - t2 + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.flyte_basics.decorating_tasks.t2 + resource_type: 1 + interface: + inputs: + variables: + x: + description: x + type: + Type: + Simple: 1 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 1 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/091_core.flyte_basics.decorating_workflows.setup_1_task.json b/pkg/compiler/test/testdata/snacks-core/compiled/091_core.flyte_basics.decorating_workflows.setup_1_task.json new file mode 100755 index 0000000000..a997de287c --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/091_core.flyte_basics.decorating_workflows.setup_1_task.json @@ -0,0 +1 @@ +{"template":{"id":{"resourceType":"TASK","name":"core.flyte_basics.decorating_workflows.setup"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{},"outputs":{}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.flyte_basics.decorating_workflows","task-name","setup"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/091_core.flyte_basics.decorating_workflows.setup_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/091_core.flyte_basics.decorating_workflows.setup_1_task.yaml new file mode 100755 index 0000000000..ad1ed53be6 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/091_core.flyte_basics.decorating_workflows.setup_1_task.yaml @@ -0,0 +1,45 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.decorating_workflows + - task-name + - setup + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.flyte_basics.decorating_workflows.setup + resource_type: 1 + interface: + inputs: {} + outputs: {} + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/092_core.flyte_basics.decorating_workflows.teardown_1_task.json b/pkg/compiler/test/testdata/snacks-core/compiled/092_core.flyte_basics.decorating_workflows.teardown_1_task.json new file mode 100755 index 0000000000..aa598cc0b6 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/092_core.flyte_basics.decorating_workflows.teardown_1_task.json @@ -0,0 +1 @@ +{"template":{"id":{"resourceType":"TASK","name":"core.flyte_basics.decorating_workflows.teardown"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{},"outputs":{}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.flyte_basics.decorating_workflows","task-name","teardown"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/092_core.flyte_basics.decorating_workflows.teardown_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/092_core.flyte_basics.decorating_workflows.teardown_1_task.yaml new file mode 100755 index 0000000000..ea344ea06b --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/092_core.flyte_basics.decorating_workflows.teardown_1_task.yaml @@ -0,0 +1,45 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.decorating_workflows + - task-name + - teardown + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.flyte_basics.decorating_workflows.teardown + resource_type: 1 + interface: + inputs: {} + outputs: {} + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/093_core.flyte_basics.decorating_workflows.t1_1_task.json b/pkg/compiler/test/testdata/snacks-core/compiled/093_core.flyte_basics.decorating_workflows.t1_1_task.json new file mode 100755 index 0000000000..9d2fa632fe --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/093_core.flyte_basics.decorating_workflows.t1_1_task.json @@ -0,0 +1 @@ +{"template":{"id":{"resourceType":"TASK","name":"core.flyte_basics.decorating_workflows.t1"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"x":{"type":{"simple":"FLOAT"},"description":"x"}}},"outputs":{"variables":{"o0":{"type":{"simple":"FLOAT"},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.flyte_basics.decorating_workflows","task-name","t1"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/093_core.flyte_basics.decorating_workflows.t1_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/093_core.flyte_basics.decorating_workflows.t1_1_task.yaml new file mode 100755 index 0000000000..a2a6ce0bc2 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/093_core.flyte_basics.decorating_workflows.t1_1_task.yaml @@ -0,0 +1,57 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.decorating_workflows + - task-name + - t1 + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.flyte_basics.decorating_workflows.t1 + resource_type: 1 + interface: + inputs: + variables: + x: + description: x + type: + Type: + Simple: 2 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 2 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/094_core.flyte_basics.decorating_workflows.t2_1_task.json b/pkg/compiler/test/testdata/snacks-core/compiled/094_core.flyte_basics.decorating_workflows.t2_1_task.json new file mode 100755 index 0000000000..1ff8097a11 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/094_core.flyte_basics.decorating_workflows.t2_1_task.json @@ -0,0 +1 @@ +{"template":{"id":{"resourceType":"TASK","name":"core.flyte_basics.decorating_workflows.t2"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"x":{"type":{"simple":"FLOAT"},"description":"x"}}},"outputs":{"variables":{"o0":{"type":{"simple":"FLOAT"},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.flyte_basics.decorating_workflows","task-name","t2"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/094_core.flyte_basics.decorating_workflows.t2_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/094_core.flyte_basics.decorating_workflows.t2_1_task.yaml new file mode 100755 index 0000000000..05101da678 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/094_core.flyte_basics.decorating_workflows.t2_1_task.yaml @@ -0,0 +1,57 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.decorating_workflows + - task-name + - t2 + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.flyte_basics.decorating_workflows.t2 + resource_type: 1 + interface: + inputs: + variables: + x: + description: x + type: + Type: + Simple: 2 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 2 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/095_core.flyte_basics.decorating_workflows.wf_2_wf.json b/pkg/compiler/test/testdata/snacks-core/compiled/095_core.flyte_basics.decorating_workflows.wf_2_wf.json new file mode 100755 index 0000000000..464a6ba15a --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/095_core.flyte_basics.decorating_workflows.wf_2_wf.json @@ -0,0 +1 @@ +{"primary":{"template":{"id":{"resourceType":"WORKFLOW","name":"core.flyte_basics.decorating_workflows.wf"},"metadata":{},"interface":{"inputs":{"variables":{"x":{"type":{"simple":"FLOAT"},"description":"x"}}},"outputs":{"variables":{"o0":{"type":{"simple":"FLOAT"},"description":"o0"}}}},"nodes":[{"id":"start-node"},{"id":"end-node","inputs":[{"var":"o0","binding":{"promise":{"nodeId":"n2","var":"o0"}}}]},{"id":"n0","metadata":{"name":"setup","retries":{}},"taskNode":{"referenceId":{"resourceType":"TASK","name":"core.flyte_basics.decorating_workflows.setup"},"overrides":{}}},{"id":"n1","metadata":{"name":"t1","retries":{}},"inputs":[{"var":"x","binding":{"promise":{"nodeId":"start-node","var":"x"}}}],"upstreamNodeIds":["n0"],"taskNode":{"referenceId":{"resourceType":"TASK","name":"core.flyte_basics.decorating_workflows.t1"},"overrides":{}}},{"id":"n2","metadata":{"name":"t2","retries":{}},"inputs":[{"var":"x","binding":{"promise":{"nodeId":"n1","var":"o0"}}}],"upstreamNodeIds":["n1"],"taskNode":{"referenceId":{"resourceType":"TASK","name":"core.flyte_basics.decorating_workflows.t2"},"overrides":{}}},{"id":"n3","metadata":{"name":"teardown","retries":{}},"upstreamNodeIds":["n2"],"taskNode":{"referenceId":{"resourceType":"TASK","name":"core.flyte_basics.decorating_workflows.teardown"},"overrides":{}}}],"outputs":[{"var":"o0","binding":{"promise":{"nodeId":"n2","var":"o0"}}}],"metadataDefaults":{}},"connections":{"downstream":{"n0":{"ids":["n1"]},"n1":{"ids":["n2"]},"n2":{"ids":["end-node","n3"]},"n3":{"ids":["end-node"]},"start-node":{"ids":["n0","n1"]}},"upstream":{"end-node":{"ids":["n2","n3"]},"n0":{"ids":["start-node"]},"n1":{"ids":["n0","start-node"]},"n2":{"ids":["n1"]},"n3":{"ids":["n2"]}}}},"tasks":[{"template":{"id":{"resourceType":"TASK","name":"core.flyte_basics.decorating_workflows.setup"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{},"outputs":{}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.flyte_basics.decorating_workflows","task-name","setup"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},{"template":{"id":{"resourceType":"TASK","name":"core.flyte_basics.decorating_workflows.t1"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"x":{"type":{"simple":"FLOAT"},"description":"x"}}},"outputs":{"variables":{"o0":{"type":{"simple":"FLOAT"},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.flyte_basics.decorating_workflows","task-name","t1"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},{"template":{"id":{"resourceType":"TASK","name":"core.flyte_basics.decorating_workflows.t2"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"x":{"type":{"simple":"FLOAT"},"description":"x"}}},"outputs":{"variables":{"o0":{"type":{"simple":"FLOAT"},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.flyte_basics.decorating_workflows","task-name","t2"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},{"template":{"id":{"resourceType":"TASK","name":"core.flyte_basics.decorating_workflows.teardown"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{},"outputs":{}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.flyte_basics.decorating_workflows","task-name","teardown"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}}]} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/095_core.flyte_basics.decorating_workflows.wf_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/095_core.flyte_basics.decorating_workflows.wf_2_wf.yaml new file mode 100755 index 0000000000..ac0947fce4 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/095_core.flyte_basics.decorating_workflows.wf_2_wf.yaml @@ -0,0 +1,351 @@ +primary: + connections: + downstream: + n0: + ids: + - n1 + n1: + ids: + - n2 + n2: + ids: + - end-node + - n3 + n3: + ids: + - end-node + start-node: + ids: + - n0 + - n1 + upstream: + end-node: + ids: + - n2 + - n3 + n0: + ids: + - start-node + n1: + ids: + - n0 + - start-node + n2: + ids: + - n1 + n3: + ids: + - n2 + template: + id: + name: core.flyte_basics.decorating_workflows.wf + resource_type: 2 + interface: + inputs: + variables: + x: + description: x + type: + Type: + Simple: 2 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 2 + metadata: {} + metadata_defaults: {} + nodes: + - Target: null + id: start-node + - Target: null + id: end-node + inputs: + - binding: + Value: + Promise: + node_id: n2 + var: o0 + var: o0 + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.flyte_basics.decorating_workflows.setup + resource_type: 1 + overrides: {} + id: n0 + metadata: + InterruptibleValue: null + name: setup + retries: {} + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.flyte_basics.decorating_workflows.t1 + resource_type: 1 + overrides: {} + id: n1 + inputs: + - binding: + Value: + Promise: + node_id: start-node + var: x + var: x + metadata: + InterruptibleValue: null + name: t1 + retries: {} + upstream_node_ids: + - n0 + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.flyte_basics.decorating_workflows.t2 + resource_type: 1 + overrides: {} + id: n2 + inputs: + - binding: + Value: + Promise: + node_id: n1 + var: o0 + var: x + metadata: + InterruptibleValue: null + name: t2 + retries: {} + upstream_node_ids: + - n1 + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.flyte_basics.decorating_workflows.teardown + resource_type: 1 + overrides: {} + id: n3 + metadata: + InterruptibleValue: null + name: teardown + retries: {} + upstream_node_ids: + - n2 + outputs: + - binding: + Value: + Promise: + node_id: n2 + var: o0 + var: o0 +tasks: +- template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.decorating_workflows + - task-name + - setup + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.flyte_basics.decorating_workflows.setup + resource_type: 1 + interface: + inputs: {} + outputs: {} + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task +- template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.decorating_workflows + - task-name + - t1 + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.flyte_basics.decorating_workflows.t1 + resource_type: 1 + interface: + inputs: + variables: + x: + description: x + type: + Type: + Simple: 2 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 2 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task +- template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.decorating_workflows + - task-name + - t2 + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.flyte_basics.decorating_workflows.t2 + resource_type: 1 + interface: + inputs: + variables: + x: + description: x + type: + Type: + Simple: 2 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 2 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task +- template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.decorating_workflows + - task-name + - teardown + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.flyte_basics.decorating_workflows.teardown + resource_type: 1 + interface: + inputs: {} + outputs: {} + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/097_core.flyte_basics.documented_workflow.add_data_1_task.json b/pkg/compiler/test/testdata/snacks-core/compiled/097_core.flyte_basics.documented_workflow.add_data_1_task.json new file mode 100755 index 0000000000..741f66ad56 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/097_core.flyte_basics.documented_workflow.add_data_1_task.json @@ -0,0 +1 @@ +{"template":{"id":{"resourceType":"TASK","name":"core.flyte_basics.documented_workflow.add_data"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"data":{"type":{"simple":"STRUCT","metadata":{"$ref":"#/definitions/PandasdataSchema","$schema":"http://json-schema.org/draft-07/schema#","definitions":{"PandasdataSchema":{"additionalProperties":false,"properties":{"id":{"default":3,"title":"id","type":"integer"},"name":{"default":"Bonnie","title":"name","type":"string"}},"type":"object"}}}},"description":"data"},"df":{"type":{"structuredDatasetType":{"format":"parquet"}},"description":"df"}}},"outputs":{"variables":{"o0":{"type":{"structuredDatasetType":{"format":"parquet"}},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.flyte_basics.documented_workflow","task-name","add_data"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/097_core.flyte_basics.documented_workflow.add_data_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/097_core.flyte_basics.documented_workflow.add_data_1_task.yaml new file mode 100755 index 0000000000..a944d1b37e --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/097_core.flyte_basics.documented_workflow.add_data_1_task.yaml @@ -0,0 +1,80 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.documented_workflow + - task-name + - add_data + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.flyte_basics.documented_workflow.add_data + resource_type: 1 + interface: + inputs: + variables: + data: + description: data + type: + Type: + Simple: 9 + metadata: + $ref: '#/definitions/PandasdataSchema' + $schema: http://json-schema.org/draft-07/schema# + definitions: + PandasdataSchema: + additionalProperties: false + properties: + id: + default: 3 + title: id + type: integer + name: + default: Bonnie + title: name + type: string + type: object + df: + description: df + type: + Type: + StructuredDatasetType: + format: parquet + outputs: + variables: + o0: + description: o0 + type: + Type: + StructuredDatasetType: + format: parquet + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/098_core.flyte_basics.documented_workflow.sphinx_docstring_2_wf.json b/pkg/compiler/test/testdata/snacks-core/compiled/098_core.flyte_basics.documented_workflow.sphinx_docstring_2_wf.json new file mode 100755 index 0000000000..73df8e6591 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/098_core.flyte_basics.documented_workflow.sphinx_docstring_2_wf.json @@ -0,0 +1 @@ +{"primary":{"template":{"id":{"resourceType":"WORKFLOW","name":"core.flyte_basics.documented_workflow.sphinx_docstring"},"metadata":{},"interface":{"inputs":{"variables":{"data":{"type":{"simple":"STRUCT","metadata":{"$ref":"#/definitions/PandasdataSchema","$schema":"http://json-schema.org/draft-07/schema#","definitions":{"PandasdataSchema":{"additionalProperties":false,"properties":{"id":{"default":3,"title":"id","type":"integer"},"name":{"default":"Bonnie","title":"name","type":"string"}},"type":"object"}}}},"description":"A data class pertaining to the new record to be stored in the DataFrame"},"df":{"type":{"structuredDatasetType":{"format":"parquet"}},"description":"Pandas DataFrame"}}},"outputs":{"variables":{"o0":{"type":{"structuredDatasetType":{"format":"parquet"}},"description":"Pandas DataFrame"}}}},"nodes":[{"id":"start-node"},{"id":"end-node","inputs":[{"var":"o0","binding":{"promise":{"nodeId":"n0","var":"o0"}}}]},{"id":"n0","metadata":{"name":"add_data","retries":{}},"inputs":[{"var":"data","binding":{"promise":{"nodeId":"start-node","var":"data"}}},{"var":"df","binding":{"promise":{"nodeId":"start-node","var":"df"}}}],"taskNode":{"referenceId":{"resourceType":"TASK","name":"core.flyte_basics.documented_workflow.add_data"},"overrides":{}}}],"outputs":[{"var":"o0","binding":{"promise":{"nodeId":"n0","var":"o0"}}}],"metadataDefaults":{}},"connections":{"downstream":{"n0":{"ids":["end-node"]},"start-node":{"ids":["n0"]}},"upstream":{"end-node":{"ids":["n0"]},"n0":{"ids":["start-node"]}}}},"tasks":[{"template":{"id":{"resourceType":"TASK","name":"core.flyte_basics.documented_workflow.add_data"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"data":{"type":{"simple":"STRUCT","metadata":{"$ref":"#/definitions/PandasdataSchema","$schema":"http://json-schema.org/draft-07/schema#","definitions":{"PandasdataSchema":{"additionalProperties":false,"properties":{"id":{"default":3,"title":"id","type":"integer"},"name":{"default":"Bonnie","title":"name","type":"string"}},"type":"object"}}}},"description":"data"},"df":{"type":{"structuredDatasetType":{"format":"parquet"}},"description":"df"}}},"outputs":{"variables":{"o0":{"type":{"structuredDatasetType":{"format":"parquet"}},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.flyte_basics.documented_workflow","task-name","add_data"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}}]} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/098_core.flyte_basics.documented_workflow.sphinx_docstring_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/098_core.flyte_basics.documented_workflow.sphinx_docstring_2_wf.yaml new file mode 100755 index 0000000000..df003fe4cb --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/098_core.flyte_basics.documented_workflow.sphinx_docstring_2_wf.yaml @@ -0,0 +1,186 @@ +primary: + connections: + downstream: + n0: + ids: + - end-node + start-node: + ids: + - n0 + upstream: + end-node: + ids: + - n0 + n0: + ids: + - start-node + template: + id: + name: core.flyte_basics.documented_workflow.sphinx_docstring + resource_type: 2 + interface: + inputs: + variables: + data: + description: A data class pertaining to the new record to be stored in + the DataFrame + type: + Type: + Simple: 9 + metadata: + $ref: '#/definitions/PandasdataSchema' + $schema: http://json-schema.org/draft-07/schema# + definitions: + PandasdataSchema: + additionalProperties: false + properties: + id: + default: 3 + title: id + type: integer + name: + default: Bonnie + title: name + type: string + type: object + df: + description: Pandas DataFrame + type: + Type: + StructuredDatasetType: + format: parquet + outputs: + variables: + o0: + description: Pandas DataFrame + type: + Type: + StructuredDatasetType: + format: parquet + metadata: {} + metadata_defaults: {} + nodes: + - Target: null + id: start-node + - Target: null + id: end-node + inputs: + - binding: + Value: + Promise: + node_id: n0 + var: o0 + var: o0 + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.flyte_basics.documented_workflow.add_data + resource_type: 1 + overrides: {} + id: n0 + inputs: + - binding: + Value: + Promise: + node_id: start-node + var: data + var: data + - binding: + Value: + Promise: + node_id: start-node + var: df + var: df + metadata: + InterruptibleValue: null + name: add_data + retries: {} + outputs: + - binding: + Value: + Promise: + node_id: n0 + var: o0 + var: o0 +tasks: +- template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.documented_workflow + - task-name + - add_data + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.flyte_basics.documented_workflow.add_data + resource_type: 1 + interface: + inputs: + variables: + data: + description: data + type: + Type: + Simple: 9 + metadata: + $ref: '#/definitions/PandasdataSchema' + $schema: http://json-schema.org/draft-07/schema# + definitions: + PandasdataSchema: + additionalProperties: false + properties: + id: + default: 3 + title: id + type: integer + name: + default: Bonnie + title: name + type: string + type: object + df: + description: df + type: + Type: + StructuredDatasetType: + format: parquet + outputs: + variables: + o0: + description: o0 + type: + Type: + StructuredDatasetType: + format: parquet + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/100_core.flyte_basics.documented_workflow.numpy_docstring_2_wf.json b/pkg/compiler/test/testdata/snacks-core/compiled/100_core.flyte_basics.documented_workflow.numpy_docstring_2_wf.json new file mode 100755 index 0000000000..10da117efe --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/100_core.flyte_basics.documented_workflow.numpy_docstring_2_wf.json @@ -0,0 +1 @@ +{"primary":{"template":{"id":{"resourceType":"WORKFLOW","name":"core.flyte_basics.documented_workflow.numpy_docstring"},"metadata":{},"interface":{"inputs":{"variables":{"data":{"type":{"simple":"STRUCT","metadata":{"$ref":"#/definitions/PandasdataSchema","$schema":"http://json-schema.org/draft-07/schema#","definitions":{"PandasdataSchema":{"additionalProperties":false,"properties":{"id":{"default":3,"title":"id","type":"integer"},"name":{"default":"Bonnie","title":"name","type":"string"}},"type":"object"}}}},"description":"A data class pertaining to the new record to be stored in the DataFrame"},"df":{"type":{"structuredDatasetType":{"format":"parquet"}},"description":"Pandas DataFrame"}}},"outputs":{"variables":{"o0":{"type":{"structuredDatasetType":{"format":"parquet"}},"description":"Pandas DataFrame"}}}},"nodes":[{"id":"start-node"},{"id":"end-node","inputs":[{"var":"o0","binding":{"promise":{"nodeId":"n0","var":"o0"}}}]},{"id":"n0","metadata":{"name":"add_data","retries":{}},"inputs":[{"var":"data","binding":{"promise":{"nodeId":"start-node","var":"data"}}},{"var":"df","binding":{"promise":{"nodeId":"start-node","var":"df"}}}],"taskNode":{"referenceId":{"resourceType":"TASK","name":"core.flyte_basics.documented_workflow.add_data"},"overrides":{}}}],"outputs":[{"var":"o0","binding":{"promise":{"nodeId":"n0","var":"o0"}}}],"metadataDefaults":{}},"connections":{"downstream":{"n0":{"ids":["end-node"]},"start-node":{"ids":["n0"]}},"upstream":{"end-node":{"ids":["n0"]},"n0":{"ids":["start-node"]}}}},"tasks":[{"template":{"id":{"resourceType":"TASK","name":"core.flyte_basics.documented_workflow.add_data"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"data":{"type":{"simple":"STRUCT","metadata":{"$ref":"#/definitions/PandasdataSchema","$schema":"http://json-schema.org/draft-07/schema#","definitions":{"PandasdataSchema":{"additionalProperties":false,"properties":{"id":{"default":3,"title":"id","type":"integer"},"name":{"default":"Bonnie","title":"name","type":"string"}},"type":"object"}}}},"description":"data"},"df":{"type":{"structuredDatasetType":{"format":"parquet"}},"description":"df"}}},"outputs":{"variables":{"o0":{"type":{"structuredDatasetType":{"format":"parquet"}},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.flyte_basics.documented_workflow","task-name","add_data"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}}]} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/100_core.flyte_basics.documented_workflow.numpy_docstring_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/100_core.flyte_basics.documented_workflow.numpy_docstring_2_wf.yaml new file mode 100755 index 0000000000..a2a7dabb14 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/100_core.flyte_basics.documented_workflow.numpy_docstring_2_wf.yaml @@ -0,0 +1,186 @@ +primary: + connections: + downstream: + n0: + ids: + - end-node + start-node: + ids: + - n0 + upstream: + end-node: + ids: + - n0 + n0: + ids: + - start-node + template: + id: + name: core.flyte_basics.documented_workflow.numpy_docstring + resource_type: 2 + interface: + inputs: + variables: + data: + description: A data class pertaining to the new record to be stored in + the DataFrame + type: + Type: + Simple: 9 + metadata: + $ref: '#/definitions/PandasdataSchema' + $schema: http://json-schema.org/draft-07/schema# + definitions: + PandasdataSchema: + additionalProperties: false + properties: + id: + default: 3 + title: id + type: integer + name: + default: Bonnie + title: name + type: string + type: object + df: + description: Pandas DataFrame + type: + Type: + StructuredDatasetType: + format: parquet + outputs: + variables: + o0: + description: Pandas DataFrame + type: + Type: + StructuredDatasetType: + format: parquet + metadata: {} + metadata_defaults: {} + nodes: + - Target: null + id: start-node + - Target: null + id: end-node + inputs: + - binding: + Value: + Promise: + node_id: n0 + var: o0 + var: o0 + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.flyte_basics.documented_workflow.add_data + resource_type: 1 + overrides: {} + id: n0 + inputs: + - binding: + Value: + Promise: + node_id: start-node + var: data + var: data + - binding: + Value: + Promise: + node_id: start-node + var: df + var: df + metadata: + InterruptibleValue: null + name: add_data + retries: {} + outputs: + - binding: + Value: + Promise: + node_id: n0 + var: o0 + var: o0 +tasks: +- template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.documented_workflow + - task-name + - add_data + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.flyte_basics.documented_workflow.add_data + resource_type: 1 + interface: + inputs: + variables: + data: + description: data + type: + Type: + Simple: 9 + metadata: + $ref: '#/definitions/PandasdataSchema' + $schema: http://json-schema.org/draft-07/schema# + definitions: + PandasdataSchema: + additionalProperties: false + properties: + id: + default: 3 + title: id + type: integer + name: + default: Bonnie + title: name + type: string + type: object + df: + description: df + type: + Type: + StructuredDatasetType: + format: parquet + outputs: + variables: + o0: + description: o0 + type: + Type: + StructuredDatasetType: + format: parquet + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/102_core.flyte_basics.documented_workflow.google_docstring_2_wf.json b/pkg/compiler/test/testdata/snacks-core/compiled/102_core.flyte_basics.documented_workflow.google_docstring_2_wf.json new file mode 100755 index 0000000000..27f2f5d87f --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/102_core.flyte_basics.documented_workflow.google_docstring_2_wf.json @@ -0,0 +1 @@ +{"primary":{"template":{"id":{"resourceType":"WORKFLOW","name":"core.flyte_basics.documented_workflow.google_docstring"},"metadata":{},"interface":{"inputs":{"variables":{"data":{"type":{"simple":"STRUCT","metadata":{"$ref":"#/definitions/PandasdataSchema","$schema":"http://json-schema.org/draft-07/schema#","definitions":{"PandasdataSchema":{"additionalProperties":false,"properties":{"id":{"default":3,"title":"id","type":"integer"},"name":{"default":"Bonnie","title":"name","type":"string"}},"type":"object"}}}},"description":"A data class pertaining to the new record to be stored in the DataFrame"},"df":{"type":{"structuredDatasetType":{"format":"parquet"}},"description":"Pandas DataFrame"}}},"outputs":{"variables":{"o0":{"type":{"structuredDatasetType":{"format":"parquet"}},"description":"Pandas DataFrame"}}}},"nodes":[{"id":"start-node"},{"id":"end-node","inputs":[{"var":"o0","binding":{"promise":{"nodeId":"n0","var":"o0"}}}]},{"id":"n0","metadata":{"name":"add_data","retries":{}},"inputs":[{"var":"data","binding":{"promise":{"nodeId":"start-node","var":"data"}}},{"var":"df","binding":{"promise":{"nodeId":"start-node","var":"df"}}}],"taskNode":{"referenceId":{"resourceType":"TASK","name":"core.flyte_basics.documented_workflow.add_data"},"overrides":{}}}],"outputs":[{"var":"o0","binding":{"promise":{"nodeId":"n0","var":"o0"}}}],"metadataDefaults":{}},"connections":{"downstream":{"n0":{"ids":["end-node"]},"start-node":{"ids":["n0"]}},"upstream":{"end-node":{"ids":["n0"]},"n0":{"ids":["start-node"]}}}},"tasks":[{"template":{"id":{"resourceType":"TASK","name":"core.flyte_basics.documented_workflow.add_data"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"data":{"type":{"simple":"STRUCT","metadata":{"$ref":"#/definitions/PandasdataSchema","$schema":"http://json-schema.org/draft-07/schema#","definitions":{"PandasdataSchema":{"additionalProperties":false,"properties":{"id":{"default":3,"title":"id","type":"integer"},"name":{"default":"Bonnie","title":"name","type":"string"}},"type":"object"}}}},"description":"data"},"df":{"type":{"structuredDatasetType":{"format":"parquet"}},"description":"df"}}},"outputs":{"variables":{"o0":{"type":{"structuredDatasetType":{"format":"parquet"}},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.flyte_basics.documented_workflow","task-name","add_data"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}}]} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/102_core.flyte_basics.documented_workflow.google_docstring_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/102_core.flyte_basics.documented_workflow.google_docstring_2_wf.yaml new file mode 100755 index 0000000000..4c01741d60 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/102_core.flyte_basics.documented_workflow.google_docstring_2_wf.yaml @@ -0,0 +1,186 @@ +primary: + connections: + downstream: + n0: + ids: + - end-node + start-node: + ids: + - n0 + upstream: + end-node: + ids: + - n0 + n0: + ids: + - start-node + template: + id: + name: core.flyte_basics.documented_workflow.google_docstring + resource_type: 2 + interface: + inputs: + variables: + data: + description: A data class pertaining to the new record to be stored in + the DataFrame + type: + Type: + Simple: 9 + metadata: + $ref: '#/definitions/PandasdataSchema' + $schema: http://json-schema.org/draft-07/schema# + definitions: + PandasdataSchema: + additionalProperties: false + properties: + id: + default: 3 + title: id + type: integer + name: + default: Bonnie + title: name + type: string + type: object + df: + description: Pandas DataFrame + type: + Type: + StructuredDatasetType: + format: parquet + outputs: + variables: + o0: + description: Pandas DataFrame + type: + Type: + StructuredDatasetType: + format: parquet + metadata: {} + metadata_defaults: {} + nodes: + - Target: null + id: start-node + - Target: null + id: end-node + inputs: + - binding: + Value: + Promise: + node_id: n0 + var: o0 + var: o0 + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.flyte_basics.documented_workflow.add_data + resource_type: 1 + overrides: {} + id: n0 + inputs: + - binding: + Value: + Promise: + node_id: start-node + var: data + var: data + - binding: + Value: + Promise: + node_id: start-node + var: df + var: df + metadata: + InterruptibleValue: null + name: add_data + retries: {} + outputs: + - binding: + Value: + Promise: + node_id: n0 + var: o0 + var: o0 +tasks: +- template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.documented_workflow + - task-name + - add_data + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.flyte_basics.documented_workflow.add_data + resource_type: 1 + interface: + inputs: + variables: + data: + description: data + type: + Type: + Simple: 9 + metadata: + $ref: '#/definitions/PandasdataSchema' + $schema: http://json-schema.org/draft-07/schema# + definitions: + PandasdataSchema: + additionalProperties: false + properties: + id: + default: 3 + title: id + type: integer + name: + default: Bonnie + title: name + type: string + type: object + df: + description: df + type: + Type: + StructuredDatasetType: + format: parquet + outputs: + variables: + o0: + description: o0 + type: + Type: + StructuredDatasetType: + format: parquet + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/104_core.flyte_basics.files.normalize_columns_1_task.json b/pkg/compiler/test/testdata/snacks-core/compiled/104_core.flyte_basics.files.normalize_columns_1_task.json new file mode 100755 index 0000000000..5cd0439807 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/104_core.flyte_basics.files.normalize_columns_1_task.json @@ -0,0 +1 @@ +{"template":{"id":{"resourceType":"TASK","name":"core.flyte_basics.files.normalize_columns"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"column_names":{"type":{"collectionType":{"simple":"STRING"}},"description":"column_names"},"columns_to_normalize":{"type":{"collectionType":{"simple":"STRING"}},"description":"columns_to_normalize"},"csv_url":{"type":{"blob":{}},"description":"csv_url"},"output_location":{"type":{"simple":"STRING"},"description":"output_location"}}},"outputs":{"variables":{"o0":{"type":{"blob":{}},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.flyte_basics.files","task-name","normalize_columns"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/104_core.flyte_basics.files.normalize_columns_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/104_core.flyte_basics.files.normalize_columns_1_task.yaml new file mode 100755 index 0000000000..a1bc588031 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/104_core.flyte_basics.files.normalize_columns_1_task.yaml @@ -0,0 +1,76 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.files + - task-name + - normalize_columns + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.flyte_basics.files.normalize_columns + resource_type: 1 + interface: + inputs: + variables: + column_names: + description: column_names + type: + Type: + CollectionType: + Type: + Simple: 3 + columns_to_normalize: + description: columns_to_normalize + type: + Type: + CollectionType: + Type: + Simple: 3 + csv_url: + description: csv_url + type: + Type: + Blob: {} + output_location: + description: output_location + type: + Type: + Simple: 3 + outputs: + variables: + o0: + description: o0 + type: + Type: + Blob: {} + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/105_core.flyte_basics.files.normalize_csv_file_2_wf.json b/pkg/compiler/test/testdata/snacks-core/compiled/105_core.flyte_basics.files.normalize_csv_file_2_wf.json new file mode 100755 index 0000000000..26b68e68f0 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/105_core.flyte_basics.files.normalize_csv_file_2_wf.json @@ -0,0 +1 @@ +{"primary":{"template":{"id":{"resourceType":"WORKFLOW","name":"core.flyte_basics.files.normalize_csv_file"},"metadata":{},"interface":{"inputs":{"variables":{"column_names":{"type":{"collectionType":{"simple":"STRING"}},"description":"column_names"},"columns_to_normalize":{"type":{"collectionType":{"simple":"STRING"}},"description":"columns_to_normalize"},"csv_url":{"type":{"blob":{}},"description":"csv_url"},"output_location":{"type":{"simple":"STRING"},"description":"output_location"}}},"outputs":{"variables":{"o0":{"type":{"blob":{}},"description":"o0"}}}},"nodes":[{"id":"start-node"},{"id":"end-node","inputs":[{"var":"o0","binding":{"promise":{"nodeId":"n0","var":"o0"}}}]},{"id":"n0","metadata":{"name":"normalize_columns","retries":{}},"inputs":[{"var":"column_names","binding":{"promise":{"nodeId":"start-node","var":"column_names"}}},{"var":"columns_to_normalize","binding":{"promise":{"nodeId":"start-node","var":"columns_to_normalize"}}},{"var":"csv_url","binding":{"promise":{"nodeId":"start-node","var":"csv_url"}}},{"var":"output_location","binding":{"promise":{"nodeId":"start-node","var":"output_location"}}}],"taskNode":{"referenceId":{"resourceType":"TASK","name":"core.flyte_basics.files.normalize_columns"},"overrides":{}}}],"outputs":[{"var":"o0","binding":{"promise":{"nodeId":"n0","var":"o0"}}}],"metadataDefaults":{}},"connections":{"downstream":{"n0":{"ids":["end-node"]},"start-node":{"ids":["n0"]}},"upstream":{"end-node":{"ids":["n0"]},"n0":{"ids":["start-node"]}}}},"tasks":[{"template":{"id":{"resourceType":"TASK","name":"core.flyte_basics.files.normalize_columns"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"column_names":{"type":{"collectionType":{"simple":"STRING"}},"description":"column_names"},"columns_to_normalize":{"type":{"collectionType":{"simple":"STRING"}},"description":"columns_to_normalize"},"csv_url":{"type":{"blob":{}},"description":"csv_url"},"output_location":{"type":{"simple":"STRING"},"description":"output_location"}}},"outputs":{"variables":{"o0":{"type":{"blob":{}},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.flyte_basics.files","task-name","normalize_columns"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}}]} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/105_core.flyte_basics.files.normalize_csv_file_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/105_core.flyte_basics.files.normalize_csv_file_2_wf.yaml new file mode 100755 index 0000000000..7114a4db9b --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/105_core.flyte_basics.files.normalize_csv_file_2_wf.yaml @@ -0,0 +1,189 @@ +primary: + connections: + downstream: + n0: + ids: + - end-node + start-node: + ids: + - n0 + upstream: + end-node: + ids: + - n0 + n0: + ids: + - start-node + template: + id: + name: core.flyte_basics.files.normalize_csv_file + resource_type: 2 + interface: + inputs: + variables: + column_names: + description: column_names + type: + Type: + CollectionType: + Type: + Simple: 3 + columns_to_normalize: + description: columns_to_normalize + type: + Type: + CollectionType: + Type: + Simple: 3 + csv_url: + description: csv_url + type: + Type: + Blob: {} + output_location: + description: output_location + type: + Type: + Simple: 3 + outputs: + variables: + o0: + description: o0 + type: + Type: + Blob: {} + metadata: {} + metadata_defaults: {} + nodes: + - Target: null + id: start-node + - Target: null + id: end-node + inputs: + - binding: + Value: + Promise: + node_id: n0 + var: o0 + var: o0 + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.flyte_basics.files.normalize_columns + resource_type: 1 + overrides: {} + id: n0 + inputs: + - binding: + Value: + Promise: + node_id: start-node + var: column_names + var: column_names + - binding: + Value: + Promise: + node_id: start-node + var: columns_to_normalize + var: columns_to_normalize + - binding: + Value: + Promise: + node_id: start-node + var: csv_url + var: csv_url + - binding: + Value: + Promise: + node_id: start-node + var: output_location + var: output_location + metadata: + InterruptibleValue: null + name: normalize_columns + retries: {} + outputs: + - binding: + Value: + Promise: + node_id: n0 + var: o0 + var: o0 +tasks: +- template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.files + - task-name + - normalize_columns + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.flyte_basics.files.normalize_columns + resource_type: 1 + interface: + inputs: + variables: + column_names: + description: column_names + type: + Type: + CollectionType: + Type: + Simple: 3 + columns_to_normalize: + description: columns_to_normalize + type: + Type: + CollectionType: + Type: + Simple: 3 + csv_url: + description: csv_url + type: + Type: + Blob: {} + output_location: + description: output_location + type: + Type: + Simple: 3 + outputs: + variables: + o0: + description: o0 + type: + Type: + Blob: {} + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/107_core.flyte_basics.folders.download_files_1_task.json b/pkg/compiler/test/testdata/snacks-core/compiled/107_core.flyte_basics.folders.download_files_1_task.json new file mode 100755 index 0000000000..cf7cce3770 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/107_core.flyte_basics.folders.download_files_1_task.json @@ -0,0 +1 @@ +{"template":{"id":{"resourceType":"TASK","name":"core.flyte_basics.folders.download_files"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"csv_urls":{"type":{"collectionType":{"simple":"STRING"}},"description":"csv_urls"}}},"outputs":{"variables":{"o0":{"type":{"blob":{"dimensionality":"MULTIPART"}},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.flyte_basics.folders","task-name","download_files"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/107_core.flyte_basics.folders.download_files_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/107_core.flyte_basics.folders.download_files_1_task.yaml new file mode 100755 index 0000000000..a791a8bb62 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/107_core.flyte_basics.folders.download_files_1_task.yaml @@ -0,0 +1,60 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.folders + - task-name + - download_files + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.flyte_basics.folders.download_files + resource_type: 1 + interface: + inputs: + variables: + csv_urls: + description: csv_urls + type: + Type: + CollectionType: + Type: + Simple: 3 + outputs: + variables: + o0: + description: o0 + type: + Type: + Blob: + dimensionality: 1 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/108_core.flyte_basics.folders.normalize_all_files_1_task.json b/pkg/compiler/test/testdata/snacks-core/compiled/108_core.flyte_basics.folders.normalize_all_files_1_task.json new file mode 100755 index 0000000000..ca9245d4b6 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/108_core.flyte_basics.folders.normalize_all_files_1_task.json @@ -0,0 +1 @@ +{"template":{"id":{"resourceType":"TASK","name":"core.flyte_basics.folders.normalize_all_files"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"columns_metadata":{"type":{"collectionType":{"collectionType":{"simple":"STRING"}}},"description":"columns_metadata"},"columns_to_normalize_metadata":{"type":{"collectionType":{"collectionType":{"simple":"STRING"}}},"description":"columns_to_normalize_metadata"},"csv_files_dir":{"type":{"blob":{"dimensionality":"MULTIPART"}},"description":"csv_files_dir"}}},"outputs":{"variables":{"o0":{"type":{"blob":{"dimensionality":"MULTIPART"}},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.flyte_basics.folders","task-name","normalize_all_files"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/108_core.flyte_basics.folders.normalize_all_files_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/108_core.flyte_basics.folders.normalize_all_files_1_task.yaml new file mode 100755 index 0000000000..9aee2ad720 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/108_core.flyte_basics.folders.normalize_all_files_1_task.yaml @@ -0,0 +1,77 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.folders + - task-name + - normalize_all_files + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.flyte_basics.folders.normalize_all_files + resource_type: 1 + interface: + inputs: + variables: + columns_metadata: + description: columns_metadata + type: + Type: + CollectionType: + Type: + CollectionType: + Type: + Simple: 3 + columns_to_normalize_metadata: + description: columns_to_normalize_metadata + type: + Type: + CollectionType: + Type: + CollectionType: + Type: + Simple: 3 + csv_files_dir: + description: csv_files_dir + type: + Type: + Blob: + dimensionality: 1 + outputs: + variables: + o0: + description: o0 + type: + Type: + Blob: + dimensionality: 1 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/109_core.flyte_basics.folders.download_and_normalize_csv_files_2_wf.json b/pkg/compiler/test/testdata/snacks-core/compiled/109_core.flyte_basics.folders.download_and_normalize_csv_files_2_wf.json new file mode 100755 index 0000000000..3d456006bd --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/109_core.flyte_basics.folders.download_and_normalize_csv_files_2_wf.json @@ -0,0 +1 @@ +{"primary":{"template":{"id":{"resourceType":"WORKFLOW","name":"core.flyte_basics.folders.download_and_normalize_csv_files"},"metadata":{},"interface":{"inputs":{"variables":{"columns_metadata":{"type":{"collectionType":{"collectionType":{"simple":"STRING"}}},"description":"columns_metadata"},"columns_to_normalize_metadata":{"type":{"collectionType":{"collectionType":{"simple":"STRING"}}},"description":"columns_to_normalize_metadata"},"csv_urls":{"type":{"collectionType":{"simple":"STRING"}},"description":"csv_urls"}}},"outputs":{"variables":{"o0":{"type":{"blob":{"dimensionality":"MULTIPART"}},"description":"o0"}}}},"nodes":[{"id":"start-node"},{"id":"end-node","inputs":[{"var":"o0","binding":{"promise":{"nodeId":"n1","var":"o0"}}}]},{"id":"n0","metadata":{"name":"download_files","retries":{}},"inputs":[{"var":"csv_urls","binding":{"promise":{"nodeId":"start-node","var":"csv_urls"}}}],"taskNode":{"referenceId":{"resourceType":"TASK","name":"core.flyte_basics.folders.download_files"},"overrides":{}}},{"id":"n1","metadata":{"name":"normalize_all_files","retries":{}},"inputs":[{"var":"columns_metadata","binding":{"promise":{"nodeId":"start-node","var":"columns_metadata"}}},{"var":"columns_to_normalize_metadata","binding":{"promise":{"nodeId":"start-node","var":"columns_to_normalize_metadata"}}},{"var":"csv_files_dir","binding":{"promise":{"nodeId":"n0","var":"o0"}}}],"upstreamNodeIds":["n0"],"taskNode":{"referenceId":{"resourceType":"TASK","name":"core.flyte_basics.folders.normalize_all_files"},"overrides":{}}}],"outputs":[{"var":"o0","binding":{"promise":{"nodeId":"n1","var":"o0"}}}],"metadataDefaults":{}},"connections":{"downstream":{"n0":{"ids":["n1"]},"n1":{"ids":["end-node"]},"start-node":{"ids":["n0","n1"]}},"upstream":{"end-node":{"ids":["n1"]},"n0":{"ids":["start-node"]},"n1":{"ids":["n0","start-node"]}}}},"tasks":[{"template":{"id":{"resourceType":"TASK","name":"core.flyte_basics.folders.download_files"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"csv_urls":{"type":{"collectionType":{"simple":"STRING"}},"description":"csv_urls"}}},"outputs":{"variables":{"o0":{"type":{"blob":{"dimensionality":"MULTIPART"}},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.flyte_basics.folders","task-name","download_files"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},{"template":{"id":{"resourceType":"TASK","name":"core.flyte_basics.folders.normalize_all_files"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"columns_metadata":{"type":{"collectionType":{"collectionType":{"simple":"STRING"}}},"description":"columns_metadata"},"columns_to_normalize_metadata":{"type":{"collectionType":{"collectionType":{"simple":"STRING"}}},"description":"columns_to_normalize_metadata"},"csv_files_dir":{"type":{"blob":{"dimensionality":"MULTIPART"}},"description":"csv_files_dir"}}},"outputs":{"variables":{"o0":{"type":{"blob":{"dimensionality":"MULTIPART"}},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.flyte_basics.folders","task-name","normalize_all_files"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}}]} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/109_core.flyte_basics.folders.download_and_normalize_csv_files_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/109_core.flyte_basics.folders.download_and_normalize_csv_files_2_wf.yaml new file mode 100755 index 0000000000..b2e00b94bb --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/109_core.flyte_basics.folders.download_and_normalize_csv_files_2_wf.yaml @@ -0,0 +1,275 @@ +primary: + connections: + downstream: + n0: + ids: + - n1 + n1: + ids: + - end-node + start-node: + ids: + - n0 + - n1 + upstream: + end-node: + ids: + - n1 + n0: + ids: + - start-node + n1: + ids: + - n0 + - start-node + template: + id: + name: core.flyte_basics.folders.download_and_normalize_csv_files + resource_type: 2 + interface: + inputs: + variables: + columns_metadata: + description: columns_metadata + type: + Type: + CollectionType: + Type: + CollectionType: + Type: + Simple: 3 + columns_to_normalize_metadata: + description: columns_to_normalize_metadata + type: + Type: + CollectionType: + Type: + CollectionType: + Type: + Simple: 3 + csv_urls: + description: csv_urls + type: + Type: + CollectionType: + Type: + Simple: 3 + outputs: + variables: + o0: + description: o0 + type: + Type: + Blob: + dimensionality: 1 + metadata: {} + metadata_defaults: {} + nodes: + - Target: null + id: start-node + - Target: null + id: end-node + inputs: + - binding: + Value: + Promise: + node_id: n1 + var: o0 + var: o0 + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.flyte_basics.folders.download_files + resource_type: 1 + overrides: {} + id: n0 + inputs: + - binding: + Value: + Promise: + node_id: start-node + var: csv_urls + var: csv_urls + metadata: + InterruptibleValue: null + name: download_files + retries: {} + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.flyte_basics.folders.normalize_all_files + resource_type: 1 + overrides: {} + id: n1 + inputs: + - binding: + Value: + Promise: + node_id: start-node + var: columns_metadata + var: columns_metadata + - binding: + Value: + Promise: + node_id: start-node + var: columns_to_normalize_metadata + var: columns_to_normalize_metadata + - binding: + Value: + Promise: + node_id: n0 + var: o0 + var: csv_files_dir + metadata: + InterruptibleValue: null + name: normalize_all_files + retries: {} + upstream_node_ids: + - n0 + outputs: + - binding: + Value: + Promise: + node_id: n1 + var: o0 + var: o0 +tasks: +- template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.folders + - task-name + - download_files + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.flyte_basics.folders.download_files + resource_type: 1 + interface: + inputs: + variables: + csv_urls: + description: csv_urls + type: + Type: + CollectionType: + Type: + Simple: 3 + outputs: + variables: + o0: + description: o0 + type: + Type: + Blob: + dimensionality: 1 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task +- template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.folders + - task-name + - normalize_all_files + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.flyte_basics.folders.normalize_all_files + resource_type: 1 + interface: + inputs: + variables: + columns_metadata: + description: columns_metadata + type: + Type: + CollectionType: + Type: + CollectionType: + Type: + Simple: 3 + columns_to_normalize_metadata: + description: columns_to_normalize_metadata + type: + Type: + CollectionType: + Type: + CollectionType: + Type: + Simple: 3 + csv_files_dir: + description: csv_files_dir + type: + Type: + Blob: + dimensionality: 1 + outputs: + variables: + o0: + description: o0 + type: + Type: + Blob: + dimensionality: 1 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/111_core.flyte_basics.hello_world.say_hello_1_task.json b/pkg/compiler/test/testdata/snacks-core/compiled/111_core.flyte_basics.hello_world.say_hello_1_task.json new file mode 100755 index 0000000000..1708ae0d74 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/111_core.flyte_basics.hello_world.say_hello_1_task.json @@ -0,0 +1 @@ +{"template":{"id":{"resourceType":"TASK","name":"core.flyte_basics.hello_world.say_hello"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{},"outputs":{"variables":{"o0":{"type":{"simple":"STRING"},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.flyte_basics.hello_world","task-name","say_hello"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/111_core.flyte_basics.hello_world.say_hello_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/111_core.flyte_basics.hello_world.say_hello_1_task.yaml new file mode 100755 index 0000000000..5f7e8df0fd --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/111_core.flyte_basics.hello_world.say_hello_1_task.yaml @@ -0,0 +1,51 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.hello_world + - task-name + - say_hello + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.flyte_basics.hello_world.say_hello + resource_type: 1 + interface: + inputs: {} + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 3 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/112_core.flyte_basics.hello_world.my_wf_2_wf.json b/pkg/compiler/test/testdata/snacks-core/compiled/112_core.flyte_basics.hello_world.my_wf_2_wf.json new file mode 100755 index 0000000000..8471d46530 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/112_core.flyte_basics.hello_world.my_wf_2_wf.json @@ -0,0 +1 @@ +{"primary":{"template":{"id":{"resourceType":"WORKFLOW","name":"core.flyte_basics.hello_world.my_wf"},"metadata":{},"interface":{"inputs":{},"outputs":{"variables":{"o0":{"type":{"simple":"STRING"},"description":"o0"}}}},"nodes":[{"id":"start-node"},{"id":"end-node","inputs":[{"var":"o0","binding":{"promise":{"nodeId":"n0","var":"o0"}}}]},{"id":"n0","metadata":{"name":"say_hello","retries":{}},"taskNode":{"referenceId":{"resourceType":"TASK","name":"core.flyte_basics.hello_world.say_hello"},"overrides":{}}}],"outputs":[{"var":"o0","binding":{"promise":{"nodeId":"n0","var":"o0"}}}],"metadataDefaults":{}},"connections":{"downstream":{"n0":{"ids":["end-node"]},"start-node":{"ids":["n0"]}},"upstream":{"end-node":{"ids":["n0"]},"n0":{"ids":["start-node"]}}}},"tasks":[{"template":{"id":{"resourceType":"TASK","name":"core.flyte_basics.hello_world.say_hello"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{},"outputs":{"variables":{"o0":{"type":{"simple":"STRING"},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.flyte_basics.hello_world","task-name","say_hello"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}}]} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/112_core.flyte_basics.hello_world.my_wf_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/112_core.flyte_basics.hello_world.my_wf_2_wf.yaml new file mode 100755 index 0000000000..854d8d3f83 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/112_core.flyte_basics.hello_world.my_wf_2_wf.yaml @@ -0,0 +1,114 @@ +primary: + connections: + downstream: + n0: + ids: + - end-node + start-node: + ids: + - n0 + upstream: + end-node: + ids: + - n0 + n0: + ids: + - start-node + template: + id: + name: core.flyte_basics.hello_world.my_wf + resource_type: 2 + interface: + inputs: {} + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 3 + metadata: {} + metadata_defaults: {} + nodes: + - Target: null + id: start-node + - Target: null + id: end-node + inputs: + - binding: + Value: + Promise: + node_id: n0 + var: o0 + var: o0 + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.flyte_basics.hello_world.say_hello + resource_type: 1 + overrides: {} + id: n0 + metadata: + InterruptibleValue: null + name: say_hello + retries: {} + outputs: + - binding: + Value: + Promise: + node_id: n0 + var: o0 + var: o0 +tasks: +- template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.hello_world + - task-name + - say_hello + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.flyte_basics.hello_world.say_hello + resource_type: 1 + interface: + inputs: {} + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 3 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/114_core.flyte_basics.imperative_wf_style.t1_1_task.json b/pkg/compiler/test/testdata/snacks-core/compiled/114_core.flyte_basics.imperative_wf_style.t1_1_task.json new file mode 100755 index 0000000000..8b27eb7506 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/114_core.flyte_basics.imperative_wf_style.t1_1_task.json @@ -0,0 +1 @@ +{"template":{"id":{"resourceType":"TASK","name":"core.flyte_basics.imperative_wf_style.t1"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"a":{"type":{"simple":"STRING"},"description":"a"}}},"outputs":{"variables":{"o0":{"type":{"simple":"STRING"},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.flyte_basics.imperative_wf_style","task-name","t1"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/114_core.flyte_basics.imperative_wf_style.t1_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/114_core.flyte_basics.imperative_wf_style.t1_1_task.yaml new file mode 100755 index 0000000000..1ca8539fcd --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/114_core.flyte_basics.imperative_wf_style.t1_1_task.yaml @@ -0,0 +1,57 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.imperative_wf_style + - task-name + - t1 + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.flyte_basics.imperative_wf_style.t1 + resource_type: 1 + interface: + inputs: + variables: + a: + description: a + type: + Type: + Simple: 3 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 3 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/115_core.flyte_basics.imperative_wf_style.t2_1_task.json b/pkg/compiler/test/testdata/snacks-core/compiled/115_core.flyte_basics.imperative_wf_style.t2_1_task.json new file mode 100755 index 0000000000..9c819da2cd --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/115_core.flyte_basics.imperative_wf_style.t2_1_task.json @@ -0,0 +1 @@ +{"template":{"id":{"resourceType":"TASK","name":"core.flyte_basics.imperative_wf_style.t2"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{},"outputs":{}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.flyte_basics.imperative_wf_style","task-name","t2"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/115_core.flyte_basics.imperative_wf_style.t2_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/115_core.flyte_basics.imperative_wf_style.t2_1_task.yaml new file mode 100755 index 0000000000..a12ef17894 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/115_core.flyte_basics.imperative_wf_style.t2_1_task.yaml @@ -0,0 +1,45 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.imperative_wf_style + - task-name + - t2 + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.flyte_basics.imperative_wf_style.t2 + resource_type: 1 + interface: + inputs: {} + outputs: {} + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/116_core.flyte_basics.imperative_wf_style.t3_1_task.json b/pkg/compiler/test/testdata/snacks-core/compiled/116_core.flyte_basics.imperative_wf_style.t3_1_task.json new file mode 100755 index 0000000000..d3c8187abe --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/116_core.flyte_basics.imperative_wf_style.t3_1_task.json @@ -0,0 +1 @@ +{"template":{"id":{"resourceType":"TASK","name":"core.flyte_basics.imperative_wf_style.t3"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"a":{"type":{"collectionType":{"simple":"STRING"}},"description":"a"}}},"outputs":{"variables":{"o0":{"type":{"simple":"STRING"},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.flyte_basics.imperative_wf_style","task-name","t3"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/116_core.flyte_basics.imperative_wf_style.t3_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/116_core.flyte_basics.imperative_wf_style.t3_1_task.yaml new file mode 100755 index 0000000000..ce91ab3341 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/116_core.flyte_basics.imperative_wf_style.t3_1_task.yaml @@ -0,0 +1,59 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.imperative_wf_style + - task-name + - t3 + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.flyte_basics.imperative_wf_style.t3 + resource_type: 1 + interface: + inputs: + variables: + a: + description: a + type: + Type: + CollectionType: + Type: + Simple: 3 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 3 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/117_my.imperative.workflow.example_2_wf.json b/pkg/compiler/test/testdata/snacks-core/compiled/117_my.imperative.workflow.example_2_wf.json new file mode 100755 index 0000000000..5699bf6cf2 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/117_my.imperative.workflow.example_2_wf.json @@ -0,0 +1 @@ +{"primary":{"template":{"id":{"resourceType":"WORKFLOW","name":"my.imperative.workflow.example"},"metadata":{},"interface":{"inputs":{"variables":{"in1":{"type":{"simple":"STRING"},"description":"in1"},"in2":{"type":{"simple":"STRING"},"description":"in2"}}},"outputs":{"variables":{"output_from_t1":{"type":{"simple":"STRING"},"description":"output_from_t1"},"output_list":{"type":{"collectionType":{"simple":"STRING"}},"description":"output_list"}}}},"nodes":[{"id":"start-node"},{"id":"end-node","inputs":[{"var":"output_from_t1","binding":{"promise":{"nodeId":"n0","var":"o0"}}},{"var":"output_list","binding":{"collection":{"bindings":[{"promise":{"nodeId":"n0","var":"o0"}},{"promise":{"nodeId":"n2","var":"o0"}}]}}}]},{"id":"n0","metadata":{"name":"t1","retries":{}},"inputs":[{"var":"a","binding":{"promise":{"nodeId":"start-node","var":"in1"}}}],"taskNode":{"referenceId":{"resourceType":"TASK","name":"core.flyte_basics.imperative_wf_style.t1"},"overrides":{}}},{"id":"n1","metadata":{"name":"t2","retries":{}},"taskNode":{"referenceId":{"resourceType":"TASK","name":"core.flyte_basics.imperative_wf_style.t2"},"overrides":{}}},{"id":"n2","metadata":{"name":"t3","retries":{}},"inputs":[{"var":"a","binding":{"collection":{"bindings":[{"promise":{"nodeId":"start-node","var":"in1"}},{"promise":{"nodeId":"start-node","var":"in2"}}]}}}],"taskNode":{"referenceId":{"resourceType":"TASK","name":"core.flyte_basics.imperative_wf_style.t3"},"overrides":{}}}],"outputs":[{"var":"output_from_t1","binding":{"promise":{"nodeId":"n0","var":"o0"}}},{"var":"output_list","binding":{"collection":{"bindings":[{"promise":{"nodeId":"n0","var":"o0"}},{"promise":{"nodeId":"n2","var":"o0"}}]}}}],"metadataDefaults":{}},"connections":{"downstream":{"n0":{"ids":["end-node"]},"n1":{"ids":["end-node"]},"n2":{"ids":["end-node"]},"start-node":{"ids":["n0","n1","n2"]}},"upstream":{"end-node":{"ids":["n0","n1","n2"]},"n0":{"ids":["start-node"]},"n1":{"ids":["start-node"]},"n2":{"ids":["start-node"]}}}},"tasks":[{"template":{"id":{"resourceType":"TASK","name":"core.flyte_basics.imperative_wf_style.t1"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"a":{"type":{"simple":"STRING"},"description":"a"}}},"outputs":{"variables":{"o0":{"type":{"simple":"STRING"},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.flyte_basics.imperative_wf_style","task-name","t1"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},{"template":{"id":{"resourceType":"TASK","name":"core.flyte_basics.imperative_wf_style.t2"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{},"outputs":{}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.flyte_basics.imperative_wf_style","task-name","t2"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},{"template":{"id":{"resourceType":"TASK","name":"core.flyte_basics.imperative_wf_style.t3"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"a":{"type":{"collectionType":{"simple":"STRING"}},"description":"a"}}},"outputs":{"variables":{"o0":{"type":{"simple":"STRING"},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.flyte_basics.imperative_wf_style","task-name","t3"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}}]} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/117_my.imperative.workflow.example_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/117_my.imperative.workflow.example_2_wf.yaml new file mode 100755 index 0000000000..995b858161 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/117_my.imperative.workflow.example_2_wf.yaml @@ -0,0 +1,329 @@ +primary: + connections: + downstream: + n0: + ids: + - end-node + n1: + ids: + - end-node + n2: + ids: + - end-node + start-node: + ids: + - n0 + - n1 + - n2 + upstream: + end-node: + ids: + - n0 + - n1 + - n2 + n0: + ids: + - start-node + n1: + ids: + - start-node + n2: + ids: + - start-node + template: + id: + name: my.imperative.workflow.example + resource_type: 2 + interface: + inputs: + variables: + in1: + description: in1 + type: + Type: + Simple: 3 + in2: + description: in2 + type: + Type: + Simple: 3 + outputs: + variables: + output_from_t1: + description: output_from_t1 + type: + Type: + Simple: 3 + output_list: + description: output_list + type: + Type: + CollectionType: + Type: + Simple: 3 + metadata: {} + metadata_defaults: {} + nodes: + - Target: null + id: start-node + - Target: null + id: end-node + inputs: + - binding: + Value: + Promise: + node_id: n0 + var: o0 + var: output_from_t1 + - binding: + Value: + Collection: + bindings: + - Value: + Promise: + node_id: n0 + var: o0 + - Value: + Promise: + node_id: n2 + var: o0 + var: output_list + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.flyte_basics.imperative_wf_style.t1 + resource_type: 1 + overrides: {} + id: n0 + inputs: + - binding: + Value: + Promise: + node_id: start-node + var: in1 + var: a + metadata: + InterruptibleValue: null + name: t1 + retries: {} + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.flyte_basics.imperative_wf_style.t2 + resource_type: 1 + overrides: {} + id: n1 + metadata: + InterruptibleValue: null + name: t2 + retries: {} + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.flyte_basics.imperative_wf_style.t3 + resource_type: 1 + overrides: {} + id: n2 + inputs: + - binding: + Value: + Collection: + bindings: + - Value: + Promise: + node_id: start-node + var: in1 + - Value: + Promise: + node_id: start-node + var: in2 + var: a + metadata: + InterruptibleValue: null + name: t3 + retries: {} + outputs: + - binding: + Value: + Promise: + node_id: n0 + var: o0 + var: output_from_t1 + - binding: + Value: + Collection: + bindings: + - Value: + Promise: + node_id: n0 + var: o0 + - Value: + Promise: + node_id: n2 + var: o0 + var: output_list +tasks: +- template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.imperative_wf_style + - task-name + - t1 + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.flyte_basics.imperative_wf_style.t1 + resource_type: 1 + interface: + inputs: + variables: + a: + description: a + type: + Type: + Simple: 3 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 3 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task +- template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.imperative_wf_style + - task-name + - t2 + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.flyte_basics.imperative_wf_style.t2 + resource_type: 1 + interface: + inputs: {} + outputs: {} + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task +- template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.imperative_wf_style + - task-name + - t3 + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.flyte_basics.imperative_wf_style.t3 + resource_type: 1 + interface: + inputs: + variables: + a: + description: a + type: + Type: + CollectionType: + Type: + Simple: 3 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 3 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/119_core.flyte_basics.lp.square_1_task.json b/pkg/compiler/test/testdata/snacks-core/compiled/119_core.flyte_basics.lp.square_1_task.json new file mode 100755 index 0000000000..d18526292c --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/119_core.flyte_basics.lp.square_1_task.json @@ -0,0 +1 @@ +{"template":{"id":{"resourceType":"TASK","name":"core.flyte_basics.lp.square"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"val":{"type":{"simple":"INTEGER"},"description":"val"}}},"outputs":{"variables":{"o0":{"type":{"simple":"INTEGER"},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.flyte_basics.lp","task-name","square"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/119_core.flyte_basics.lp.square_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/119_core.flyte_basics.lp.square_1_task.yaml new file mode 100755 index 0000000000..c124729504 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/119_core.flyte_basics.lp.square_1_task.yaml @@ -0,0 +1,57 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.lp + - task-name + - square + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.flyte_basics.lp.square + resource_type: 1 + interface: + inputs: + variables: + val: + description: val + type: + Type: + Simple: 1 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 1 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/120_core.flyte_basics.lp.my_wf_2_wf.json b/pkg/compiler/test/testdata/snacks-core/compiled/120_core.flyte_basics.lp.my_wf_2_wf.json new file mode 100755 index 0000000000..d2dc1524a2 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/120_core.flyte_basics.lp.my_wf_2_wf.json @@ -0,0 +1 @@ +{"primary":{"template":{"id":{"resourceType":"WORKFLOW","name":"core.flyte_basics.lp.my_wf"},"metadata":{},"interface":{"inputs":{"variables":{"val":{"type":{"simple":"INTEGER"},"description":"val"}}},"outputs":{"variables":{"o0":{"type":{"simple":"INTEGER"},"description":"o0"}}}},"nodes":[{"id":"start-node"},{"id":"end-node","inputs":[{"var":"o0","binding":{"promise":{"nodeId":"n0","var":"o0"}}}]},{"id":"n0","metadata":{"name":"square","retries":{}},"inputs":[{"var":"val","binding":{"promise":{"nodeId":"start-node","var":"val"}}}],"taskNode":{"referenceId":{"resourceType":"TASK","name":"core.flyte_basics.lp.square"},"overrides":{}}}],"outputs":[{"var":"o0","binding":{"promise":{"nodeId":"n0","var":"o0"}}}],"metadataDefaults":{}},"connections":{"downstream":{"n0":{"ids":["end-node"]},"start-node":{"ids":["n0"]}},"upstream":{"end-node":{"ids":["n0"]},"n0":{"ids":["start-node"]}}}},"tasks":[{"template":{"id":{"resourceType":"TASK","name":"core.flyte_basics.lp.square"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"val":{"type":{"simple":"INTEGER"},"description":"val"}}},"outputs":{"variables":{"o0":{"type":{"simple":"INTEGER"},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.flyte_basics.lp","task-name","square"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}}]} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/120_core.flyte_basics.lp.my_wf_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/120_core.flyte_basics.lp.my_wf_2_wf.yaml new file mode 100755 index 0000000000..e65a21d916 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/120_core.flyte_basics.lp.my_wf_2_wf.yaml @@ -0,0 +1,133 @@ +primary: + connections: + downstream: + n0: + ids: + - end-node + start-node: + ids: + - n0 + upstream: + end-node: + ids: + - n0 + n0: + ids: + - start-node + template: + id: + name: core.flyte_basics.lp.my_wf + resource_type: 2 + interface: + inputs: + variables: + val: + description: val + type: + Type: + Simple: 1 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 1 + metadata: {} + metadata_defaults: {} + nodes: + - Target: null + id: start-node + - Target: null + id: end-node + inputs: + - binding: + Value: + Promise: + node_id: n0 + var: o0 + var: o0 + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.flyte_basics.lp.square + resource_type: 1 + overrides: {} + id: n0 + inputs: + - binding: + Value: + Promise: + node_id: start-node + var: val + var: val + metadata: + InterruptibleValue: null + name: square + retries: {} + outputs: + - binding: + Value: + Promise: + node_id: n0 + var: o0 + var: o0 +tasks: +- template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.lp + - task-name + - square + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.flyte_basics.lp.square + resource_type: 1 + interface: + inputs: + variables: + val: + description: val + type: + Type: + Simple: 1 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 1 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/124_core.flyte_basics.lp.greet_1_task.json b/pkg/compiler/test/testdata/snacks-core/compiled/124_core.flyte_basics.lp.greet_1_task.json new file mode 100755 index 0000000000..19d7e60184 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/124_core.flyte_basics.lp.greet_1_task.json @@ -0,0 +1 @@ +{"template":{"id":{"resourceType":"TASK","name":"core.flyte_basics.lp.greet"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"am":{"type":{"simple":"BOOLEAN"},"description":"am"},"day_of_week":{"type":{"simple":"STRING"},"description":"day_of_week"},"number":{"type":{"simple":"INTEGER"},"description":"number"}}},"outputs":{"variables":{"o0":{"type":{"simple":"STRING"},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.flyte_basics.lp","task-name","greet"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/124_core.flyte_basics.lp.greet_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/124_core.flyte_basics.lp.greet_1_task.yaml new file mode 100755 index 0000000000..e1f4b461de --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/124_core.flyte_basics.lp.greet_1_task.yaml @@ -0,0 +1,67 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.lp + - task-name + - greet + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.flyte_basics.lp.greet + resource_type: 1 + interface: + inputs: + variables: + am: + description: am + type: + Type: + Simple: 4 + day_of_week: + description: day_of_week + type: + Type: + Simple: 3 + number: + description: number + type: + Type: + Simple: 1 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 3 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/125_core.flyte_basics.lp.go_greet_2_wf.json b/pkg/compiler/test/testdata/snacks-core/compiled/125_core.flyte_basics.lp.go_greet_2_wf.json new file mode 100755 index 0000000000..ea8bdf3aba --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/125_core.flyte_basics.lp.go_greet_2_wf.json @@ -0,0 +1 @@ +{"primary":{"template":{"id":{"resourceType":"WORKFLOW","name":"core.flyte_basics.lp.go_greet"},"metadata":{},"interface":{"inputs":{"variables":{"am":{"type":{"simple":"BOOLEAN"},"description":"am"},"day_of_week":{"type":{"simple":"STRING"},"description":"day_of_week"},"number":{"type":{"simple":"INTEGER"},"description":"number"}}},"outputs":{"variables":{"o0":{"type":{"simple":"STRING"},"description":"o0"}}}},"nodes":[{"id":"start-node"},{"id":"end-node","inputs":[{"var":"o0","binding":{"promise":{"nodeId":"n0","var":"o0"}}}]},{"id":"n0","metadata":{"name":"greet","retries":{}},"inputs":[{"var":"am","binding":{"promise":{"nodeId":"start-node","var":"am"}}},{"var":"day_of_week","binding":{"promise":{"nodeId":"start-node","var":"day_of_week"}}},{"var":"number","binding":{"promise":{"nodeId":"start-node","var":"number"}}}],"taskNode":{"referenceId":{"resourceType":"TASK","name":"core.flyte_basics.lp.greet"},"overrides":{}}}],"outputs":[{"var":"o0","binding":{"promise":{"nodeId":"n0","var":"o0"}}}],"metadataDefaults":{}},"connections":{"downstream":{"n0":{"ids":["end-node"]},"start-node":{"ids":["n0"]}},"upstream":{"end-node":{"ids":["n0"]},"n0":{"ids":["start-node"]}}}},"tasks":[{"template":{"id":{"resourceType":"TASK","name":"core.flyte_basics.lp.greet"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"am":{"type":{"simple":"BOOLEAN"},"description":"am"},"day_of_week":{"type":{"simple":"STRING"},"description":"day_of_week"},"number":{"type":{"simple":"INTEGER"},"description":"number"}}},"outputs":{"variables":{"o0":{"type":{"simple":"STRING"},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.flyte_basics.lp","task-name","greet"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}}]} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/125_core.flyte_basics.lp.go_greet_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/125_core.flyte_basics.lp.go_greet_2_wf.yaml new file mode 100755 index 0000000000..108e7c2f3a --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/125_core.flyte_basics.lp.go_greet_2_wf.yaml @@ -0,0 +1,165 @@ +primary: + connections: + downstream: + n0: + ids: + - end-node + start-node: + ids: + - n0 + upstream: + end-node: + ids: + - n0 + n0: + ids: + - start-node + template: + id: + name: core.flyte_basics.lp.go_greet + resource_type: 2 + interface: + inputs: + variables: + am: + description: am + type: + Type: + Simple: 4 + day_of_week: + description: day_of_week + type: + Type: + Simple: 3 + number: + description: number + type: + Type: + Simple: 1 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 3 + metadata: {} + metadata_defaults: {} + nodes: + - Target: null + id: start-node + - Target: null + id: end-node + inputs: + - binding: + Value: + Promise: + node_id: n0 + var: o0 + var: o0 + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.flyte_basics.lp.greet + resource_type: 1 + overrides: {} + id: n0 + inputs: + - binding: + Value: + Promise: + node_id: start-node + var: am + var: am + - binding: + Value: + Promise: + node_id: start-node + var: day_of_week + var: day_of_week + - binding: + Value: + Promise: + node_id: start-node + var: number + var: number + metadata: + InterruptibleValue: null + name: greet + retries: {} + outputs: + - binding: + Value: + Promise: + node_id: n0 + var: o0 + var: o0 +tasks: +- template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.lp + - task-name + - greet + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.flyte_basics.lp.greet + resource_type: 1 + interface: + inputs: + variables: + am: + description: am + type: + Type: + Simple: 4 + day_of_week: + description: day_of_week + type: + Type: + Simple: 3 + number: + description: number + type: + Type: + Simple: 1 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 3 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/128_core.flyte_basics.named_outputs.say_hello_1_task.json b/pkg/compiler/test/testdata/snacks-core/compiled/128_core.flyte_basics.named_outputs.say_hello_1_task.json new file mode 100755 index 0000000000..61a70d7d52 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/128_core.flyte_basics.named_outputs.say_hello_1_task.json @@ -0,0 +1 @@ +{"template":{"id":{"resourceType":"TASK","name":"core.flyte_basics.named_outputs.say_hello"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{},"outputs":{"variables":{"greet":{"type":{"simple":"STRING"},"description":"greet"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.flyte_basics.named_outputs","task-name","say_hello"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/128_core.flyte_basics.named_outputs.say_hello_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/128_core.flyte_basics.named_outputs.say_hello_1_task.yaml new file mode 100755 index 0000000000..6a66d8ed7c --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/128_core.flyte_basics.named_outputs.say_hello_1_task.yaml @@ -0,0 +1,51 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.named_outputs + - task-name + - say_hello + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.flyte_basics.named_outputs.say_hello + resource_type: 1 + interface: + inputs: {} + outputs: + variables: + greet: + description: greet + type: + Type: + Simple: 3 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/129_core.flyte_basics.named_outputs.my_wf_2_wf.json b/pkg/compiler/test/testdata/snacks-core/compiled/129_core.flyte_basics.named_outputs.my_wf_2_wf.json new file mode 100755 index 0000000000..4e727dbebc --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/129_core.flyte_basics.named_outputs.my_wf_2_wf.json @@ -0,0 +1 @@ +{"primary":{"template":{"id":{"resourceType":"WORKFLOW","name":"core.flyte_basics.named_outputs.my_wf"},"metadata":{},"interface":{"inputs":{},"outputs":{"variables":{"greet1":{"type":{"simple":"STRING"},"description":"greet1"},"greet2":{"type":{"simple":"STRING"},"description":"greet2"}}}},"nodes":[{"id":"start-node"},{"id":"end-node","inputs":[{"var":"greet1","binding":{"promise":{"nodeId":"n0","var":"greet"}}},{"var":"greet2","binding":{"promise":{"nodeId":"n1","var":"greet"}}}]},{"id":"n0","metadata":{"name":"say_hello","retries":{}},"taskNode":{"referenceId":{"resourceType":"TASK","name":"core.flyte_basics.named_outputs.say_hello"},"overrides":{}}},{"id":"n1","metadata":{"name":"say_hello","retries":{}},"taskNode":{"referenceId":{"resourceType":"TASK","name":"core.flyte_basics.named_outputs.say_hello"},"overrides":{}}}],"outputs":[{"var":"greet1","binding":{"promise":{"nodeId":"n0","var":"greet"}}},{"var":"greet2","binding":{"promise":{"nodeId":"n1","var":"greet"}}}],"metadataDefaults":{}},"connections":{"downstream":{"n0":{"ids":["end-node"]},"n1":{"ids":["end-node"]},"start-node":{"ids":["n0","n1"]}},"upstream":{"end-node":{"ids":["n0","n1"]},"n0":{"ids":["start-node"]},"n1":{"ids":["start-node"]}}}},"tasks":[{"template":{"id":{"resourceType":"TASK","name":"core.flyte_basics.named_outputs.say_hello"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{},"outputs":{"variables":{"greet":{"type":{"simple":"STRING"},"description":"greet"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.flyte_basics.named_outputs","task-name","say_hello"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}}]} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/129_core.flyte_basics.named_outputs.my_wf_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/129_core.flyte_basics.named_outputs.my_wf_2_wf.yaml new file mode 100755 index 0000000000..98ca530c6f --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/129_core.flyte_basics.named_outputs.my_wf_2_wf.yaml @@ -0,0 +1,151 @@ +primary: + connections: + downstream: + n0: + ids: + - end-node + n1: + ids: + - end-node + start-node: + ids: + - n0 + - n1 + upstream: + end-node: + ids: + - n0 + - n1 + n0: + ids: + - start-node + n1: + ids: + - start-node + template: + id: + name: core.flyte_basics.named_outputs.my_wf + resource_type: 2 + interface: + inputs: {} + outputs: + variables: + greet1: + description: greet1 + type: + Type: + Simple: 3 + greet2: + description: greet2 + type: + Type: + Simple: 3 + metadata: {} + metadata_defaults: {} + nodes: + - Target: null + id: start-node + - Target: null + id: end-node + inputs: + - binding: + Value: + Promise: + node_id: n0 + var: greet + var: greet1 + - binding: + Value: + Promise: + node_id: n1 + var: greet + var: greet2 + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.flyte_basics.named_outputs.say_hello + resource_type: 1 + overrides: {} + id: n0 + metadata: + InterruptibleValue: null + name: say_hello + retries: {} + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.flyte_basics.named_outputs.say_hello + resource_type: 1 + overrides: {} + id: n1 + metadata: + InterruptibleValue: null + name: say_hello + retries: {} + outputs: + - binding: + Value: + Promise: + node_id: n0 + var: greet + var: greet1 + - binding: + Value: + Promise: + node_id: n1 + var: greet + var: greet2 +tasks: +- template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.named_outputs + - task-name + - say_hello + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.flyte_basics.named_outputs.say_hello + resource_type: 1 + interface: + inputs: {} + outputs: + variables: + greet: + description: greet + type: + Type: + Simple: 3 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/133__bash.task_1_1_task.json b/pkg/compiler/test/testdata/snacks-core/compiled/133__bash.task_1_1_task.json new file mode 100755 index 0000000000..bfe53d4075 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/133__bash.task_1_1_task.json @@ -0,0 +1 @@ +{"template":{"id":{"resourceType":"TASK","name":"_bash.task_1"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{},"outputs":{}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","flytekit.extras.tasks.shell","task-name","_dummy_task_func"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/133__bash.task_1_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/133__bash.task_1_1_task.yaml new file mode 100755 index 0000000000..1a0528ae30 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/133__bash.task_1_1_task.yaml @@ -0,0 +1,45 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - flytekit.extras.tasks.shell + - task-name + - _dummy_task_func + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: _bash.task_1 + resource_type: 1 + interface: + inputs: {} + outputs: {} + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/134_task_1_1_task.json b/pkg/compiler/test/testdata/snacks-core/compiled/134_task_1_1_task.json new file mode 100755 index 0000000000..9e0d31f616 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/134_task_1_1_task.json @@ -0,0 +1 @@ +{"template":{"id":{"resourceType":"TASK","name":"task_1"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"x":{"type":{"blob":{}},"description":"x"}}},"outputs":{"variables":{"i":{"type":{"blob":{}},"description":"i"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.flyte_basics.shell_task","task-name","t1"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/134_task_1_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/134_task_1_1_task.yaml new file mode 100755 index 0000000000..589d2bb6ce --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/134_task_1_1_task.yaml @@ -0,0 +1,57 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.shell_task + - task-name + - t1 + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: task_1 + resource_type: 1 + interface: + inputs: + variables: + x: + description: x + type: + Type: + Blob: {} + outputs: + variables: + i: + description: i + type: + Type: + Blob: {} + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/135__bash.task_2_1_task.json b/pkg/compiler/test/testdata/snacks-core/compiled/135__bash.task_2_1_task.json new file mode 100755 index 0000000000..7fbfb30aac --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/135__bash.task_2_1_task.json @@ -0,0 +1 @@ +{"template":{"id":{"resourceType":"TASK","name":"_bash.task_2"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{},"outputs":{}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","flytekit.extras.tasks.shell","task-name","_dummy_task_func"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/135__bash.task_2_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/135__bash.task_2_1_task.yaml new file mode 100755 index 0000000000..897397a2ed --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/135__bash.task_2_1_task.yaml @@ -0,0 +1,45 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - flytekit.extras.tasks.shell + - task-name + - _dummy_task_func + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: _bash.task_2 + resource_type: 1 + interface: + inputs: {} + outputs: {} + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/136_task_2_1_task.json b/pkg/compiler/test/testdata/snacks-core/compiled/136_task_2_1_task.json new file mode 100755 index 0000000000..b07811e7ad --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/136_task_2_1_task.json @@ -0,0 +1 @@ +{"template":{"id":{"resourceType":"TASK","name":"task_2"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"x":{"type":{"blob":{}},"description":"x"},"y":{"type":{"blob":{"dimensionality":"MULTIPART"}},"description":"y"}}},"outputs":{"variables":{"j":{"type":{"blob":{}},"description":"j"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.flyte_basics.shell_task","task-name","t2"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/136_task_2_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/136_task_2_1_task.yaml new file mode 100755 index 0000000000..2681009404 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/136_task_2_1_task.yaml @@ -0,0 +1,63 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.shell_task + - task-name + - t2 + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: task_2 + resource_type: 1 + interface: + inputs: + variables: + x: + description: x + type: + Type: + Blob: {} + "y": + description: "y" + type: + Type: + Blob: + dimensionality: 1 + outputs: + variables: + j: + description: j + type: + Type: + Blob: {} + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/137__bash.task_3_1_task.json b/pkg/compiler/test/testdata/snacks-core/compiled/137__bash.task_3_1_task.json new file mode 100755 index 0000000000..a7801ca80c --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/137__bash.task_3_1_task.json @@ -0,0 +1 @@ +{"template":{"id":{"resourceType":"TASK","name":"_bash.task_3"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{},"outputs":{}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","flytekit.extras.tasks.shell","task-name","_dummy_task_func"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/137__bash.task_3_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/137__bash.task_3_1_task.yaml new file mode 100755 index 0000000000..7eafdfe253 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/137__bash.task_3_1_task.yaml @@ -0,0 +1,45 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - flytekit.extras.tasks.shell + - task-name + - _dummy_task_func + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: _bash.task_3 + resource_type: 1 + interface: + inputs: {} + outputs: {} + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/138_task_3_1_task.json b/pkg/compiler/test/testdata/snacks-core/compiled/138_task_3_1_task.json new file mode 100755 index 0000000000..1a1811cae0 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/138_task_3_1_task.json @@ -0,0 +1 @@ +{"template":{"id":{"resourceType":"TASK","name":"task_3"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"x":{"type":{"blob":{}},"description":"x"},"y":{"type":{"blob":{"dimensionality":"MULTIPART"}},"description":"y"},"z":{"type":{"blob":{}},"description":"z"}}},"outputs":{"variables":{"k":{"type":{"blob":{}},"description":"k"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.flyte_basics.shell_task","task-name","t3"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/138_task_3_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/138_task_3_1_task.yaml new file mode 100755 index 0000000000..0b355865c9 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/138_task_3_1_task.yaml @@ -0,0 +1,68 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.shell_task + - task-name + - t3 + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: task_3 + resource_type: 1 + interface: + inputs: + variables: + x: + description: x + type: + Type: + Blob: {} + "y": + description: "y" + type: + Type: + Blob: + dimensionality: 1 + z: + description: z + type: + Type: + Blob: {} + outputs: + variables: + k: + description: k + type: + Type: + Blob: {} + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/139_core.flyte_basics.shell_task.create_entities_1_task.json b/pkg/compiler/test/testdata/snacks-core/compiled/139_core.flyte_basics.shell_task.create_entities_1_task.json new file mode 100755 index 0000000000..55fe197d16 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/139_core.flyte_basics.shell_task.create_entities_1_task.json @@ -0,0 +1 @@ +{"template":{"id":{"resourceType":"TASK","name":"core.flyte_basics.shell_task.create_entities"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{},"outputs":{"variables":{"o0":{"type":{"blob":{}},"description":"o0"},"o1":{"type":{"blob":{"dimensionality":"MULTIPART"}},"description":"o1"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.flyte_basics.shell_task","task-name","create_entities"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/139_core.flyte_basics.shell_task.create_entities_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/139_core.flyte_basics.shell_task.create_entities_1_task.yaml new file mode 100755 index 0000000000..817744621d --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/139_core.flyte_basics.shell_task.create_entities_1_task.yaml @@ -0,0 +1,57 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.shell_task + - task-name + - create_entities + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.flyte_basics.shell_task.create_entities + resource_type: 1 + interface: + inputs: {} + outputs: + variables: + o0: + description: o0 + type: + Type: + Blob: {} + o1: + description: o1 + type: + Type: + Blob: + dimensionality: 1 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/140_core.flyte_basics.shell_task.wf_2_wf.json b/pkg/compiler/test/testdata/snacks-core/compiled/140_core.flyte_basics.shell_task.wf_2_wf.json new file mode 100755 index 0000000000..8976a06fd9 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/140_core.flyte_basics.shell_task.wf_2_wf.json @@ -0,0 +1 @@ +{"primary":{"template":{"id":{"resourceType":"WORKFLOW","name":"core.flyte_basics.shell_task.wf"},"metadata":{},"interface":{"inputs":{},"outputs":{"variables":{"o0":{"type":{"blob":{}},"description":"o0"}}}},"nodes":[{"id":"start-node"},{"id":"end-node","inputs":[{"var":"o0","binding":{"promise":{"nodeId":"n3","var":"k"}}}]},{"id":"n0","metadata":{"name":"create_entities","retries":{}},"taskNode":{"referenceId":{"resourceType":"TASK","name":"core.flyte_basics.shell_task.create_entities"},"overrides":{}}},{"id":"n1","metadata":{"name":"task_1","retries":{}},"inputs":[{"var":"x","binding":{"promise":{"nodeId":"n0","var":"o0"}}}],"upstreamNodeIds":["n0"],"taskNode":{"referenceId":{"resourceType":"TASK","name":"task_1"},"overrides":{}}},{"id":"n2","metadata":{"name":"task_2","retries":{}},"inputs":[{"var":"x","binding":{"promise":{"nodeId":"n1","var":"i"}}},{"var":"y","binding":{"promise":{"nodeId":"n0","var":"o1"}}}],"upstreamNodeIds":["n0","n1"],"taskNode":{"referenceId":{"resourceType":"TASK","name":"task_2"},"overrides":{}}},{"id":"n3","metadata":{"name":"task_3","retries":{}},"inputs":[{"var":"x","binding":{"promise":{"nodeId":"n0","var":"o0"}}},{"var":"y","binding":{"promise":{"nodeId":"n0","var":"o1"}}},{"var":"z","binding":{"promise":{"nodeId":"n2","var":"j"}}}],"upstreamNodeIds":["n0","n2"],"taskNode":{"referenceId":{"resourceType":"TASK","name":"task_3"},"overrides":{}}}],"outputs":[{"var":"o0","binding":{"promise":{"nodeId":"n3","var":"k"}}}],"metadataDefaults":{}},"connections":{"downstream":{"n0":{"ids":["n1","n2","n3"]},"n1":{"ids":["n2"]},"n2":{"ids":["n3"]},"n3":{"ids":["end-node"]},"start-node":{"ids":["n0"]}},"upstream":{"end-node":{"ids":["n3"]},"n0":{"ids":["start-node"]},"n1":{"ids":["n0"]},"n2":{"ids":["n0","n1"]},"n3":{"ids":["n0","n2"]}}}},"tasks":[{"template":{"id":{"resourceType":"TASK","name":"core.flyte_basics.shell_task.create_entities"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{},"outputs":{"variables":{"o0":{"type":{"blob":{}},"description":"o0"},"o1":{"type":{"blob":{"dimensionality":"MULTIPART"}},"description":"o1"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.flyte_basics.shell_task","task-name","create_entities"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},{"template":{"id":{"resourceType":"TASK","name":"task_1"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"x":{"type":{"blob":{}},"description":"x"}}},"outputs":{"variables":{"i":{"type":{"blob":{}},"description":"i"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.flyte_basics.shell_task","task-name","t1"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},{"template":{"id":{"resourceType":"TASK","name":"task_2"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"x":{"type":{"blob":{}},"description":"x"},"y":{"type":{"blob":{"dimensionality":"MULTIPART"}},"description":"y"}}},"outputs":{"variables":{"j":{"type":{"blob":{}},"description":"j"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.flyte_basics.shell_task","task-name","t2"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},{"template":{"id":{"resourceType":"TASK","name":"task_3"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"x":{"type":{"blob":{}},"description":"x"},"y":{"type":{"blob":{"dimensionality":"MULTIPART"}},"description":"y"},"z":{"type":{"blob":{}},"description":"z"}}},"outputs":{"variables":{"k":{"type":{"blob":{}},"description":"k"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.flyte_basics.shell_task","task-name","t3"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}}]} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/140_core.flyte_basics.shell_task.wf_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/140_core.flyte_basics.shell_task.wf_2_wf.yaml new file mode 100755 index 0000000000..be10236b26 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/140_core.flyte_basics.shell_task.wf_2_wf.yaml @@ -0,0 +1,413 @@ +primary: + connections: + downstream: + n0: + ids: + - n1 + - n2 + - n3 + n1: + ids: + - n2 + n2: + ids: + - n3 + n3: + ids: + - end-node + start-node: + ids: + - n0 + upstream: + end-node: + ids: + - n3 + n0: + ids: + - start-node + n1: + ids: + - n0 + n2: + ids: + - n0 + - n1 + n3: + ids: + - n0 + - n2 + template: + id: + name: core.flyte_basics.shell_task.wf + resource_type: 2 + interface: + inputs: {} + outputs: + variables: + o0: + description: o0 + type: + Type: + Blob: {} + metadata: {} + metadata_defaults: {} + nodes: + - Target: null + id: start-node + - Target: null + id: end-node + inputs: + - binding: + Value: + Promise: + node_id: n3 + var: k + var: o0 + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.flyte_basics.shell_task.create_entities + resource_type: 1 + overrides: {} + id: n0 + metadata: + InterruptibleValue: null + name: create_entities + retries: {} + - Target: + TaskNode: + Reference: + ReferenceId: + name: task_1 + resource_type: 1 + overrides: {} + id: n1 + inputs: + - binding: + Value: + Promise: + node_id: n0 + var: o0 + var: x + metadata: + InterruptibleValue: null + name: task_1 + retries: {} + upstream_node_ids: + - n0 + - Target: + TaskNode: + Reference: + ReferenceId: + name: task_2 + resource_type: 1 + overrides: {} + id: n2 + inputs: + - binding: + Value: + Promise: + node_id: n1 + var: i + var: x + - binding: + Value: + Promise: + node_id: n0 + var: o1 + var: "y" + metadata: + InterruptibleValue: null + name: task_2 + retries: {} + upstream_node_ids: + - n0 + - n1 + - Target: + TaskNode: + Reference: + ReferenceId: + name: task_3 + resource_type: 1 + overrides: {} + id: n3 + inputs: + - binding: + Value: + Promise: + node_id: n0 + var: o0 + var: x + - binding: + Value: + Promise: + node_id: n0 + var: o1 + var: "y" + - binding: + Value: + Promise: + node_id: n2 + var: j + var: z + metadata: + InterruptibleValue: null + name: task_3 + retries: {} + upstream_node_ids: + - n0 + - n2 + outputs: + - binding: + Value: + Promise: + node_id: n3 + var: k + var: o0 +tasks: +- template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.shell_task + - task-name + - create_entities + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.flyte_basics.shell_task.create_entities + resource_type: 1 + interface: + inputs: {} + outputs: + variables: + o0: + description: o0 + type: + Type: + Blob: {} + o1: + description: o1 + type: + Type: + Blob: + dimensionality: 1 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task +- template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.shell_task + - task-name + - t1 + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: task_1 + resource_type: 1 + interface: + inputs: + variables: + x: + description: x + type: + Type: + Blob: {} + outputs: + variables: + i: + description: i + type: + Type: + Blob: {} + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task +- template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.shell_task + - task-name + - t2 + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: task_2 + resource_type: 1 + interface: + inputs: + variables: + x: + description: x + type: + Type: + Blob: {} + "y": + description: "y" + type: + Type: + Blob: + dimensionality: 1 + outputs: + variables: + j: + description: j + type: + Type: + Blob: {} + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task +- template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.shell_task + - task-name + - t3 + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: task_3 + resource_type: 1 + interface: + inputs: + variables: + x: + description: x + type: + Type: + Blob: {} + "y": + description: "y" + type: + Type: + Blob: + dimensionality: 1 + z: + description: z + type: + Type: + Blob: {} + outputs: + variables: + k: + description: k + type: + Type: + Blob: {} + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/142_core.flyte_basics.task.square_1_task.json b/pkg/compiler/test/testdata/snacks-core/compiled/142_core.flyte_basics.task.square_1_task.json new file mode 100755 index 0000000000..ce87c5eb43 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/142_core.flyte_basics.task.square_1_task.json @@ -0,0 +1 @@ +{"template":{"id":{"resourceType":"TASK","name":"core.flyte_basics.task.square"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"n":{"type":{"simple":"INTEGER"},"description":"n"}}},"outputs":{"variables":{"o0":{"type":{"simple":"INTEGER"},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.flyte_basics.task","task-name","square"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/142_core.flyte_basics.task.square_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/142_core.flyte_basics.task.square_1_task.yaml new file mode 100755 index 0000000000..e75da1ead5 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/142_core.flyte_basics.task.square_1_task.yaml @@ -0,0 +1,57 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.task + - task-name + - square + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.flyte_basics.task.square + resource_type: 1 + interface: + inputs: + variables: + "n": + description: "n" + type: + Type: + Simple: 1 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 1 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/143_core.flyte_basics.task_cache.square_1_task.json b/pkg/compiler/test/testdata/snacks-core/compiled/143_core.flyte_basics.task_cache.square_1_task.json new file mode 100755 index 0000000000..821e48cb29 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/143_core.flyte_basics.task_cache.square_1_task.json @@ -0,0 +1 @@ +{"template":{"id":{"resourceType":"TASK","name":"core.flyte_basics.task_cache.square"},"type":"python-task","metadata":{"discoverable":true,"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{},"discoveryVersion":"1.0"},"interface":{"inputs":{"variables":{"n":{"type":{"simple":"INTEGER"},"description":"n"}}},"outputs":{"variables":{"o0":{"type":{"simple":"INTEGER"},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.flyte_basics.task_cache","task-name","square"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/143_core.flyte_basics.task_cache.square_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/143_core.flyte_basics.task_cache.square_1_task.yaml new file mode 100755 index 0000000000..b4f3b0059d --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/143_core.flyte_basics.task_cache.square_1_task.yaml @@ -0,0 +1,59 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.task_cache + - task-name + - square + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.flyte_basics.task_cache.square + resource_type: 1 + interface: + inputs: + variables: + "n": + description: "n" + type: + Type: + Simple: 1 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 1 + metadata: + InterruptibleValue: null + discoverable: true + discovery_version: "1.0" + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/144_core.flyte_basics.task_cache.uncached_data_reading_task_1_task.json b/pkg/compiler/test/testdata/snacks-core/compiled/144_core.flyte_basics.task_cache.uncached_data_reading_task_1_task.json new file mode 100755 index 0000000000..d7c0f78c79 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/144_core.flyte_basics.task_cache.uncached_data_reading_task_1_task.json @@ -0,0 +1 @@ +{"template":{"id":{"resourceType":"TASK","name":"core.flyte_basics.task_cache.uncached_data_reading_task"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{},"outputs":{"variables":{"o0":{"type":{"structuredDatasetType":{"format":"parquet"}},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.flyte_basics.task_cache","task-name","uncached_data_reading_task"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/144_core.flyte_basics.task_cache.uncached_data_reading_task_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/144_core.flyte_basics.task_cache.uncached_data_reading_task_1_task.yaml new file mode 100755 index 0000000000..08cc1b4679 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/144_core.flyte_basics.task_cache.uncached_data_reading_task_1_task.yaml @@ -0,0 +1,52 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.task_cache + - task-name + - uncached_data_reading_task + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.flyte_basics.task_cache.uncached_data_reading_task + resource_type: 1 + interface: + inputs: {} + outputs: + variables: + o0: + description: o0 + type: + Type: + StructuredDatasetType: + format: parquet + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/145_core.flyte_basics.task_cache.cached_data_processing_task_1_task.json b/pkg/compiler/test/testdata/snacks-core/compiled/145_core.flyte_basics.task_cache.cached_data_processing_task_1_task.json new file mode 100755 index 0000000000..dbd48434ee --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/145_core.flyte_basics.task_cache.cached_data_processing_task_1_task.json @@ -0,0 +1 @@ +{"template":{"id":{"resourceType":"TASK","name":"core.flyte_basics.task_cache.cached_data_processing_task"},"type":"python-task","metadata":{"discoverable":true,"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{},"discoveryVersion":"1.0"},"interface":{"inputs":{"variables":{"df":{"type":{"structuredDatasetType":{"format":"parquet"}},"description":"df"}}},"outputs":{"variables":{"o0":{"type":{"structuredDatasetType":{"format":"parquet"}},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.flyte_basics.task_cache","task-name","cached_data_processing_task"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/145_core.flyte_basics.task_cache.cached_data_processing_task_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/145_core.flyte_basics.task_cache.cached_data_processing_task_1_task.yaml new file mode 100755 index 0000000000..81122326ef --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/145_core.flyte_basics.task_cache.cached_data_processing_task_1_task.yaml @@ -0,0 +1,61 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.task_cache + - task-name + - cached_data_processing_task + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.flyte_basics.task_cache.cached_data_processing_task + resource_type: 1 + interface: + inputs: + variables: + df: + description: df + type: + Type: + StructuredDatasetType: + format: parquet + outputs: + variables: + o0: + description: o0 + type: + Type: + StructuredDatasetType: + format: parquet + metadata: + InterruptibleValue: null + discoverable: true + discovery_version: "1.0" + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/146_core.flyte_basics.task_cache.compare_dataframes_1_task.json b/pkg/compiler/test/testdata/snacks-core/compiled/146_core.flyte_basics.task_cache.compare_dataframes_1_task.json new file mode 100755 index 0000000000..1051ddb944 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/146_core.flyte_basics.task_cache.compare_dataframes_1_task.json @@ -0,0 +1 @@ +{"template":{"id":{"resourceType":"TASK","name":"core.flyte_basics.task_cache.compare_dataframes"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"df1":{"type":{"structuredDatasetType":{"format":"parquet"}},"description":"df1"},"df2":{"type":{"structuredDatasetType":{"format":"parquet"}},"description":"df2"}}},"outputs":{}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.flyte_basics.task_cache","task-name","compare_dataframes"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/146_core.flyte_basics.task_cache.compare_dataframes_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/146_core.flyte_basics.task_cache.compare_dataframes_1_task.yaml new file mode 100755 index 0000000000..19c4a0b565 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/146_core.flyte_basics.task_cache.compare_dataframes_1_task.yaml @@ -0,0 +1,58 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.task_cache + - task-name + - compare_dataframes + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.flyte_basics.task_cache.compare_dataframes + resource_type: 1 + interface: + inputs: + variables: + df1: + description: df1 + type: + Type: + StructuredDatasetType: + format: parquet + df2: + description: df2 + type: + Type: + StructuredDatasetType: + format: parquet + outputs: {} + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/147_core.flyte_basics.task_cache.cached_dataframe_wf_2_wf.json b/pkg/compiler/test/testdata/snacks-core/compiled/147_core.flyte_basics.task_cache.cached_dataframe_wf_2_wf.json new file mode 100755 index 0000000000..52cc09b532 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/147_core.flyte_basics.task_cache.cached_dataframe_wf_2_wf.json @@ -0,0 +1 @@ +{"primary":{"template":{"id":{"resourceType":"WORKFLOW","name":"core.flyte_basics.task_cache.cached_dataframe_wf"},"metadata":{},"interface":{"inputs":{},"outputs":{}},"nodes":[{"id":"start-node"},{"id":"end-node"},{"id":"n0","metadata":{"name":"uncached_data_reading_task","retries":{}},"taskNode":{"referenceId":{"resourceType":"TASK","name":"core.flyte_basics.task_cache.uncached_data_reading_task"},"overrides":{}}},{"id":"n1","metadata":{"name":"cached_data_processing_task","retries":{}},"inputs":[{"var":"df","binding":{"promise":{"nodeId":"n0","var":"o0"}}}],"upstreamNodeIds":["n0"],"taskNode":{"referenceId":{"resourceType":"TASK","name":"core.flyte_basics.task_cache.cached_data_processing_task"},"overrides":{}}},{"id":"n2","metadata":{"name":"cached_data_processing_task","retries":{}},"inputs":[{"var":"df","binding":{"promise":{"nodeId":"n0","var":"o0"}}}],"upstreamNodeIds":["n0","n1"],"taskNode":{"referenceId":{"resourceType":"TASK","name":"core.flyte_basics.task_cache.cached_data_processing_task"},"overrides":{}}},{"id":"n3","metadata":{"name":"compare_dataframes","retries":{}},"inputs":[{"var":"df1","binding":{"promise":{"nodeId":"n1","var":"o0"}}},{"var":"df2","binding":{"promise":{"nodeId":"n2","var":"o0"}}}],"upstreamNodeIds":["n1","n2"],"taskNode":{"referenceId":{"resourceType":"TASK","name":"core.flyte_basics.task_cache.compare_dataframes"},"overrides":{}}}],"metadataDefaults":{}},"connections":{"downstream":{"n0":{"ids":["n1","n2"]},"n1":{"ids":["n2","n3"]},"n2":{"ids":["n3"]},"n3":{"ids":["end-node"]},"start-node":{"ids":["n0"]}},"upstream":{"end-node":{"ids":["n3"]},"n0":{"ids":["start-node"]},"n1":{"ids":["n0"]},"n2":{"ids":["n0","n1"]},"n3":{"ids":["n1","n2"]}}}},"tasks":[{"template":{"id":{"resourceType":"TASK","name":"core.flyte_basics.task_cache.cached_data_processing_task"},"type":"python-task","metadata":{"discoverable":true,"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{},"discoveryVersion":"1.0"},"interface":{"inputs":{"variables":{"df":{"type":{"structuredDatasetType":{"format":"parquet"}},"description":"df"}}},"outputs":{"variables":{"o0":{"type":{"structuredDatasetType":{"format":"parquet"}},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.flyte_basics.task_cache","task-name","cached_data_processing_task"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},{"template":{"id":{"resourceType":"TASK","name":"core.flyte_basics.task_cache.compare_dataframes"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"df1":{"type":{"structuredDatasetType":{"format":"parquet"}},"description":"df1"},"df2":{"type":{"structuredDatasetType":{"format":"parquet"}},"description":"df2"}}},"outputs":{}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.flyte_basics.task_cache","task-name","compare_dataframes"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},{"template":{"id":{"resourceType":"TASK","name":"core.flyte_basics.task_cache.uncached_data_reading_task"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{},"outputs":{"variables":{"o0":{"type":{"structuredDatasetType":{"format":"parquet"}},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.flyte_basics.task_cache","task-name","uncached_data_reading_task"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}}]} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/147_core.flyte_basics.task_cache.cached_dataframe_wf_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/147_core.flyte_basics.task_cache.cached_dataframe_wf_2_wf.yaml new file mode 100755 index 0000000000..f9c1413ad3 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/147_core.flyte_basics.task_cache.cached_dataframe_wf_2_wf.yaml @@ -0,0 +1,307 @@ +primary: + connections: + downstream: + n0: + ids: + - n1 + - n2 + n1: + ids: + - n2 + - n3 + n2: + ids: + - n3 + n3: + ids: + - end-node + start-node: + ids: + - n0 + upstream: + end-node: + ids: + - n3 + n0: + ids: + - start-node + n1: + ids: + - n0 + n2: + ids: + - n0 + - n1 + n3: + ids: + - n1 + - n2 + template: + id: + name: core.flyte_basics.task_cache.cached_dataframe_wf + resource_type: 2 + interface: + inputs: {} + outputs: {} + metadata: {} + metadata_defaults: {} + nodes: + - Target: null + id: start-node + - Target: null + id: end-node + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.flyte_basics.task_cache.uncached_data_reading_task + resource_type: 1 + overrides: {} + id: n0 + metadata: + InterruptibleValue: null + name: uncached_data_reading_task + retries: {} + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.flyte_basics.task_cache.cached_data_processing_task + resource_type: 1 + overrides: {} + id: n1 + inputs: + - binding: + Value: + Promise: + node_id: n0 + var: o0 + var: df + metadata: + InterruptibleValue: null + name: cached_data_processing_task + retries: {} + upstream_node_ids: + - n0 + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.flyte_basics.task_cache.cached_data_processing_task + resource_type: 1 + overrides: {} + id: n2 + inputs: + - binding: + Value: + Promise: + node_id: n0 + var: o0 + var: df + metadata: + InterruptibleValue: null + name: cached_data_processing_task + retries: {} + upstream_node_ids: + - n0 + - n1 + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.flyte_basics.task_cache.compare_dataframes + resource_type: 1 + overrides: {} + id: n3 + inputs: + - binding: + Value: + Promise: + node_id: n1 + var: o0 + var: df1 + - binding: + Value: + Promise: + node_id: n2 + var: o0 + var: df2 + metadata: + InterruptibleValue: null + name: compare_dataframes + retries: {} + upstream_node_ids: + - n1 + - n2 +tasks: +- template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.task_cache + - task-name + - cached_data_processing_task + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.flyte_basics.task_cache.cached_data_processing_task + resource_type: 1 + interface: + inputs: + variables: + df: + description: df + type: + Type: + StructuredDatasetType: + format: parquet + outputs: + variables: + o0: + description: o0 + type: + Type: + StructuredDatasetType: + format: parquet + metadata: + InterruptibleValue: null + discoverable: true + discovery_version: "1.0" + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task +- template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.task_cache + - task-name + - compare_dataframes + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.flyte_basics.task_cache.compare_dataframes + resource_type: 1 + interface: + inputs: + variables: + df1: + description: df1 + type: + Type: + StructuredDatasetType: + format: parquet + df2: + description: df2 + type: + Type: + StructuredDatasetType: + format: parquet + outputs: {} + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task +- template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.task_cache + - task-name + - uncached_data_reading_task + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.flyte_basics.task_cache.uncached_data_reading_task + resource_type: 1 + interface: + inputs: {} + outputs: + variables: + o0: + description: o0 + type: + Type: + StructuredDatasetType: + format: parquet + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/149_core.flyte_basics.task_cache_serialize.square_1_task.json b/pkg/compiler/test/testdata/snacks-core/compiled/149_core.flyte_basics.task_cache_serialize.square_1_task.json new file mode 100755 index 0000000000..bcb06af455 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/149_core.flyte_basics.task_cache_serialize.square_1_task.json @@ -0,0 +1 @@ +{"template":{"id":{"resourceType":"TASK","name":"core.flyte_basics.task_cache_serialize.square"},"type":"python-task","metadata":{"discoverable":true,"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{},"discoveryVersion":"1.0","cacheSerializable":true},"interface":{"inputs":{"variables":{"n":{"type":{"simple":"INTEGER"},"description":"n"}}},"outputs":{"variables":{"o0":{"type":{"simple":"INTEGER"},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.flyte_basics.task_cache_serialize","task-name","square"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/149_core.flyte_basics.task_cache_serialize.square_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/149_core.flyte_basics.task_cache_serialize.square_1_task.yaml new file mode 100755 index 0000000000..f911b28178 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/149_core.flyte_basics.task_cache_serialize.square_1_task.yaml @@ -0,0 +1,60 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.task_cache_serialize + - task-name + - square + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.flyte_basics.task_cache_serialize.square + resource_type: 1 + interface: + inputs: + variables: + "n": + description: "n" + type: + Type: + Simple: 1 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 1 + metadata: + InterruptibleValue: null + cache_serializable: true + discoverable: true + discovery_version: "1.0" + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/150_core.scheduled_workflows.lp_schedules.format_date_1_task.json b/pkg/compiler/test/testdata/snacks-core/compiled/150_core.scheduled_workflows.lp_schedules.format_date_1_task.json new file mode 100755 index 0000000000..40b21406f5 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/150_core.scheduled_workflows.lp_schedules.format_date_1_task.json @@ -0,0 +1 @@ +{"template":{"id":{"resourceType":"TASK","name":"core.scheduled_workflows.lp_schedules.format_date"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"run_date":{"type":{"simple":"DATETIME"},"description":"run_date"}}},"outputs":{"variables":{"o0":{"type":{"simple":"STRING"},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.scheduled_workflows.lp_schedules","task-name","format_date"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/150_core.scheduled_workflows.lp_schedules.format_date_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/150_core.scheduled_workflows.lp_schedules.format_date_1_task.yaml new file mode 100755 index 0000000000..dd6011d2a7 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/150_core.scheduled_workflows.lp_schedules.format_date_1_task.yaml @@ -0,0 +1,57 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.scheduled_workflows.lp_schedules + - task-name + - format_date + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.scheduled_workflows.lp_schedules.format_date + resource_type: 1 + interface: + inputs: + variables: + run_date: + description: run_date + type: + Type: + Simple: 5 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 3 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/151_core.scheduled_workflows.lp_schedules.date_formatter_wf_2_wf.json b/pkg/compiler/test/testdata/snacks-core/compiled/151_core.scheduled_workflows.lp_schedules.date_formatter_wf_2_wf.json new file mode 100755 index 0000000000..e505856319 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/151_core.scheduled_workflows.lp_schedules.date_formatter_wf_2_wf.json @@ -0,0 +1 @@ +{"primary":{"template":{"id":{"resourceType":"WORKFLOW","name":"core.scheduled_workflows.lp_schedules.date_formatter_wf"},"metadata":{},"interface":{"inputs":{"variables":{"kickoff_time":{"type":{"simple":"DATETIME"},"description":"kickoff_time"}}},"outputs":{}},"nodes":[{"id":"start-node"},{"id":"end-node"},{"id":"n0","metadata":{"name":"format_date","retries":{}},"inputs":[{"var":"run_date","binding":{"promise":{"nodeId":"start-node","var":"kickoff_time"}}}],"taskNode":{"referenceId":{"resourceType":"TASK","name":"core.scheduled_workflows.lp_schedules.format_date"},"overrides":{}}}],"metadataDefaults":{}},"connections":{"downstream":{"n0":{"ids":["end-node"]},"start-node":{"ids":["n0"]}},"upstream":{"end-node":{"ids":["n0"]},"n0":{"ids":["start-node"]}}}},"tasks":[{"template":{"id":{"resourceType":"TASK","name":"core.scheduled_workflows.lp_schedules.format_date"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"run_date":{"type":{"simple":"DATETIME"},"description":"run_date"}}},"outputs":{"variables":{"o0":{"type":{"simple":"STRING"},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.scheduled_workflows.lp_schedules","task-name","format_date"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}}]} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/151_core.scheduled_workflows.lp_schedules.date_formatter_wf_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/151_core.scheduled_workflows.lp_schedules.date_formatter_wf_2_wf.yaml new file mode 100755 index 0000000000..63b51ed885 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/151_core.scheduled_workflows.lp_schedules.date_formatter_wf_2_wf.yaml @@ -0,0 +1,113 @@ +primary: + connections: + downstream: + n0: + ids: + - end-node + start-node: + ids: + - n0 + upstream: + end-node: + ids: + - n0 + n0: + ids: + - start-node + template: + id: + name: core.scheduled_workflows.lp_schedules.date_formatter_wf + resource_type: 2 + interface: + inputs: + variables: + kickoff_time: + description: kickoff_time + type: + Type: + Simple: 5 + outputs: {} + metadata: {} + metadata_defaults: {} + nodes: + - Target: null + id: start-node + - Target: null + id: end-node + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.scheduled_workflows.lp_schedules.format_date + resource_type: 1 + overrides: {} + id: n0 + inputs: + - binding: + Value: + Promise: + node_id: start-node + var: kickoff_time + var: run_date + metadata: + InterruptibleValue: null + name: format_date + retries: {} +tasks: +- template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.scheduled_workflows.lp_schedules + - task-name + - format_date + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.scheduled_workflows.lp_schedules.format_date + resource_type: 1 + interface: + inputs: + variables: + run_date: + description: run_date + type: + Type: + Simple: 5 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 3 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/154_core.scheduled_workflows.lp_schedules.be_positive_1_task.json b/pkg/compiler/test/testdata/snacks-core/compiled/154_core.scheduled_workflows.lp_schedules.be_positive_1_task.json new file mode 100755 index 0000000000..619ee36633 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/154_core.scheduled_workflows.lp_schedules.be_positive_1_task.json @@ -0,0 +1 @@ +{"template":{"id":{"resourceType":"TASK","name":"core.scheduled_workflows.lp_schedules.be_positive"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"name":{"type":{"simple":"STRING"},"description":"name"}}},"outputs":{"variables":{"o0":{"type":{"simple":"STRING"},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.scheduled_workflows.lp_schedules","task-name","be_positive"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/154_core.scheduled_workflows.lp_schedules.be_positive_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/154_core.scheduled_workflows.lp_schedules.be_positive_1_task.yaml new file mode 100755 index 0000000000..8307254f44 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/154_core.scheduled_workflows.lp_schedules.be_positive_1_task.yaml @@ -0,0 +1,57 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.scheduled_workflows.lp_schedules + - task-name + - be_positive + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.scheduled_workflows.lp_schedules.be_positive + resource_type: 1 + interface: + inputs: + variables: + name: + description: name + type: + Type: + Simple: 3 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 3 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/155_core.scheduled_workflows.lp_schedules.positive_wf_2_wf.json b/pkg/compiler/test/testdata/snacks-core/compiled/155_core.scheduled_workflows.lp_schedules.positive_wf_2_wf.json new file mode 100755 index 0000000000..8148e1f46f --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/155_core.scheduled_workflows.lp_schedules.positive_wf_2_wf.json @@ -0,0 +1 @@ +{"primary":{"template":{"id":{"resourceType":"WORKFLOW","name":"core.scheduled_workflows.lp_schedules.positive_wf"},"metadata":{},"interface":{"inputs":{"variables":{"name":{"type":{"simple":"STRING"},"description":"name"}}},"outputs":{}},"nodes":[{"id":"start-node"},{"id":"end-node"},{"id":"n0","metadata":{"name":"be_positive","retries":{}},"inputs":[{"var":"name","binding":{"promise":{"nodeId":"start-node","var":"name"}}}],"taskNode":{"referenceId":{"resourceType":"TASK","name":"core.scheduled_workflows.lp_schedules.be_positive"},"overrides":{}}}],"metadataDefaults":{}},"connections":{"downstream":{"n0":{"ids":["end-node"]},"start-node":{"ids":["n0"]}},"upstream":{"end-node":{"ids":["n0"]},"n0":{"ids":["start-node"]}}}},"tasks":[{"template":{"id":{"resourceType":"TASK","name":"core.scheduled_workflows.lp_schedules.be_positive"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"name":{"type":{"simple":"STRING"},"description":"name"}}},"outputs":{"variables":{"o0":{"type":{"simple":"STRING"},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.scheduled_workflows.lp_schedules","task-name","be_positive"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}}]} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/155_core.scheduled_workflows.lp_schedules.positive_wf_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/155_core.scheduled_workflows.lp_schedules.positive_wf_2_wf.yaml new file mode 100755 index 0000000000..ce6892e8ad --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/155_core.scheduled_workflows.lp_schedules.positive_wf_2_wf.yaml @@ -0,0 +1,113 @@ +primary: + connections: + downstream: + n0: + ids: + - end-node + start-node: + ids: + - n0 + upstream: + end-node: + ids: + - n0 + n0: + ids: + - start-node + template: + id: + name: core.scheduled_workflows.lp_schedules.positive_wf + resource_type: 2 + interface: + inputs: + variables: + name: + description: name + type: + Type: + Simple: 3 + outputs: {} + metadata: {} + metadata_defaults: {} + nodes: + - Target: null + id: start-node + - Target: null + id: end-node + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.scheduled_workflows.lp_schedules.be_positive + resource_type: 1 + overrides: {} + id: n0 + inputs: + - binding: + Value: + Promise: + node_id: start-node + var: name + var: name + metadata: + InterruptibleValue: null + name: be_positive + retries: {} +tasks: +- template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.scheduled_workflows.lp_schedules + - task-name + - be_positive + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.scheduled_workflows.lp_schedules.be_positive + resource_type: 1 + interface: + inputs: + variables: + name: + description: name + type: + Type: + Simple: 3 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 3 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/158_core.type_system.custom_objects.stringify_1_task.json b/pkg/compiler/test/testdata/snacks-core/compiled/158_core.type_system.custom_objects.stringify_1_task.json new file mode 100755 index 0000000000..16f49422f1 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/158_core.type_system.custom_objects.stringify_1_task.json @@ -0,0 +1 @@ +{"template":{"id":{"resourceType":"TASK","name":"core.type_system.custom_objects.stringify"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"x":{"type":{"simple":"INTEGER"},"description":"x"}}},"outputs":{"variables":{"o0":{"type":{"simple":"STRUCT","metadata":{"$ref":"#/definitions/DatumSchema","$schema":"http://json-schema.org/draft-07/schema#","definitions":{"DatumSchema":{"additionalProperties":false,"properties":{"x":{"title":"x","type":"integer"},"y":{"title":"y","type":"string"},"z":{"additionalProperties":{"title":"z","type":"string"},"title":"z","type":"object"}},"type":"object"}}}},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.type_system.custom_objects","task-name","stringify"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/158_core.type_system.custom_objects.stringify_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/158_core.type_system.custom_objects.stringify_1_task.yaml new file mode 100755 index 0000000000..0a1c333c4f --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/158_core.type_system.custom_objects.stringify_1_task.yaml @@ -0,0 +1,77 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.type_system.custom_objects + - task-name + - stringify + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.type_system.custom_objects.stringify + resource_type: 1 + interface: + inputs: + variables: + x: + description: x + type: + Type: + Simple: 1 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 9 + metadata: + $ref: '#/definitions/DatumSchema' + $schema: http://json-schema.org/draft-07/schema# + definitions: + DatumSchema: + additionalProperties: false + properties: + x: + title: x + type: integer + "y": + title: "y" + type: string + z: + additionalProperties: + title: z + type: string + title: z + type: object + type: object + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/159_core.type_system.custom_objects.add_1_task.json b/pkg/compiler/test/testdata/snacks-core/compiled/159_core.type_system.custom_objects.add_1_task.json new file mode 100755 index 0000000000..738a607eb2 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/159_core.type_system.custom_objects.add_1_task.json @@ -0,0 +1 @@ +{"template":{"id":{"resourceType":"TASK","name":"core.type_system.custom_objects.add"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"x":{"type":{"simple":"STRUCT","metadata":{"$ref":"#/definitions/DatumSchema","$schema":"http://json-schema.org/draft-07/schema#","definitions":{"DatumSchema":{"additionalProperties":false,"properties":{"x":{"title":"x","type":"integer"},"y":{"title":"y","type":"string"},"z":{"additionalProperties":{"title":"z","type":"string"},"title":"z","type":"object"}},"type":"object"}}}},"description":"x"},"y":{"type":{"simple":"STRUCT","metadata":{"$ref":"#/definitions/DatumSchema","$schema":"http://json-schema.org/draft-07/schema#","definitions":{"DatumSchema":{"additionalProperties":false,"properties":{"x":{"title":"x","type":"integer"},"y":{"title":"y","type":"string"},"z":{"additionalProperties":{"title":"z","type":"string"},"title":"z","type":"object"}},"type":"object"}}}},"description":"y"}}},"outputs":{"variables":{"o0":{"type":{"simple":"STRUCT","metadata":{"$ref":"#/definitions/DatumSchema","$schema":"http://json-schema.org/draft-07/schema#","definitions":{"DatumSchema":{"additionalProperties":false,"properties":{"x":{"title":"x","type":"integer"},"y":{"title":"y","type":"string"},"z":{"additionalProperties":{"title":"z","type":"string"},"title":"z","type":"object"}},"type":"object"}}}},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.type_system.custom_objects","task-name","add"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/159_core.type_system.custom_objects.add_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/159_core.type_system.custom_objects.add_1_task.yaml new file mode 100755 index 0000000000..58518ef722 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/159_core.type_system.custom_objects.add_1_task.yaml @@ -0,0 +1,122 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.type_system.custom_objects + - task-name + - add + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.type_system.custom_objects.add + resource_type: 1 + interface: + inputs: + variables: + x: + description: x + type: + Type: + Simple: 9 + metadata: + $ref: '#/definitions/DatumSchema' + $schema: http://json-schema.org/draft-07/schema# + definitions: + DatumSchema: + additionalProperties: false + properties: + x: + title: x + type: integer + "y": + title: "y" + type: string + z: + additionalProperties: + title: z + type: string + title: z + type: object + type: object + "y": + description: "y" + type: + Type: + Simple: 9 + metadata: + $ref: '#/definitions/DatumSchema' + $schema: http://json-schema.org/draft-07/schema# + definitions: + DatumSchema: + additionalProperties: false + properties: + x: + title: x + type: integer + "y": + title: "y" + type: string + z: + additionalProperties: + title: z + type: string + title: z + type: object + type: object + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 9 + metadata: + $ref: '#/definitions/DatumSchema' + $schema: http://json-schema.org/draft-07/schema# + definitions: + DatumSchema: + additionalProperties: false + properties: + x: + title: x + type: integer + "y": + title: "y" + type: string + z: + additionalProperties: + title: z + type: string + title: z + type: object + type: object + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/160_core.type_system.custom_objects.upload_result_1_task.json b/pkg/compiler/test/testdata/snacks-core/compiled/160_core.type_system.custom_objects.upload_result_1_task.json new file mode 100755 index 0000000000..f5ea2a7e23 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/160_core.type_system.custom_objects.upload_result_1_task.json @@ -0,0 +1 @@ +{"template":{"id":{"resourceType":"TASK","name":"core.type_system.custom_objects.upload_result"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{},"outputs":{"variables":{"o0":{"type":{"simple":"STRUCT","metadata":{"$ref":"#/definitions/ResultSchema","$schema":"http://json-schema.org/draft-07/schema#","definitions":{"FlytedirectorySchema":{"additionalProperties":false,"properties":{"path":{"title":"path","type":"string"}},"type":"object"},"FlytefileSchema":{"additionalProperties":false,"properties":{"path":{"title":"path","type":"string"}},"type":"object"},"FlyteschemaSchema":{"additionalProperties":false,"properties":{"remote_path":{"title":"remote_path","type":"string"}},"type":"object"},"ResultSchema":{"additionalProperties":false,"properties":{"directory":{"$ref":"#/definitions/FlytedirectorySchema","field_many":false,"type":"object"},"file":{"$ref":"#/definitions/FlytefileSchema","field_many":false,"type":"object"},"schema":{"$ref":"#/definitions/FlyteschemaSchema","field_many":false,"type":"object"}},"type":"object"}}}},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.type_system.custom_objects","task-name","upload_result"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/160_core.type_system.custom_objects.upload_result_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/160_core.type_system.custom_objects.upload_result_1_task.yaml new file mode 100755 index 0000000000..6f7e41c468 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/160_core.type_system.custom_objects.upload_result_1_task.yaml @@ -0,0 +1,92 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.type_system.custom_objects + - task-name + - upload_result + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.type_system.custom_objects.upload_result + resource_type: 1 + interface: + inputs: {} + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 9 + metadata: + $ref: '#/definitions/ResultSchema' + $schema: http://json-schema.org/draft-07/schema# + definitions: + FlytedirectorySchema: + additionalProperties: false + properties: + path: + title: path + type: string + type: object + FlytefileSchema: + additionalProperties: false + properties: + path: + title: path + type: string + type: object + FlyteschemaSchema: + additionalProperties: false + properties: + remote_path: + title: remote_path + type: string + type: object + ResultSchema: + additionalProperties: false + properties: + directory: + $ref: '#/definitions/FlytedirectorySchema' + field_many: false + type: object + file: + $ref: '#/definitions/FlytefileSchema' + field_many: false + type: object + schema: + $ref: '#/definitions/FlyteschemaSchema' + field_many: false + type: object + type: object + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/161_core.type_system.custom_objects.download_result_1_task.json b/pkg/compiler/test/testdata/snacks-core/compiled/161_core.type_system.custom_objects.download_result_1_task.json new file mode 100755 index 0000000000..a987d01ec9 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/161_core.type_system.custom_objects.download_result_1_task.json @@ -0,0 +1 @@ +{"template":{"id":{"resourceType":"TASK","name":"core.type_system.custom_objects.download_result"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"res":{"type":{"simple":"STRUCT","metadata":{"$ref":"#/definitions/ResultSchema","$schema":"http://json-schema.org/draft-07/schema#","definitions":{"FlytedirectorySchema":{"additionalProperties":false,"properties":{"path":{"title":"path","type":"string"}},"type":"object"},"FlytefileSchema":{"additionalProperties":false,"properties":{"path":{"title":"path","type":"string"}},"type":"object"},"FlyteschemaSchema":{"additionalProperties":false,"properties":{"remote_path":{"title":"remote_path","type":"string"}},"type":"object"},"ResultSchema":{"additionalProperties":false,"properties":{"directory":{"$ref":"#/definitions/FlytedirectorySchema","field_many":false,"type":"object"},"file":{"$ref":"#/definitions/FlytefileSchema","field_many":false,"type":"object"},"schema":{"$ref":"#/definitions/FlyteschemaSchema","field_many":false,"type":"object"}},"type":"object"}}}},"description":"res"}}},"outputs":{}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.type_system.custom_objects","task-name","download_result"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/161_core.type_system.custom_objects.download_result_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/161_core.type_system.custom_objects.download_result_1_task.yaml new file mode 100755 index 0000000000..c81454a1a8 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/161_core.type_system.custom_objects.download_result_1_task.yaml @@ -0,0 +1,92 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.type_system.custom_objects + - task-name + - download_result + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.type_system.custom_objects.download_result + resource_type: 1 + interface: + inputs: + variables: + res: + description: res + type: + Type: + Simple: 9 + metadata: + $ref: '#/definitions/ResultSchema' + $schema: http://json-schema.org/draft-07/schema# + definitions: + FlytedirectorySchema: + additionalProperties: false + properties: + path: + title: path + type: string + type: object + FlytefileSchema: + additionalProperties: false + properties: + path: + title: path + type: string + type: object + FlyteschemaSchema: + additionalProperties: false + properties: + remote_path: + title: remote_path + type: string + type: object + ResultSchema: + additionalProperties: false + properties: + directory: + $ref: '#/definitions/FlytedirectorySchema' + field_many: false + type: object + file: + $ref: '#/definitions/FlytefileSchema' + field_many: false + type: object + schema: + $ref: '#/definitions/FlyteschemaSchema' + field_many: false + type: object + type: object + outputs: {} + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/162_core.type_system.custom_objects.wf_2_wf.json b/pkg/compiler/test/testdata/snacks-core/compiled/162_core.type_system.custom_objects.wf_2_wf.json new file mode 100755 index 0000000000..19de654c0b --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/162_core.type_system.custom_objects.wf_2_wf.json @@ -0,0 +1 @@ +{"primary":{"template":{"id":{"resourceType":"WORKFLOW","name":"core.type_system.custom_objects.wf"},"metadata":{},"interface":{"inputs":{"variables":{"x":{"type":{"simple":"INTEGER"},"description":"x"},"y":{"type":{"simple":"INTEGER"},"description":"y"}}},"outputs":{"variables":{"o0":{"type":{"simple":"STRUCT","metadata":{"$ref":"#/definitions/DatumSchema","$schema":"http://json-schema.org/draft-07/schema#","definitions":{"DatumSchema":{"additionalProperties":false,"properties":{"x":{"title":"x","type":"integer"},"y":{"title":"y","type":"string"},"z":{"additionalProperties":{"title":"z","type":"string"},"title":"z","type":"object"}},"type":"object"}}}},"description":"o0"},"o1":{"type":{"simple":"STRUCT","metadata":{"$ref":"#/definitions/ResultSchema","$schema":"http://json-schema.org/draft-07/schema#","definitions":{"FlytedirectorySchema":{"additionalProperties":false,"properties":{"path":{"title":"path","type":"string"}},"type":"object"},"FlytefileSchema":{"additionalProperties":false,"properties":{"path":{"title":"path","type":"string"}},"type":"object"},"FlyteschemaSchema":{"additionalProperties":false,"properties":{"remote_path":{"title":"remote_path","type":"string"}},"type":"object"},"ResultSchema":{"additionalProperties":false,"properties":{"directory":{"$ref":"#/definitions/FlytedirectorySchema","field_many":false,"type":"object"},"file":{"$ref":"#/definitions/FlytefileSchema","field_many":false,"type":"object"},"schema":{"$ref":"#/definitions/FlyteschemaSchema","field_many":false,"type":"object"}},"type":"object"}}}},"description":"o1"}}}},"nodes":[{"id":"start-node"},{"id":"end-node","inputs":[{"var":"o0","binding":{"promise":{"nodeId":"n4","var":"o0"}}},{"var":"o1","binding":{"promise":{"nodeId":"n0","var":"o0"}}}]},{"id":"n0","metadata":{"name":"upload_result","retries":{}},"taskNode":{"referenceId":{"resourceType":"TASK","name":"core.type_system.custom_objects.upload_result"},"overrides":{}}},{"id":"n1","metadata":{"name":"download_result","retries":{}},"inputs":[{"var":"res","binding":{"promise":{"nodeId":"n0","var":"o0"}}}],"upstreamNodeIds":["n0"],"taskNode":{"referenceId":{"resourceType":"TASK","name":"core.type_system.custom_objects.download_result"},"overrides":{}}},{"id":"n2","metadata":{"name":"stringify","retries":{}},"inputs":[{"var":"x","binding":{"promise":{"nodeId":"start-node","var":"x"}}}],"taskNode":{"referenceId":{"resourceType":"TASK","name":"core.type_system.custom_objects.stringify"},"overrides":{}}},{"id":"n3","metadata":{"name":"stringify","retries":{}},"inputs":[{"var":"x","binding":{"promise":{"nodeId":"start-node","var":"y"}}}],"taskNode":{"referenceId":{"resourceType":"TASK","name":"core.type_system.custom_objects.stringify"},"overrides":{}}},{"id":"n4","metadata":{"name":"add","retries":{}},"inputs":[{"var":"x","binding":{"promise":{"nodeId":"n2","var":"o0"}}},{"var":"y","binding":{"promise":{"nodeId":"n3","var":"o0"}}}],"upstreamNodeIds":["n2","n3"],"taskNode":{"referenceId":{"resourceType":"TASK","name":"core.type_system.custom_objects.add"},"overrides":{}}}],"outputs":[{"var":"o0","binding":{"promise":{"nodeId":"n4","var":"o0"}}},{"var":"o1","binding":{"promise":{"nodeId":"n0","var":"o0"}}}],"metadataDefaults":{}},"connections":{"downstream":{"n0":{"ids":["end-node","n1"]},"n1":{"ids":["end-node"]},"n2":{"ids":["n4"]},"n3":{"ids":["n4"]},"n4":{"ids":["end-node"]},"start-node":{"ids":["n0","n2","n3"]}},"upstream":{"end-node":{"ids":["n0","n1","n4"]},"n0":{"ids":["start-node"]},"n1":{"ids":["n0"]},"n2":{"ids":["start-node"]},"n3":{"ids":["start-node"]},"n4":{"ids":["n2","n3"]}}}},"tasks":[{"template":{"id":{"resourceType":"TASK","name":"core.type_system.custom_objects.add"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"x":{"type":{"simple":"STRUCT","metadata":{"$ref":"#/definitions/DatumSchema","$schema":"http://json-schema.org/draft-07/schema#","definitions":{"DatumSchema":{"additionalProperties":false,"properties":{"x":{"title":"x","type":"integer"},"y":{"title":"y","type":"string"},"z":{"additionalProperties":{"title":"z","type":"string"},"title":"z","type":"object"}},"type":"object"}}}},"description":"x"},"y":{"type":{"simple":"STRUCT","metadata":{"$ref":"#/definitions/DatumSchema","$schema":"http://json-schema.org/draft-07/schema#","definitions":{"DatumSchema":{"additionalProperties":false,"properties":{"x":{"title":"x","type":"integer"},"y":{"title":"y","type":"string"},"z":{"additionalProperties":{"title":"z","type":"string"},"title":"z","type":"object"}},"type":"object"}}}},"description":"y"}}},"outputs":{"variables":{"o0":{"type":{"simple":"STRUCT","metadata":{"$ref":"#/definitions/DatumSchema","$schema":"http://json-schema.org/draft-07/schema#","definitions":{"DatumSchema":{"additionalProperties":false,"properties":{"x":{"title":"x","type":"integer"},"y":{"title":"y","type":"string"},"z":{"additionalProperties":{"title":"z","type":"string"},"title":"z","type":"object"}},"type":"object"}}}},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.type_system.custom_objects","task-name","add"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},{"template":{"id":{"resourceType":"TASK","name":"core.type_system.custom_objects.download_result"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"res":{"type":{"simple":"STRUCT","metadata":{"$ref":"#/definitions/ResultSchema","$schema":"http://json-schema.org/draft-07/schema#","definitions":{"FlytedirectorySchema":{"additionalProperties":false,"properties":{"path":{"title":"path","type":"string"}},"type":"object"},"FlytefileSchema":{"additionalProperties":false,"properties":{"path":{"title":"path","type":"string"}},"type":"object"},"FlyteschemaSchema":{"additionalProperties":false,"properties":{"remote_path":{"title":"remote_path","type":"string"}},"type":"object"},"ResultSchema":{"additionalProperties":false,"properties":{"directory":{"$ref":"#/definitions/FlytedirectorySchema","field_many":false,"type":"object"},"file":{"$ref":"#/definitions/FlytefileSchema","field_many":false,"type":"object"},"schema":{"$ref":"#/definitions/FlyteschemaSchema","field_many":false,"type":"object"}},"type":"object"}}}},"description":"res"}}},"outputs":{}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.type_system.custom_objects","task-name","download_result"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},{"template":{"id":{"resourceType":"TASK","name":"core.type_system.custom_objects.stringify"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"x":{"type":{"simple":"INTEGER"},"description":"x"}}},"outputs":{"variables":{"o0":{"type":{"simple":"STRUCT","metadata":{"$ref":"#/definitions/DatumSchema","$schema":"http://json-schema.org/draft-07/schema#","definitions":{"DatumSchema":{"additionalProperties":false,"properties":{"x":{"title":"x","type":"integer"},"y":{"title":"y","type":"string"},"z":{"additionalProperties":{"title":"z","type":"string"},"title":"z","type":"object"}},"type":"object"}}}},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.type_system.custom_objects","task-name","stringify"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},{"template":{"id":{"resourceType":"TASK","name":"core.type_system.custom_objects.upload_result"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{},"outputs":{"variables":{"o0":{"type":{"simple":"STRUCT","metadata":{"$ref":"#/definitions/ResultSchema","$schema":"http://json-schema.org/draft-07/schema#","definitions":{"FlytedirectorySchema":{"additionalProperties":false,"properties":{"path":{"title":"path","type":"string"}},"type":"object"},"FlytefileSchema":{"additionalProperties":false,"properties":{"path":{"title":"path","type":"string"}},"type":"object"},"FlyteschemaSchema":{"additionalProperties":false,"properties":{"remote_path":{"title":"remote_path","type":"string"}},"type":"object"},"ResultSchema":{"additionalProperties":false,"properties":{"directory":{"$ref":"#/definitions/FlytedirectorySchema","field_many":false,"type":"object"},"file":{"$ref":"#/definitions/FlytefileSchema","field_many":false,"type":"object"},"schema":{"$ref":"#/definitions/FlyteschemaSchema","field_many":false,"type":"object"}},"type":"object"}}}},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.type_system.custom_objects","task-name","upload_result"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}}]} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/162_core.type_system.custom_objects.wf_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/162_core.type_system.custom_objects.wf_2_wf.yaml new file mode 100755 index 0000000000..309ba81034 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/162_core.type_system.custom_objects.wf_2_wf.yaml @@ -0,0 +1,652 @@ +primary: + connections: + downstream: + n0: + ids: + - end-node + - n1 + n1: + ids: + - end-node + n2: + ids: + - n4 + n3: + ids: + - n4 + n4: + ids: + - end-node + start-node: + ids: + - n0 + - n2 + - n3 + upstream: + end-node: + ids: + - n0 + - n1 + - n4 + n0: + ids: + - start-node + n1: + ids: + - n0 + n2: + ids: + - start-node + n3: + ids: + - start-node + n4: + ids: + - n2 + - n3 + template: + id: + name: core.type_system.custom_objects.wf + resource_type: 2 + interface: + inputs: + variables: + x: + description: x + type: + Type: + Simple: 1 + "y": + description: "y" + type: + Type: + Simple: 1 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 9 + metadata: + $ref: '#/definitions/DatumSchema' + $schema: http://json-schema.org/draft-07/schema# + definitions: + DatumSchema: + additionalProperties: false + properties: + x: + title: x + type: integer + "y": + title: "y" + type: string + z: + additionalProperties: + title: z + type: string + title: z + type: object + type: object + o1: + description: o1 + type: + Type: + Simple: 9 + metadata: + $ref: '#/definitions/ResultSchema' + $schema: http://json-schema.org/draft-07/schema# + definitions: + FlytedirectorySchema: + additionalProperties: false + properties: + path: + title: path + type: string + type: object + FlytefileSchema: + additionalProperties: false + properties: + path: + title: path + type: string + type: object + FlyteschemaSchema: + additionalProperties: false + properties: + remote_path: + title: remote_path + type: string + type: object + ResultSchema: + additionalProperties: false + properties: + directory: + $ref: '#/definitions/FlytedirectorySchema' + field_many: false + type: object + file: + $ref: '#/definitions/FlytefileSchema' + field_many: false + type: object + schema: + $ref: '#/definitions/FlyteschemaSchema' + field_many: false + type: object + type: object + metadata: {} + metadata_defaults: {} + nodes: + - Target: null + id: start-node + - Target: null + id: end-node + inputs: + - binding: + Value: + Promise: + node_id: n4 + var: o0 + var: o0 + - binding: + Value: + Promise: + node_id: n0 + var: o0 + var: o1 + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.type_system.custom_objects.upload_result + resource_type: 1 + overrides: {} + id: n0 + metadata: + InterruptibleValue: null + name: upload_result + retries: {} + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.type_system.custom_objects.download_result + resource_type: 1 + overrides: {} + id: n1 + inputs: + - binding: + Value: + Promise: + node_id: n0 + var: o0 + var: res + metadata: + InterruptibleValue: null + name: download_result + retries: {} + upstream_node_ids: + - n0 + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.type_system.custom_objects.stringify + resource_type: 1 + overrides: {} + id: n2 + inputs: + - binding: + Value: + Promise: + node_id: start-node + var: x + var: x + metadata: + InterruptibleValue: null + name: stringify + retries: {} + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.type_system.custom_objects.stringify + resource_type: 1 + overrides: {} + id: n3 + inputs: + - binding: + Value: + Promise: + node_id: start-node + var: "y" + var: x + metadata: + InterruptibleValue: null + name: stringify + retries: {} + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.type_system.custom_objects.add + resource_type: 1 + overrides: {} + id: n4 + inputs: + - binding: + Value: + Promise: + node_id: n2 + var: o0 + var: x + - binding: + Value: + Promise: + node_id: n3 + var: o0 + var: "y" + metadata: + InterruptibleValue: null + name: add + retries: {} + upstream_node_ids: + - n2 + - n3 + outputs: + - binding: + Value: + Promise: + node_id: n4 + var: o0 + var: o0 + - binding: + Value: + Promise: + node_id: n0 + var: o0 + var: o1 +tasks: +- template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.type_system.custom_objects + - task-name + - add + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.type_system.custom_objects.add + resource_type: 1 + interface: + inputs: + variables: + x: + description: x + type: + Type: + Simple: 9 + metadata: + $ref: '#/definitions/DatumSchema' + $schema: http://json-schema.org/draft-07/schema# + definitions: + DatumSchema: + additionalProperties: false + properties: + x: + title: x + type: integer + "y": + title: "y" + type: string + z: + additionalProperties: + title: z + type: string + title: z + type: object + type: object + "y": + description: "y" + type: + Type: + Simple: 9 + metadata: + $ref: '#/definitions/DatumSchema' + $schema: http://json-schema.org/draft-07/schema# + definitions: + DatumSchema: + additionalProperties: false + properties: + x: + title: x + type: integer + "y": + title: "y" + type: string + z: + additionalProperties: + title: z + type: string + title: z + type: object + type: object + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 9 + metadata: + $ref: '#/definitions/DatumSchema' + $schema: http://json-schema.org/draft-07/schema# + definitions: + DatumSchema: + additionalProperties: false + properties: + x: + title: x + type: integer + "y": + title: "y" + type: string + z: + additionalProperties: + title: z + type: string + title: z + type: object + type: object + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task +- template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.type_system.custom_objects + - task-name + - download_result + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.type_system.custom_objects.download_result + resource_type: 1 + interface: + inputs: + variables: + res: + description: res + type: + Type: + Simple: 9 + metadata: + $ref: '#/definitions/ResultSchema' + $schema: http://json-schema.org/draft-07/schema# + definitions: + FlytedirectorySchema: + additionalProperties: false + properties: + path: + title: path + type: string + type: object + FlytefileSchema: + additionalProperties: false + properties: + path: + title: path + type: string + type: object + FlyteschemaSchema: + additionalProperties: false + properties: + remote_path: + title: remote_path + type: string + type: object + ResultSchema: + additionalProperties: false + properties: + directory: + $ref: '#/definitions/FlytedirectorySchema' + field_many: false + type: object + file: + $ref: '#/definitions/FlytefileSchema' + field_many: false + type: object + schema: + $ref: '#/definitions/FlyteschemaSchema' + field_many: false + type: object + type: object + outputs: {} + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task +- template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.type_system.custom_objects + - task-name + - stringify + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.type_system.custom_objects.stringify + resource_type: 1 + interface: + inputs: + variables: + x: + description: x + type: + Type: + Simple: 1 + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 9 + metadata: + $ref: '#/definitions/DatumSchema' + $schema: http://json-schema.org/draft-07/schema# + definitions: + DatumSchema: + additionalProperties: false + properties: + x: + title: x + type: integer + "y": + title: "y" + type: string + z: + additionalProperties: + title: z + type: string + title: z + type: object + type: object + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task +- template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.type_system.custom_objects + - task-name + - upload_result + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.type_system.custom_objects.upload_result + resource_type: 1 + interface: + inputs: {} + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 9 + metadata: + $ref: '#/definitions/ResultSchema' + $schema: http://json-schema.org/draft-07/schema# + definitions: + FlytedirectorySchema: + additionalProperties: false + properties: + path: + title: path + type: string + type: object + FlytefileSchema: + additionalProperties: false + properties: + path: + title: path + type: string + type: object + FlyteschemaSchema: + additionalProperties: false + properties: + remote_path: + title: remote_path + type: string + type: object + ResultSchema: + additionalProperties: false + properties: + directory: + $ref: '#/definitions/FlytedirectorySchema' + field_many: false + type: object + file: + $ref: '#/definitions/FlytefileSchema' + field_many: false + type: object + schema: + $ref: '#/definitions/FlyteschemaSchema' + field_many: false + type: object + type: object + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/164_core.type_system.enums.enum_stringify_1_task.json b/pkg/compiler/test/testdata/snacks-core/compiled/164_core.type_system.enums.enum_stringify_1_task.json new file mode 100755 index 0000000000..eca10f268b --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/164_core.type_system.enums.enum_stringify_1_task.json @@ -0,0 +1 @@ +{"template":{"id":{"resourceType":"TASK","name":"core.type_system.enums.enum_stringify"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"c":{"type":{"enumType":{"values":["red","green","blue"]}},"description":"c"}}},"outputs":{"variables":{"o0":{"type":{"simple":"STRING"},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.type_system.enums","task-name","enum_stringify"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/164_core.type_system.enums.enum_stringify_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/164_core.type_system.enums.enum_stringify_1_task.yaml new file mode 100755 index 0000000000..0fae7d3a9c --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/164_core.type_system.enums.enum_stringify_1_task.yaml @@ -0,0 +1,61 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.type_system.enums + - task-name + - enum_stringify + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.type_system.enums.enum_stringify + resource_type: 1 + interface: + inputs: + variables: + c: + description: c + type: + Type: + EnumType: + values: + - red + - green + - blue + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 3 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/165_core.type_system.enums.string_to_enum_1_task.json b/pkg/compiler/test/testdata/snacks-core/compiled/165_core.type_system.enums.string_to_enum_1_task.json new file mode 100755 index 0000000000..0e72476280 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/165_core.type_system.enums.string_to_enum_1_task.json @@ -0,0 +1 @@ +{"template":{"id":{"resourceType":"TASK","name":"core.type_system.enums.string_to_enum"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"c":{"type":{"simple":"STRING"},"description":"c"}}},"outputs":{"variables":{"o0":{"type":{"enumType":{"values":["red","green","blue"]}},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.type_system.enums","task-name","string_to_enum"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/165_core.type_system.enums.string_to_enum_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/165_core.type_system.enums.string_to_enum_1_task.yaml new file mode 100755 index 0000000000..b462494f06 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/165_core.type_system.enums.string_to_enum_1_task.yaml @@ -0,0 +1,61 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.type_system.enums + - task-name + - string_to_enum + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.type_system.enums.string_to_enum + resource_type: 1 + interface: + inputs: + variables: + c: + description: c + type: + Type: + Simple: 3 + outputs: + variables: + o0: + description: o0 + type: + Type: + EnumType: + values: + - red + - green + - blue + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/166_core.type_system.enums.enum_wf_2_wf.json b/pkg/compiler/test/testdata/snacks-core/compiled/166_core.type_system.enums.enum_wf_2_wf.json new file mode 100755 index 0000000000..d7138ddcd3 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/166_core.type_system.enums.enum_wf_2_wf.json @@ -0,0 +1 @@ +{"primary":{"template":{"id":{"resourceType":"WORKFLOW","name":"core.type_system.enums.enum_wf"},"metadata":{},"interface":{"inputs":{"variables":{"c":{"type":{"enumType":{"values":["red","green","blue"]}},"description":"c"}}},"outputs":{"variables":{"o0":{"type":{"enumType":{"values":["red","green","blue"]}},"description":"o0"},"o1":{"type":{"simple":"STRING"},"description":"o1"}}}},"nodes":[{"id":"start-node"},{"id":"end-node","inputs":[{"var":"o0","binding":{"promise":{"nodeId":"n1","var":"o0"}}},{"var":"o1","binding":{"promise":{"nodeId":"n0","var":"o0"}}}]},{"id":"n0","metadata":{"name":"enum_stringify","retries":{}},"inputs":[{"var":"c","binding":{"promise":{"nodeId":"start-node","var":"c"}}}],"taskNode":{"referenceId":{"resourceType":"TASK","name":"core.type_system.enums.enum_stringify"},"overrides":{}}},{"id":"n1","metadata":{"name":"string_to_enum","retries":{}},"inputs":[{"var":"c","binding":{"promise":{"nodeId":"n0","var":"o0"}}}],"upstreamNodeIds":["n0"],"taskNode":{"referenceId":{"resourceType":"TASK","name":"core.type_system.enums.string_to_enum"},"overrides":{}}}],"outputs":[{"var":"o0","binding":{"promise":{"nodeId":"n1","var":"o0"}}},{"var":"o1","binding":{"promise":{"nodeId":"n0","var":"o0"}}}],"metadataDefaults":{}},"connections":{"downstream":{"n0":{"ids":["end-node","n1"]},"n1":{"ids":["end-node"]},"start-node":{"ids":["n0"]}},"upstream":{"end-node":{"ids":["n0","n1"]},"n0":{"ids":["start-node"]},"n1":{"ids":["n0"]}}}},"tasks":[{"template":{"id":{"resourceType":"TASK","name":"core.type_system.enums.enum_stringify"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"c":{"type":{"enumType":{"values":["red","green","blue"]}},"description":"c"}}},"outputs":{"variables":{"o0":{"type":{"simple":"STRING"},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.type_system.enums","task-name","enum_stringify"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},{"template":{"id":{"resourceType":"TASK","name":"core.type_system.enums.string_to_enum"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"c":{"type":{"simple":"STRING"},"description":"c"}}},"outputs":{"variables":{"o0":{"type":{"enumType":{"values":["red","green","blue"]}},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.type_system.enums","task-name","string_to_enum"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}}]} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/166_core.type_system.enums.enum_wf_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/166_core.type_system.enums.enum_wf_2_wf.yaml new file mode 100755 index 0000000000..bba8ffd73b --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/166_core.type_system.enums.enum_wf_2_wf.yaml @@ -0,0 +1,252 @@ +primary: + connections: + downstream: + n0: + ids: + - end-node + - n1 + n1: + ids: + - end-node + start-node: + ids: + - n0 + upstream: + end-node: + ids: + - n0 + - n1 + n0: + ids: + - start-node + n1: + ids: + - n0 + template: + id: + name: core.type_system.enums.enum_wf + resource_type: 2 + interface: + inputs: + variables: + c: + description: c + type: + Type: + EnumType: + values: + - red + - green + - blue + outputs: + variables: + o0: + description: o0 + type: + Type: + EnumType: + values: + - red + - green + - blue + o1: + description: o1 + type: + Type: + Simple: 3 + metadata: {} + metadata_defaults: {} + nodes: + - Target: null + id: start-node + - Target: null + id: end-node + inputs: + - binding: + Value: + Promise: + node_id: n1 + var: o0 + var: o0 + - binding: + Value: + Promise: + node_id: n0 + var: o0 + var: o1 + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.type_system.enums.enum_stringify + resource_type: 1 + overrides: {} + id: n0 + inputs: + - binding: + Value: + Promise: + node_id: start-node + var: c + var: c + metadata: + InterruptibleValue: null + name: enum_stringify + retries: {} + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.type_system.enums.string_to_enum + resource_type: 1 + overrides: {} + id: n1 + inputs: + - binding: + Value: + Promise: + node_id: n0 + var: o0 + var: c + metadata: + InterruptibleValue: null + name: string_to_enum + retries: {} + upstream_node_ids: + - n0 + outputs: + - binding: + Value: + Promise: + node_id: n1 + var: o0 + var: o0 + - binding: + Value: + Promise: + node_id: n0 + var: o0 + var: o1 +tasks: +- template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.type_system.enums + - task-name + - enum_stringify + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.type_system.enums.enum_stringify + resource_type: 1 + interface: + inputs: + variables: + c: + description: c + type: + Type: + EnumType: + values: + - red + - green + - blue + outputs: + variables: + o0: + description: o0 + type: + Type: + Simple: 3 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task +- template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.type_system.enums + - task-name + - string_to_enum + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.type_system.enums.string_to_enum + resource_type: 1 + interface: + inputs: + variables: + c: + description: c + type: + Type: + Simple: 3 + outputs: + variables: + o0: + description: o0 + type: + Type: + EnumType: + values: + - red + - green + - blue + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/168_core.type_system.flyte_pickle.greet_1_task.json b/pkg/compiler/test/testdata/snacks-core/compiled/168_core.type_system.flyte_pickle.greet_1_task.json new file mode 100755 index 0000000000..a526ad6d9f --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/168_core.type_system.flyte_pickle.greet_1_task.json @@ -0,0 +1 @@ +{"template":{"id":{"resourceType":"TASK","name":"core.type_system.flyte_pickle.greet"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"name":{"type":{"simple":"STRING"},"description":"name"}}},"outputs":{"variables":{"o0":{"type":{"blob":{"format":"PythonPickle"},"metadata":{"python_class_name":"People"}},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.type_system.flyte_pickle","task-name","greet"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/168_core.type_system.flyte_pickle.greet_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/168_core.type_system.flyte_pickle.greet_1_task.yaml new file mode 100755 index 0000000000..369fb4eb60 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/168_core.type_system.flyte_pickle.greet_1_task.yaml @@ -0,0 +1,60 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.type_system.flyte_pickle + - task-name + - greet + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.type_system.flyte_pickle.greet + resource_type: 1 + interface: + inputs: + variables: + name: + description: name + type: + Type: + Simple: 3 + outputs: + variables: + o0: + description: o0 + type: + Type: + Blob: + format: PythonPickle + metadata: + python_class_name: People + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/169_core.type_system.flyte_pickle.welcome_2_wf.json b/pkg/compiler/test/testdata/snacks-core/compiled/169_core.type_system.flyte_pickle.welcome_2_wf.json new file mode 100755 index 0000000000..0b70aad970 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/169_core.type_system.flyte_pickle.welcome_2_wf.json @@ -0,0 +1 @@ +{"primary":{"template":{"id":{"resourceType":"WORKFLOW","name":"core.type_system.flyte_pickle.welcome"},"metadata":{},"interface":{"inputs":{"variables":{"name":{"type":{"simple":"STRING"},"description":"name"}}},"outputs":{"variables":{"o0":{"type":{"blob":{"format":"PythonPickle"},"metadata":{"python_class_name":"People"}},"description":"o0"}}}},"nodes":[{"id":"start-node"},{"id":"end-node","inputs":[{"var":"o0","binding":{"promise":{"nodeId":"n0","var":"o0"}}}]},{"id":"n0","metadata":{"name":"greet","retries":{}},"inputs":[{"var":"name","binding":{"promise":{"nodeId":"start-node","var":"name"}}}],"taskNode":{"referenceId":{"resourceType":"TASK","name":"core.type_system.flyte_pickle.greet"},"overrides":{}}}],"outputs":[{"var":"o0","binding":{"promise":{"nodeId":"n0","var":"o0"}}}],"metadataDefaults":{}},"connections":{"downstream":{"n0":{"ids":["end-node"]},"start-node":{"ids":["n0"]}},"upstream":{"end-node":{"ids":["n0"]},"n0":{"ids":["start-node"]}}}},"tasks":[{"template":{"id":{"resourceType":"TASK","name":"core.type_system.flyte_pickle.greet"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"name":{"type":{"simple":"STRING"},"description":"name"}}},"outputs":{"variables":{"o0":{"type":{"blob":{"format":"PythonPickle"},"metadata":{"python_class_name":"People"}},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.type_system.flyte_pickle","task-name","greet"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}}]} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/169_core.type_system.flyte_pickle.welcome_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/169_core.type_system.flyte_pickle.welcome_2_wf.yaml new file mode 100755 index 0000000000..4e64513d35 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/169_core.type_system.flyte_pickle.welcome_2_wf.yaml @@ -0,0 +1,139 @@ +primary: + connections: + downstream: + n0: + ids: + - end-node + start-node: + ids: + - n0 + upstream: + end-node: + ids: + - n0 + n0: + ids: + - start-node + template: + id: + name: core.type_system.flyte_pickle.welcome + resource_type: 2 + interface: + inputs: + variables: + name: + description: name + type: + Type: + Simple: 3 + outputs: + variables: + o0: + description: o0 + type: + Type: + Blob: + format: PythonPickle + metadata: + python_class_name: People + metadata: {} + metadata_defaults: {} + nodes: + - Target: null + id: start-node + - Target: null + id: end-node + inputs: + - binding: + Value: + Promise: + node_id: n0 + var: o0 + var: o0 + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.type_system.flyte_pickle.greet + resource_type: 1 + overrides: {} + id: n0 + inputs: + - binding: + Value: + Promise: + node_id: start-node + var: name + var: name + metadata: + InterruptibleValue: null + name: greet + retries: {} + outputs: + - binding: + Value: + Promise: + node_id: n0 + var: o0 + var: o0 +tasks: +- template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.type_system.flyte_pickle + - task-name + - greet + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.type_system.flyte_pickle.greet + resource_type: 1 + interface: + inputs: + variables: + name: + description: name + type: + Type: + Simple: 3 + outputs: + variables: + o0: + description: o0 + type: + Type: + Blob: + format: PythonPickle + metadata: + python_class_name: People + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/171_core.type_system.schema.get_df_1_task.json b/pkg/compiler/test/testdata/snacks-core/compiled/171_core.type_system.schema.get_df_1_task.json new file mode 100755 index 0000000000..5534e1938b --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/171_core.type_system.schema.get_df_1_task.json @@ -0,0 +1 @@ +{"template":{"id":{"resourceType":"TASK","name":"core.type_system.schema.get_df"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"a":{"type":{"simple":"INTEGER"},"description":"a"}}},"outputs":{"variables":{"o0":{"type":{"structuredDatasetType":{"format":"parquet"}},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.type_system.schema","task-name","get_df"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/171_core.type_system.schema.get_df_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/171_core.type_system.schema.get_df_1_task.yaml new file mode 100755 index 0000000000..2d3cc79ba0 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/171_core.type_system.schema.get_df_1_task.yaml @@ -0,0 +1,58 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.type_system.schema + - task-name + - get_df + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.type_system.schema.get_df + resource_type: 1 + interface: + inputs: + variables: + a: + description: a + type: + Type: + Simple: 1 + outputs: + variables: + o0: + description: o0 + type: + Type: + StructuredDatasetType: + format: parquet + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/172_core.type_system.schema.add_df_1_task.json b/pkg/compiler/test/testdata/snacks-core/compiled/172_core.type_system.schema.add_df_1_task.json new file mode 100755 index 0000000000..2b07493ac2 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/172_core.type_system.schema.add_df_1_task.json @@ -0,0 +1 @@ +{"template":{"id":{"resourceType":"TASK","name":"core.type_system.schema.add_df"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"df":{"type":{"structuredDatasetType":{"format":"parquet"}},"description":"df"}}},"outputs":{"variables":{"o0":{"type":{"structuredDatasetType":{"format":"parquet"}},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.type_system.schema","task-name","add_df"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/172_core.type_system.schema.add_df_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/172_core.type_system.schema.add_df_1_task.yaml new file mode 100755 index 0000000000..1aa87e184c --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/172_core.type_system.schema.add_df_1_task.yaml @@ -0,0 +1,59 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.type_system.schema + - task-name + - add_df + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.type_system.schema.add_df + resource_type: 1 + interface: + inputs: + variables: + df: + description: df + type: + Type: + StructuredDatasetType: + format: parquet + outputs: + variables: + o0: + description: o0 + type: + Type: + StructuredDatasetType: + format: parquet + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/173_core.type_system.schema.df_wf_2_wf.json b/pkg/compiler/test/testdata/snacks-core/compiled/173_core.type_system.schema.df_wf_2_wf.json new file mode 100755 index 0000000000..9525ad31a6 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/173_core.type_system.schema.df_wf_2_wf.json @@ -0,0 +1 @@ +{"primary":{"template":{"id":{"resourceType":"WORKFLOW","name":"core.type_system.schema.df_wf"},"metadata":{},"interface":{"inputs":{"variables":{"a":{"type":{"simple":"INTEGER"},"description":"a"}}},"outputs":{"variables":{"o0":{"type":{"structuredDatasetType":{"format":"parquet"}},"description":"o0"}}}},"nodes":[{"id":"start-node"},{"id":"end-node","inputs":[{"var":"o0","binding":{"promise":{"nodeId":"n1","var":"o0"}}}]},{"id":"n0","metadata":{"name":"get_df","retries":{}},"inputs":[{"var":"a","binding":{"promise":{"nodeId":"start-node","var":"a"}}}],"taskNode":{"referenceId":{"resourceType":"TASK","name":"core.type_system.schema.get_df"},"overrides":{}}},{"id":"n1","metadata":{"name":"add_df","retries":{}},"inputs":[{"var":"df","binding":{"promise":{"nodeId":"n0","var":"o0"}}}],"upstreamNodeIds":["n0"],"taskNode":{"referenceId":{"resourceType":"TASK","name":"core.type_system.schema.add_df"},"overrides":{}}}],"outputs":[{"var":"o0","binding":{"promise":{"nodeId":"n1","var":"o0"}}}],"metadataDefaults":{}},"connections":{"downstream":{"n0":{"ids":["n1"]},"n1":{"ids":["end-node"]},"start-node":{"ids":["n0"]}},"upstream":{"end-node":{"ids":["n1"]},"n0":{"ids":["start-node"]},"n1":{"ids":["n0"]}}}},"tasks":[{"template":{"id":{"resourceType":"TASK","name":"core.type_system.schema.add_df"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"df":{"type":{"structuredDatasetType":{"format":"parquet"}},"description":"df"}}},"outputs":{"variables":{"o0":{"type":{"structuredDatasetType":{"format":"parquet"}},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.type_system.schema","task-name","add_df"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},{"template":{"id":{"resourceType":"TASK","name":"core.type_system.schema.get_df"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"a":{"type":{"simple":"INTEGER"},"description":"a"}}},"outputs":{"variables":{"o0":{"type":{"structuredDatasetType":{"format":"parquet"}},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.type_system.schema","task-name","get_df"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}}]} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/173_core.type_system.schema.df_wf_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/173_core.type_system.schema.df_wf_2_wf.yaml new file mode 100755 index 0000000000..0705cf08fd --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/173_core.type_system.schema.df_wf_2_wf.yaml @@ -0,0 +1,221 @@ +primary: + connections: + downstream: + n0: + ids: + - n1 + n1: + ids: + - end-node + start-node: + ids: + - n0 + upstream: + end-node: + ids: + - n1 + n0: + ids: + - start-node + n1: + ids: + - n0 + template: + id: + name: core.type_system.schema.df_wf + resource_type: 2 + interface: + inputs: + variables: + a: + description: a + type: + Type: + Simple: 1 + outputs: + variables: + o0: + description: o0 + type: + Type: + StructuredDatasetType: + format: parquet + metadata: {} + metadata_defaults: {} + nodes: + - Target: null + id: start-node + - Target: null + id: end-node + inputs: + - binding: + Value: + Promise: + node_id: n1 + var: o0 + var: o0 + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.type_system.schema.get_df + resource_type: 1 + overrides: {} + id: n0 + inputs: + - binding: + Value: + Promise: + node_id: start-node + var: a + var: a + metadata: + InterruptibleValue: null + name: get_df + retries: {} + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.type_system.schema.add_df + resource_type: 1 + overrides: {} + id: n1 + inputs: + - binding: + Value: + Promise: + node_id: n0 + var: o0 + var: df + metadata: + InterruptibleValue: null + name: add_df + retries: {} + upstream_node_ids: + - n0 + outputs: + - binding: + Value: + Promise: + node_id: n1 + var: o0 + var: o0 +tasks: +- template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.type_system.schema + - task-name + - add_df + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.type_system.schema.add_df + resource_type: 1 + interface: + inputs: + variables: + df: + description: df + type: + Type: + StructuredDatasetType: + format: parquet + outputs: + variables: + o0: + description: o0 + type: + Type: + StructuredDatasetType: + format: parquet + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task +- template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.type_system.schema + - task-name + - get_df + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.type_system.schema.get_df + resource_type: 1 + interface: + inputs: + variables: + a: + description: a + type: + Type: + Simple: 1 + outputs: + variables: + o0: + description: o0 + type: + Type: + StructuredDatasetType: + format: parquet + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/175_core.type_system.structured_dataset.get_df_1_task.json b/pkg/compiler/test/testdata/snacks-core/compiled/175_core.type_system.structured_dataset.get_df_1_task.json new file mode 100755 index 0000000000..7b5f4f4928 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/175_core.type_system.structured_dataset.get_df_1_task.json @@ -0,0 +1 @@ +{"template":{"id":{"resourceType":"TASK","name":"core.type_system.structured_dataset.get_df"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"a":{"type":{"simple":"INTEGER"},"description":"a"}}},"outputs":{"variables":{"o0":{"type":{"structuredDatasetType":{"columns":[{"name":"Name","literalType":{"simple":"STRING"}},{"name":"Age","literalType":{"simple":"INTEGER"}},{"name":"Height","literalType":{"simple":"INTEGER"}}],"format":"parquet"}},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.type_system.structured_dataset","task-name","get_df"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/175_core.type_system.structured_dataset.get_df_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/175_core.type_system.structured_dataset.get_df_1_task.yaml new file mode 100755 index 0000000000..67a7f94216 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/175_core.type_system.structured_dataset.get_df_1_task.yaml @@ -0,0 +1,71 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.type_system.structured_dataset + - task-name + - get_df + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.type_system.structured_dataset.get_df + resource_type: 1 + interface: + inputs: + variables: + a: + description: a + type: + Type: + Simple: 1 + outputs: + variables: + o0: + description: o0 + type: + Type: + StructuredDatasetType: + columns: + - literal_type: + Type: + Simple: 3 + name: Name + - literal_type: + Type: + Simple: 1 + name: Age + - literal_type: + Type: + Simple: 1 + name: Height + format: parquet + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/176_core.type_system.structured_dataset.get_schema_df_1_task.json b/pkg/compiler/test/testdata/snacks-core/compiled/176_core.type_system.structured_dataset.get_schema_df_1_task.json new file mode 100755 index 0000000000..b3c304c80d --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/176_core.type_system.structured_dataset.get_schema_df_1_task.json @@ -0,0 +1 @@ +{"template":{"id":{"resourceType":"TASK","name":"core.type_system.structured_dataset.get_schema_df"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"a":{"type":{"simple":"INTEGER"},"description":"a"}}},"outputs":{"variables":{"o0":{"type":{"schema":{"columns":[{"name":"Name","type":"STRING"},{"name":"Age"},{"name":"Height"}]}},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.type_system.structured_dataset","task-name","get_schema_df"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/176_core.type_system.structured_dataset.get_schema_df_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/176_core.type_system.structured_dataset.get_schema_df_1_task.yaml new file mode 100755 index 0000000000..1acf451aee --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/176_core.type_system.structured_dataset.get_schema_df_1_task.yaml @@ -0,0 +1,62 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.type_system.structured_dataset + - task-name + - get_schema_df + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.type_system.structured_dataset.get_schema_df + resource_type: 1 + interface: + inputs: + variables: + a: + description: a + type: + Type: + Simple: 1 + outputs: + variables: + o0: + description: o0 + type: + Type: + Schema: + columns: + - name: Name + type: 2 + - name: Age + - name: Height + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/177_core.type_system.structured_dataset.get_subset_df_1_task.json b/pkg/compiler/test/testdata/snacks-core/compiled/177_core.type_system.structured_dataset.get_subset_df_1_task.json new file mode 100755 index 0000000000..55abb2d7d8 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/177_core.type_system.structured_dataset.get_subset_df_1_task.json @@ -0,0 +1 @@ +{"template":{"id":{"resourceType":"TASK","name":"core.type_system.structured_dataset.get_subset_df"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"df":{"type":{"structuredDatasetType":{"columns":[{"name":"Age","literalType":{"simple":"INTEGER"}}],"format":"parquet"}},"description":"df"}}},"outputs":{"variables":{"o0":{"type":{"structuredDatasetType":{"columns":[{"name":"Age","literalType":{"simple":"INTEGER"}}],"format":"parquet"}},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.type_system.structured_dataset","task-name","get_subset_df"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/177_core.type_system.structured_dataset.get_subset_df_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/177_core.type_system.structured_dataset.get_subset_df_1_task.yaml new file mode 100755 index 0000000000..c971e8c682 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/177_core.type_system.structured_dataset.get_subset_df_1_task.yaml @@ -0,0 +1,69 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.type_system.structured_dataset + - task-name + - get_subset_df + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.type_system.structured_dataset.get_subset_df + resource_type: 1 + interface: + inputs: + variables: + df: + description: df + type: + Type: + StructuredDatasetType: + columns: + - literal_type: + Type: + Simple: 1 + name: Age + format: parquet + outputs: + variables: + o0: + description: o0 + type: + Type: + StructuredDatasetType: + columns: + - literal_type: + Type: + Simple: 1 + name: Age + format: parquet + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/178_core.type_system.structured_dataset.to_numpy_1_task.json b/pkg/compiler/test/testdata/snacks-core/compiled/178_core.type_system.structured_dataset.to_numpy_1_task.json new file mode 100755 index 0000000000..3e71837c5d --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/178_core.type_system.structured_dataset.to_numpy_1_task.json @@ -0,0 +1 @@ +{"template":{"id":{"resourceType":"TASK","name":"core.type_system.structured_dataset.to_numpy"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"ds":{"type":{"structuredDatasetType":{"columns":[{"name":"Age","literalType":{"simple":"INTEGER"}}],"format":"parquet"}},"description":"ds"}}},"outputs":{"variables":{"o0":{"type":{"structuredDatasetType":{"columns":[{"name":"Age","literalType":{"simple":"INTEGER"}}],"format":"parquet"}},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.type_system.structured_dataset","task-name","to_numpy"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/178_core.type_system.structured_dataset.to_numpy_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/178_core.type_system.structured_dataset.to_numpy_1_task.yaml new file mode 100755 index 0000000000..eaa88c9893 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/178_core.type_system.structured_dataset.to_numpy_1_task.yaml @@ -0,0 +1,69 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.type_system.structured_dataset + - task-name + - to_numpy + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.type_system.structured_dataset.to_numpy + resource_type: 1 + interface: + inputs: + variables: + ds: + description: ds + type: + Type: + StructuredDatasetType: + columns: + - literal_type: + Type: + Simple: 1 + name: Age + format: parquet + outputs: + variables: + o0: + description: o0 + type: + Type: + StructuredDatasetType: + columns: + - literal_type: + Type: + Simple: 1 + name: Age + format: parquet + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/179_core.type_system.structured_dataset.pandas_compatibility_wf_2_wf.json b/pkg/compiler/test/testdata/snacks-core/compiled/179_core.type_system.structured_dataset.pandas_compatibility_wf_2_wf.json new file mode 100755 index 0000000000..b8cbc8498b --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/179_core.type_system.structured_dataset.pandas_compatibility_wf_2_wf.json @@ -0,0 +1 @@ +{"primary":{"template":{"id":{"resourceType":"WORKFLOW","name":"core.type_system.structured_dataset.pandas_compatibility_wf"},"metadata":{},"interface":{"inputs":{"variables":{"a":{"type":{"simple":"INTEGER"},"description":"a"}}},"outputs":{"variables":{"o0":{"type":{"structuredDatasetType":{"columns":[{"name":"Age","literalType":{"simple":"INTEGER"}}],"format":"parquet"}},"description":"o0"}}}},"nodes":[{"id":"start-node"},{"id":"end-node","inputs":[{"var":"o0","binding":{"promise":{"nodeId":"n2","var":"o0"}}}]},{"id":"n0","metadata":{"name":"get_df","retries":{}},"inputs":[{"var":"a","binding":{"promise":{"nodeId":"start-node","var":"a"}}}],"taskNode":{"referenceId":{"resourceType":"TASK","name":"core.type_system.structured_dataset.get_df"},"overrides":{}}},{"id":"n1","metadata":{"name":"get_subset_df","retries":{}},"inputs":[{"var":"df","binding":{"promise":{"nodeId":"n0","var":"o0"}}}],"upstreamNodeIds":["n0"],"taskNode":{"referenceId":{"resourceType":"TASK","name":"core.type_system.structured_dataset.get_subset_df"},"overrides":{}}},{"id":"n2","metadata":{"name":"to_numpy","retries":{}},"inputs":[{"var":"ds","binding":{"promise":{"nodeId":"n1","var":"o0"}}}],"upstreamNodeIds":["n1"],"taskNode":{"referenceId":{"resourceType":"TASK","name":"core.type_system.structured_dataset.to_numpy"},"overrides":{}}}],"outputs":[{"var":"o0","binding":{"promise":{"nodeId":"n2","var":"o0"}}}],"metadataDefaults":{}},"connections":{"downstream":{"n0":{"ids":["n1"]},"n1":{"ids":["n2"]},"n2":{"ids":["end-node"]},"start-node":{"ids":["n0"]}},"upstream":{"end-node":{"ids":["n2"]},"n0":{"ids":["start-node"]},"n1":{"ids":["n0"]},"n2":{"ids":["n1"]}}}},"tasks":[{"template":{"id":{"resourceType":"TASK","name":"core.type_system.structured_dataset.get_df"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"a":{"type":{"simple":"INTEGER"},"description":"a"}}},"outputs":{"variables":{"o0":{"type":{"structuredDatasetType":{"columns":[{"name":"Name","literalType":{"simple":"STRING"}},{"name":"Age","literalType":{"simple":"INTEGER"}},{"name":"Height","literalType":{"simple":"INTEGER"}}],"format":"parquet"}},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.type_system.structured_dataset","task-name","get_df"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},{"template":{"id":{"resourceType":"TASK","name":"core.type_system.structured_dataset.get_subset_df"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"df":{"type":{"structuredDatasetType":{"columns":[{"name":"Age","literalType":{"simple":"INTEGER"}}],"format":"parquet"}},"description":"df"}}},"outputs":{"variables":{"o0":{"type":{"structuredDatasetType":{"columns":[{"name":"Age","literalType":{"simple":"INTEGER"}}],"format":"parquet"}},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.type_system.structured_dataset","task-name","get_subset_df"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},{"template":{"id":{"resourceType":"TASK","name":"core.type_system.structured_dataset.to_numpy"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"ds":{"type":{"structuredDatasetType":{"columns":[{"name":"Age","literalType":{"simple":"INTEGER"}}],"format":"parquet"}},"description":"ds"}}},"outputs":{"variables":{"o0":{"type":{"structuredDatasetType":{"columns":[{"name":"Age","literalType":{"simple":"INTEGER"}}],"format":"parquet"}},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.type_system.structured_dataset","task-name","to_numpy"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}}]} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/179_core.type_system.structured_dataset.pandas_compatibility_wf_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/179_core.type_system.structured_dataset.pandas_compatibility_wf_2_wf.yaml new file mode 100755 index 0000000000..3b370e22fc --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/179_core.type_system.structured_dataset.pandas_compatibility_wf_2_wf.yaml @@ -0,0 +1,345 @@ +primary: + connections: + downstream: + n0: + ids: + - n1 + n1: + ids: + - n2 + n2: + ids: + - end-node + start-node: + ids: + - n0 + upstream: + end-node: + ids: + - n2 + n0: + ids: + - start-node + n1: + ids: + - n0 + n2: + ids: + - n1 + template: + id: + name: core.type_system.structured_dataset.pandas_compatibility_wf + resource_type: 2 + interface: + inputs: + variables: + a: + description: a + type: + Type: + Simple: 1 + outputs: + variables: + o0: + description: o0 + type: + Type: + StructuredDatasetType: + columns: + - literal_type: + Type: + Simple: 1 + name: Age + format: parquet + metadata: {} + metadata_defaults: {} + nodes: + - Target: null + id: start-node + - Target: null + id: end-node + inputs: + - binding: + Value: + Promise: + node_id: n2 + var: o0 + var: o0 + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.type_system.structured_dataset.get_df + resource_type: 1 + overrides: {} + id: n0 + inputs: + - binding: + Value: + Promise: + node_id: start-node + var: a + var: a + metadata: + InterruptibleValue: null + name: get_df + retries: {} + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.type_system.structured_dataset.get_subset_df + resource_type: 1 + overrides: {} + id: n1 + inputs: + - binding: + Value: + Promise: + node_id: n0 + var: o0 + var: df + metadata: + InterruptibleValue: null + name: get_subset_df + retries: {} + upstream_node_ids: + - n0 + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.type_system.structured_dataset.to_numpy + resource_type: 1 + overrides: {} + id: n2 + inputs: + - binding: + Value: + Promise: + node_id: n1 + var: o0 + var: ds + metadata: + InterruptibleValue: null + name: to_numpy + retries: {} + upstream_node_ids: + - n1 + outputs: + - binding: + Value: + Promise: + node_id: n2 + var: o0 + var: o0 +tasks: +- template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.type_system.structured_dataset + - task-name + - get_df + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.type_system.structured_dataset.get_df + resource_type: 1 + interface: + inputs: + variables: + a: + description: a + type: + Type: + Simple: 1 + outputs: + variables: + o0: + description: o0 + type: + Type: + StructuredDatasetType: + columns: + - literal_type: + Type: + Simple: 3 + name: Name + - literal_type: + Type: + Simple: 1 + name: Age + - literal_type: + Type: + Simple: 1 + name: Height + format: parquet + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task +- template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.type_system.structured_dataset + - task-name + - get_subset_df + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.type_system.structured_dataset.get_subset_df + resource_type: 1 + interface: + inputs: + variables: + df: + description: df + type: + Type: + StructuredDatasetType: + columns: + - literal_type: + Type: + Simple: 1 + name: Age + format: parquet + outputs: + variables: + o0: + description: o0 + type: + Type: + StructuredDatasetType: + columns: + - literal_type: + Type: + Simple: 1 + name: Age + format: parquet + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task +- template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.type_system.structured_dataset + - task-name + - to_numpy + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.type_system.structured_dataset.to_numpy + resource_type: 1 + interface: + inputs: + variables: + ds: + description: ds + type: + Type: + StructuredDatasetType: + columns: + - literal_type: + Type: + Simple: 1 + name: Age + format: parquet + outputs: + variables: + o0: + description: o0 + type: + Type: + StructuredDatasetType: + columns: + - literal_type: + Type: + Simple: 1 + name: Age + format: parquet + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/181_core.type_system.structured_dataset.schema_compatibility_wf_2_wf.json b/pkg/compiler/test/testdata/snacks-core/compiled/181_core.type_system.structured_dataset.schema_compatibility_wf_2_wf.json new file mode 100755 index 0000000000..5801331194 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/181_core.type_system.structured_dataset.schema_compatibility_wf_2_wf.json @@ -0,0 +1 @@ +{"primary":{"template":{"id":{"resourceType":"WORKFLOW","name":"core.type_system.structured_dataset.schema_compatibility_wf"},"metadata":{},"interface":{"inputs":{"variables":{"a":{"type":{"simple":"INTEGER"},"description":"a"}}},"outputs":{"variables":{"o0":{"type":{"structuredDatasetType":{"columns":[{"name":"Age","literalType":{"simple":"INTEGER"}}],"format":"parquet"}},"description":"o0"}}}},"nodes":[{"id":"start-node"},{"id":"end-node","inputs":[{"var":"o0","binding":{"promise":{"nodeId":"n2","var":"o0"}}}]},{"id":"n0","metadata":{"name":"get_schema_df","retries":{}},"inputs":[{"var":"a","binding":{"promise":{"nodeId":"start-node","var":"a"}}}],"taskNode":{"referenceId":{"resourceType":"TASK","name":"core.type_system.structured_dataset.get_schema_df"},"overrides":{}}},{"id":"n1","metadata":{"name":"get_subset_df","retries":{}},"inputs":[{"var":"df","binding":{"promise":{"nodeId":"n0","var":"o0"}}}],"upstreamNodeIds":["n0"],"taskNode":{"referenceId":{"resourceType":"TASK","name":"core.type_system.structured_dataset.get_subset_df"},"overrides":{}}},{"id":"n2","metadata":{"name":"to_numpy","retries":{}},"inputs":[{"var":"ds","binding":{"promise":{"nodeId":"n1","var":"o0"}}}],"upstreamNodeIds":["n1"],"taskNode":{"referenceId":{"resourceType":"TASK","name":"core.type_system.structured_dataset.to_numpy"},"overrides":{}}}],"outputs":[{"var":"o0","binding":{"promise":{"nodeId":"n2","var":"o0"}}}],"metadataDefaults":{}},"connections":{"downstream":{"n0":{"ids":["n1"]},"n1":{"ids":["n2"]},"n2":{"ids":["end-node"]},"start-node":{"ids":["n0"]}},"upstream":{"end-node":{"ids":["n2"]},"n0":{"ids":["start-node"]},"n1":{"ids":["n0"]},"n2":{"ids":["n1"]}}}},"tasks":[{"template":{"id":{"resourceType":"TASK","name":"core.type_system.structured_dataset.get_schema_df"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"a":{"type":{"simple":"INTEGER"},"description":"a"}}},"outputs":{"variables":{"o0":{"type":{"schema":{"columns":[{"name":"Name","type":"STRING"},{"name":"Age"},{"name":"Height"}]}},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.type_system.structured_dataset","task-name","get_schema_df"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},{"template":{"id":{"resourceType":"TASK","name":"core.type_system.structured_dataset.get_subset_df"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"df":{"type":{"structuredDatasetType":{"columns":[{"name":"Age","literalType":{"simple":"INTEGER"}}],"format":"parquet"}},"description":"df"}}},"outputs":{"variables":{"o0":{"type":{"structuredDatasetType":{"columns":[{"name":"Age","literalType":{"simple":"INTEGER"}}],"format":"parquet"}},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.type_system.structured_dataset","task-name","get_subset_df"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},{"template":{"id":{"resourceType":"TASK","name":"core.type_system.structured_dataset.to_numpy"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"ds":{"type":{"structuredDatasetType":{"columns":[{"name":"Age","literalType":{"simple":"INTEGER"}}],"format":"parquet"}},"description":"ds"}}},"outputs":{"variables":{"o0":{"type":{"structuredDatasetType":{"columns":[{"name":"Age","literalType":{"simple":"INTEGER"}}],"format":"parquet"}},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.type_system.structured_dataset","task-name","to_numpy"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}}]} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/181_core.type_system.structured_dataset.schema_compatibility_wf_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/181_core.type_system.structured_dataset.schema_compatibility_wf_2_wf.yaml new file mode 100755 index 0000000000..b0acffc8bf --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/181_core.type_system.structured_dataset.schema_compatibility_wf_2_wf.yaml @@ -0,0 +1,336 @@ +primary: + connections: + downstream: + n0: + ids: + - n1 + n1: + ids: + - n2 + n2: + ids: + - end-node + start-node: + ids: + - n0 + upstream: + end-node: + ids: + - n2 + n0: + ids: + - start-node + n1: + ids: + - n0 + n2: + ids: + - n1 + template: + id: + name: core.type_system.structured_dataset.schema_compatibility_wf + resource_type: 2 + interface: + inputs: + variables: + a: + description: a + type: + Type: + Simple: 1 + outputs: + variables: + o0: + description: o0 + type: + Type: + StructuredDatasetType: + columns: + - literal_type: + Type: + Simple: 1 + name: Age + format: parquet + metadata: {} + metadata_defaults: {} + nodes: + - Target: null + id: start-node + - Target: null + id: end-node + inputs: + - binding: + Value: + Promise: + node_id: n2 + var: o0 + var: o0 + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.type_system.structured_dataset.get_schema_df + resource_type: 1 + overrides: {} + id: n0 + inputs: + - binding: + Value: + Promise: + node_id: start-node + var: a + var: a + metadata: + InterruptibleValue: null + name: get_schema_df + retries: {} + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.type_system.structured_dataset.get_subset_df + resource_type: 1 + overrides: {} + id: n1 + inputs: + - binding: + Value: + Promise: + node_id: n0 + var: o0 + var: df + metadata: + InterruptibleValue: null + name: get_subset_df + retries: {} + upstream_node_ids: + - n0 + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.type_system.structured_dataset.to_numpy + resource_type: 1 + overrides: {} + id: n2 + inputs: + - binding: + Value: + Promise: + node_id: n1 + var: o0 + var: ds + metadata: + InterruptibleValue: null + name: to_numpy + retries: {} + upstream_node_ids: + - n1 + outputs: + - binding: + Value: + Promise: + node_id: n2 + var: o0 + var: o0 +tasks: +- template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.type_system.structured_dataset + - task-name + - get_schema_df + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.type_system.structured_dataset.get_schema_df + resource_type: 1 + interface: + inputs: + variables: + a: + description: a + type: + Type: + Simple: 1 + outputs: + variables: + o0: + description: o0 + type: + Type: + Schema: + columns: + - name: Name + type: 2 + - name: Age + - name: Height + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task +- template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.type_system.structured_dataset + - task-name + - get_subset_df + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.type_system.structured_dataset.get_subset_df + resource_type: 1 + interface: + inputs: + variables: + df: + description: df + type: + Type: + StructuredDatasetType: + columns: + - literal_type: + Type: + Simple: 1 + name: Age + format: parquet + outputs: + variables: + o0: + description: o0 + type: + Type: + StructuredDatasetType: + columns: + - literal_type: + Type: + Simple: 1 + name: Age + format: parquet + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task +- template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.type_system.structured_dataset + - task-name + - to_numpy + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.type_system.structured_dataset.to_numpy + resource_type: 1 + interface: + inputs: + variables: + ds: + description: ds + type: + Type: + StructuredDatasetType: + columns: + - literal_type: + Type: + Simple: 1 + name: Age + format: parquet + outputs: + variables: + o0: + description: o0 + type: + Type: + StructuredDatasetType: + columns: + - literal_type: + Type: + Simple: 1 + name: Age + format: parquet + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/183_core.type_system.typed_schema.t1_1_task.json b/pkg/compiler/test/testdata/snacks-core/compiled/183_core.type_system.typed_schema.t1_1_task.json new file mode 100755 index 0000000000..2e42e5d760 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/183_core.type_system.typed_schema.t1_1_task.json @@ -0,0 +1 @@ +{"template":{"id":{"resourceType":"TASK","name":"core.type_system.typed_schema.t1"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{},"outputs":{"variables":{"o0":{"type":{"schema":{"columns":[{"name":"x"},{"name":"y","type":"STRING"}]}},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.type_system.typed_schema","task-name","t1"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/183_core.type_system.typed_schema.t1_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/183_core.type_system.typed_schema.t1_1_task.yaml new file mode 100755 index 0000000000..ef63cec2c5 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/183_core.type_system.typed_schema.t1_1_task.yaml @@ -0,0 +1,55 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.type_system.typed_schema + - task-name + - t1 + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.type_system.typed_schema.t1 + resource_type: 1 + interface: + inputs: {} + outputs: + variables: + o0: + description: o0 + type: + Type: + Schema: + columns: + - name: x + - name: "y" + type: 2 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/184_core.type_system.typed_schema.t2_1_task.json b/pkg/compiler/test/testdata/snacks-core/compiled/184_core.type_system.typed_schema.t2_1_task.json new file mode 100755 index 0000000000..7e1246f7c3 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/184_core.type_system.typed_schema.t2_1_task.json @@ -0,0 +1 @@ +{"template":{"id":{"resourceType":"TASK","name":"core.type_system.typed_schema.t2"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"schema":{"type":{"schema":{"columns":[{"name":"x"},{"name":"y","type":"STRING"}]}},"description":"schema"}}},"outputs":{"variables":{"o0":{"type":{"schema":{"columns":[{"name":"x"}]}},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.type_system.typed_schema","task-name","t2"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/184_core.type_system.typed_schema.t2_1_task.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/184_core.type_system.typed_schema.t2_1_task.yaml new file mode 100755 index 0000000000..8fb24a5051 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/184_core.type_system.typed_schema.t2_1_task.yaml @@ -0,0 +1,63 @@ +template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.type_system.typed_schema + - task-name + - t2 + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.type_system.typed_schema.t2 + resource_type: 1 + interface: + inputs: + variables: + schema: + description: schema + type: + Type: + Schema: + columns: + - name: x + - name: "y" + type: 2 + outputs: + variables: + o0: + description: o0 + type: + Type: + Schema: + columns: + - name: x + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/185_core.type_system.typed_schema.wf_2_wf.json b/pkg/compiler/test/testdata/snacks-core/compiled/185_core.type_system.typed_schema.wf_2_wf.json new file mode 100755 index 0000000000..b09b037b62 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/185_core.type_system.typed_schema.wf_2_wf.json @@ -0,0 +1 @@ +{"primary":{"template":{"id":{"resourceType":"WORKFLOW","name":"core.type_system.typed_schema.wf"},"metadata":{},"interface":{"inputs":{},"outputs":{"variables":{"o0":{"type":{"schema":{"columns":[{"name":"x"}]}},"description":"o0"}}}},"nodes":[{"id":"start-node"},{"id":"end-node","inputs":[{"var":"o0","binding":{"promise":{"nodeId":"n1","var":"o0"}}}]},{"id":"n0","metadata":{"name":"t1","retries":{}},"taskNode":{"referenceId":{"resourceType":"TASK","name":"core.type_system.typed_schema.t1"},"overrides":{}}},{"id":"n1","metadata":{"name":"t2","retries":{}},"inputs":[{"var":"schema","binding":{"promise":{"nodeId":"n0","var":"o0"}}}],"upstreamNodeIds":["n0"],"taskNode":{"referenceId":{"resourceType":"TASK","name":"core.type_system.typed_schema.t2"},"overrides":{}}}],"outputs":[{"var":"o0","binding":{"promise":{"nodeId":"n1","var":"o0"}}}],"metadataDefaults":{}},"connections":{"downstream":{"n0":{"ids":["n1"]},"n1":{"ids":["end-node"]},"start-node":{"ids":["n0"]}},"upstream":{"end-node":{"ids":["n1"]},"n0":{"ids":["start-node"]},"n1":{"ids":["n0"]}}}},"tasks":[{"template":{"id":{"resourceType":"TASK","name":"core.type_system.typed_schema.t1"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{},"outputs":{"variables":{"o0":{"type":{"schema":{"columns":[{"name":"x"},{"name":"y","type":"STRING"}]}},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.type_system.typed_schema","task-name","t1"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},{"template":{"id":{"resourceType":"TASK","name":"core.type_system.typed_schema.t2"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.32.6","flavor":"python"},"retries":{}},"interface":{"inputs":{"variables":{"schema":{"type":{"schema":{"columns":[{"name":"x"},{"name":"y","type":"STRING"}]}},"description":"schema"}}},"outputs":{"variables":{"o0":{"type":{"schema":{"columns":[{"name":"x"}]}},"description":"o0"}}}},"container":{"image":"ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077","args":["pyflyte-execute","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}","--checkpoint-path","{{.checkpointOutputPrefix}}","--prev-checkpoint","{{.prevCheckpointPrefix}}","--resolver","flytekit.core.python_auto_container.default_task_resolver","--","task-module","core.type_system.typed_schema","task-name","t2"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}}]} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/snacks-core/compiled/185_core.type_system.typed_schema.wf_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/compiled/185_core.type_system.typed_schema.wf_2_wf.yaml new file mode 100755 index 0000000000..9a9218d6f3 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/compiled/185_core.type_system.typed_schema.wf_2_wf.yaml @@ -0,0 +1,210 @@ +primary: + connections: + downstream: + n0: + ids: + - n1 + n1: + ids: + - end-node + start-node: + ids: + - n0 + upstream: + end-node: + ids: + - n1 + n0: + ids: + - start-node + n1: + ids: + - n0 + template: + id: + name: core.type_system.typed_schema.wf + resource_type: 2 + interface: + inputs: {} + outputs: + variables: + o0: + description: o0 + type: + Type: + Schema: + columns: + - name: x + metadata: {} + metadata_defaults: {} + nodes: + - Target: null + id: start-node + - Target: null + id: end-node + inputs: + - binding: + Value: + Promise: + node_id: n1 + var: o0 + var: o0 + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.type_system.typed_schema.t1 + resource_type: 1 + overrides: {} + id: n0 + metadata: + InterruptibleValue: null + name: t1 + retries: {} + - Target: + TaskNode: + Reference: + ReferenceId: + name: core.type_system.typed_schema.t2 + resource_type: 1 + overrides: {} + id: n1 + inputs: + - binding: + Value: + Promise: + node_id: n0 + var: o0 + var: schema + metadata: + InterruptibleValue: null + name: t2 + retries: {} + upstream_node_ids: + - n0 + outputs: + - binding: + Value: + Promise: + node_id: n1 + var: o0 + var: o0 +tasks: +- template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.type_system.typed_schema + - task-name + - t1 + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.type_system.typed_schema.t1 + resource_type: 1 + interface: + inputs: {} + outputs: + variables: + o0: + description: o0 + type: + Type: + Schema: + columns: + - name: x + - name: "y" + type: 2 + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task +- template: + Target: + Container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.type_system.typed_schema + - task-name + - t2 + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.type_system.typed_schema.t2 + resource_type: 1 + interface: + inputs: + variables: + schema: + description: schema + type: + Type: + Schema: + columns: + - name: x + - name: "y" + type: 2 + outputs: + variables: + o0: + description: o0 + type: + Type: + Schema: + columns: + - name: x + metadata: + InterruptibleValue: null + retries: {} + runtime: + flavor: python + type: 1 + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/k8s/002_core.containerization.multi_images.my_workflow_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/k8s/002_core.containerization.multi_images.my_workflow_2_wf.yaml new file mode 100755 index 0000000000..018b5eb189 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/k8s/002_core.containerization.multi_images.my_workflow_2_wf.yaml @@ -0,0 +1,249 @@ +apiVersion: flyte.lyft.com/v1alpha1 +executionConfig: + Interruptible: null + MaxParallelism: 0 + RecoveryExecution: {} + TaskPluginImpls: null + TaskResources: + Limits: + CPU: "0" + EphemeralStorage: "0" + GPU: "0" + Memory: "0" + Storage: "0" + Requests: + CPU: "0" + EphemeralStorage: "0" + GPU: "0" + Memory: "0" + Storage: "0" +executionId: {} +inputs: {} +kind: flyteworkflow +metadata: + creationTimestamp: null + labels: + domain: domain + execution-id: name + project: hello + shard-key: "6" + workflow-name: core-containerization-multi-images-my-workflow + name: name + namespace: namespace +node-defaults: {} +rawOutputDataConfig: {} +securityContext: {} +spec: + connections: + n0: + - n1 + n1: + - end-node + start-node: + - n0 + edges: + downstream: + n0: + - n1 + n1: + - end-node + start-node: + - n0 + upstream: + end-node: + - n1 + n0: + - start-node + n1: + - n0 + id: ::core.containerization.multi_images.my_workflow + nodes: + end-node: + id: end-node + inputBindings: + - binding: + promise: + nodeId: n1 + var: o0 + var: o0 + kind: end + resources: {} + n0: + id: n0 + kind: task + name: svm_trainer + resources: {} + retry: + minAttempts: 1 + task: 'resource_type:TASK name:"core.containerization.multi_images.svm_trainer" ' + n1: + id: n1 + inputBindings: + - binding: + promise: + nodeId: n0 + var: test_features + var: X_test + - binding: + promise: + nodeId: n0 + var: train_features + var: X_train + - binding: + promise: + nodeId: n0 + var: test_labels + var: y_test + - binding: + promise: + nodeId: n0 + var: train_labels + var: y_train + kind: task + name: svm_predictor + resources: {} + retry: + minAttempts: 1 + task: 'resource_type:TASK name:"core.containerization.multi_images.svm_predictor" ' + start-node: + id: start-node + kind: start + resources: {} + outputBindings: + - binding: + promise: + nodeId: n1 + var: o0 + var: o0 + outputs: + variables: + o0: + type: + simple: FLOAT +status: + phase: 0 +tasks: + 'resource_type:TASK name:"core.containerization.multi_images.svm_predictor" ': + container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.containerization.multi_images + - task-name + - svm_predictor + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:multi-image-predict-98b125fd57d20594026941c2ebe7ef662e5acb7d6423660a65f493ca2d9aa267 + resources: {} + id: + name: core.containerization.multi_images.svm_predictor + resourceType: TASK + interface: + inputs: + variables: + X_test: + type: + structuredDatasetType: + format: parquet + X_train: + type: + structuredDatasetType: + format: parquet + y_test: + type: + structuredDatasetType: + format: parquet + y_train: + type: + structuredDatasetType: + format: parquet + outputs: + variables: + o0: + type: + simple: FLOAT + metadata: + retries: {} + runtime: + flavor: python + type: FLYTE_SDK + version: 0.32.6 + type: python-task + 'resource_type:TASK name:"core.containerization.multi_images.svm_trainer" ': + container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.containerization.multi_images + - task-name + - svm_trainer + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-with-sklearn-baa17ccf39aa667c5950bd713a4366ce7d5fccaf7f85e6be8c07fe4b522f92c3 + resources: {} + id: + name: core.containerization.multi_images.svm_trainer + resourceType: TASK + interface: + inputs: {} + outputs: + variables: + test_features: + type: + structuredDatasetType: + format: parquet + test_labels: + type: + structuredDatasetType: + format: parquet + train_features: + type: + structuredDatasetType: + format: parquet + train_labels: + type: + structuredDatasetType: + format: parquet + metadata: + retries: {} + runtime: + flavor: python + type: FLYTE_SDK + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/k8s/010_core.containerization.raw_container.wf_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/k8s/010_core.containerization.raw_container.wf_2_wf.yaml new file mode 100755 index 0000000000..53d3aa7d55 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/k8s/010_core.containerization.raw_container.wf_2_wf.yaml @@ -0,0 +1,579 @@ +apiVersion: flyte.lyft.com/v1alpha1 +executionConfig: + Interruptible: null + MaxParallelism: 0 + RecoveryExecution: {} + TaskPluginImpls: null + TaskResources: + Limits: + CPU: "0" + EphemeralStorage: "0" + GPU: "0" + Memory: "0" + Storage: "0" + Requests: + CPU: "0" + EphemeralStorage: "0" + GPU: "0" + Memory: "0" + Storage: "0" +executionId: {} +inputs: + literals: + a: + scalar: + primitive: + floatValue: 0 + b: + scalar: + primitive: + floatValue: 0 +kind: flyteworkflow +metadata: + creationTimestamp: null + labels: + domain: domain + execution-id: name + project: hello + shard-key: "6" + workflow-name: core-containerization-raw-container-wf + name: name + namespace: namespace +node-defaults: {} +rawOutputDataConfig: {} +securityContext: {} +spec: + connections: + n0: + - n5 + n1: + - n5 + n2: + - n5 + n3: + - n5 + n4: + - n5 + n5: + - end-node + start-node: + - n0 + - n1 + - n2 + - n3 + - n4 + edges: + downstream: + n0: + - n5 + n1: + - n5 + n2: + - n5 + n3: + - n5 + n4: + - n5 + n5: + - end-node + start-node: + - n0 + - n1 + - n2 + - n3 + - n4 + upstream: + end-node: + - n5 + n0: + - start-node + n1: + - start-node + n2: + - start-node + n3: + - start-node + n4: + - start-node + n5: + - n0 + - n1 + - n2 + - n3 + - n4 + id: ::core.containerization.raw_container.wf + nodes: + end-node: + id: end-node + kind: end + resources: {} + n0: + id: n0 + inputBindings: + - binding: + promise: + nodeId: start-node + var: a + var: a + - binding: + promise: + nodeId: start-node + var: b + var: b + kind: task + name: ellipse-area-metadata-shell + resources: {} + retry: + minAttempts: 1 + task: 'resource_type:TASK name:"ellipse-area-metadata-shell" ' + n1: + id: n1 + inputBindings: + - binding: + promise: + nodeId: start-node + var: a + var: a + - binding: + promise: + nodeId: start-node + var: b + var: b + kind: task + name: ellipse-area-metadata-python + resources: {} + retry: + minAttempts: 1 + task: 'resource_type:TASK name:"ellipse-area-metadata-python" ' + n2: + id: n2 + inputBindings: + - binding: + promise: + nodeId: start-node + var: a + var: a + - binding: + promise: + nodeId: start-node + var: b + var: b + kind: task + name: ellipse-area-metadata-r + resources: {} + retry: + minAttempts: 1 + task: 'resource_type:TASK name:"ellipse-area-metadata-r" ' + n3: + id: n3 + inputBindings: + - binding: + promise: + nodeId: start-node + var: a + var: a + - binding: + promise: + nodeId: start-node + var: b + var: b + kind: task + name: ellipse-area-metadata-haskell + resources: {} + retry: + minAttempts: 1 + task: 'resource_type:TASK name:"ellipse-area-metadata-haskell" ' + n4: + id: n4 + inputBindings: + - binding: + promise: + nodeId: start-node + var: a + var: a + - binding: + promise: + nodeId: start-node + var: b + var: b + kind: task + name: ellipse-area-metadata-julia + resources: {} + retry: + minAttempts: 1 + task: 'resource_type:TASK name:"ellipse-area-metadata-julia" ' + n5: + id: n5 + inputBindings: + - binding: + promise: + nodeId: n3 + var: area + var: area_haskell + - binding: + promise: + nodeId: n4 + var: area + var: area_julia + - binding: + promise: + nodeId: n1 + var: area + var: area_python + - binding: + promise: + nodeId: n2 + var: area + var: area_r + - binding: + promise: + nodeId: n0 + var: area + var: area_shell + - binding: + promise: + nodeId: n3 + var: metadata + var: metadata_haskell + - binding: + promise: + nodeId: n4 + var: metadata + var: metadata_julia + - binding: + promise: + nodeId: n1 + var: metadata + var: metadata_python + - binding: + promise: + nodeId: n2 + var: metadata + var: metadata_r + - binding: + promise: + nodeId: n0 + var: metadata + var: metadata_shell + kind: task + name: report_all_calculated_areas + resources: {} + retry: + minAttempts: 1 + task: 'resource_type:TASK name:"core.containerization.raw_container.report_all_calculated_areas" ' + start-node: + id: start-node + kind: start + resources: {} + outputs: {} +status: + phase: 0 +tasks: + 'resource_type:TASK name:"core.containerization.raw_container.report_all_calculated_areas" ': + container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.containerization.raw_container + - task-name + - report_all_calculated_areas + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.containerization.raw_container.report_all_calculated_areas + resourceType: TASK + interface: + inputs: + variables: + area_haskell: + type: + simple: FLOAT + area_julia: + type: + simple: FLOAT + area_python: + type: + simple: FLOAT + area_r: + type: + simple: FLOAT + area_shell: + type: + simple: FLOAT + metadata_haskell: + type: + simple: STRING + metadata_julia: + type: + simple: STRING + metadata_python: + type: + simple: STRING + metadata_r: + type: + simple: STRING + metadata_shell: + type: + simple: STRING + outputs: {} + metadata: + retries: {} + runtime: + flavor: python + type: FLYTE_SDK + version: 0.32.6 + type: python-task + 'resource_type:TASK name:"ellipse-area-metadata-haskell" ': + container: + command: + - ./calculate-ellipse-area + - /var/inputs + - /var/outputs + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + dataConfig: + enabled: true + inputPath: /var/inputs + outputPath: /var/outputs + image: ghcr.io/flyteorg/rawcontainers-haskell:v1 + resources: {} + id: + name: ellipse-area-metadata-haskell + resourceType: TASK + interface: + inputs: + variables: + a: + type: + simple: FLOAT + b: + type: + simple: FLOAT + outputs: + variables: + area: + type: + simple: FLOAT + metadata: + type: + simple: STRING + metadata: + retries: {} + runtime: + flavor: python + type: FLYTE_SDK + version: 0.32.6 + type: raw-container + 'resource_type:TASK name:"ellipse-area-metadata-julia" ': + container: + command: + - julia + - calculate-ellipse-area.jl + - /var/inputs + - /var/outputs + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + dataConfig: + enabled: true + inputPath: /var/inputs + outputPath: /var/outputs + image: ghcr.io/flyteorg/rawcontainers-julia:v1 + resources: {} + id: + name: ellipse-area-metadata-julia + resourceType: TASK + interface: + inputs: + variables: + a: + type: + simple: FLOAT + b: + type: + simple: FLOAT + outputs: + variables: + area: + type: + simple: FLOAT + metadata: + type: + simple: STRING + metadata: + retries: {} + runtime: + flavor: python + type: FLYTE_SDK + version: 0.32.6 + type: raw-container + 'resource_type:TASK name:"ellipse-area-metadata-python" ': + container: + command: + - python + - calculate-ellipse-area.py + - /var/inputs + - /var/outputs + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + dataConfig: + enabled: true + inputPath: /var/inputs + outputPath: /var/outputs + image: ghcr.io/flyteorg/rawcontainers-python:v1 + resources: {} + id: + name: ellipse-area-metadata-python + resourceType: TASK + interface: + inputs: + variables: + a: + type: + simple: FLOAT + b: + type: + simple: FLOAT + outputs: + variables: + area: + type: + simple: FLOAT + metadata: + type: + simple: STRING + metadata: + retries: {} + runtime: + flavor: python + type: FLYTE_SDK + version: 0.32.6 + type: raw-container + 'resource_type:TASK name:"ellipse-area-metadata-r" ': + container: + command: + - Rscript + - --vanilla + - calculate-ellipse-area.R + - /var/inputs + - /var/outputs + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + dataConfig: + enabled: true + inputPath: /var/inputs + outputPath: /var/outputs + image: ghcr.io/flyteorg/rawcontainers-r:v1 + resources: {} + id: + name: ellipse-area-metadata-r + resourceType: TASK + interface: + inputs: + variables: + a: + type: + simple: FLOAT + b: + type: + simple: FLOAT + outputs: + variables: + area: + type: + simple: FLOAT + metadata: + type: + simple: STRING + metadata: + retries: {} + runtime: + flavor: python + type: FLYTE_SDK + version: 0.32.6 + type: raw-container + 'resource_type:TASK name:"ellipse-area-metadata-shell" ': + container: + command: + - ./calculate-ellipse-area.sh + - /var/inputs + - /var/outputs + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + dataConfig: + enabled: true + inputPath: /var/inputs + outputPath: /var/outputs + image: ghcr.io/flyteorg/rawcontainers-shell:v1 + resources: {} + id: + name: ellipse-area-metadata-shell + resourceType: TASK + interface: + inputs: + variables: + a: + type: + simple: FLOAT + b: + type: + simple: FLOAT + outputs: + variables: + area: + type: + simple: FLOAT + metadata: + type: + simple: STRING + metadata: + retries: {} + runtime: + flavor: python + type: FLYTE_SDK + version: 0.32.6 + type: raw-container diff --git a/pkg/compiler/test/testdata/snacks-core/k8s/015_core.containerization.use_secrets.my_secret_workflow_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/k8s/015_core.containerization.use_secrets.my_secret_workflow_2_wf.yaml new file mode 100755 index 0000000000..29bc1b6b93 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/k8s/015_core.containerization.use_secrets.my_secret_workflow_2_wf.yaml @@ -0,0 +1,338 @@ +apiVersion: flyte.lyft.com/v1alpha1 +executionConfig: + Interruptible: null + MaxParallelism: 0 + RecoveryExecution: {} + TaskPluginImpls: null + TaskResources: + Limits: + CPU: "0" + EphemeralStorage: "0" + GPU: "0" + Memory: "0" + Storage: "0" + Requests: + CPU: "0" + EphemeralStorage: "0" + GPU: "0" + Memory: "0" + Storage: "0" +executionId: {} +inputs: {} +kind: flyteworkflow +metadata: + creationTimestamp: null + labels: + domain: domain + execution-id: name + project: hello + shard-key: "6" + workflow-name: core-containerization-use-secrets-my-secret-workflow + name: name + namespace: namespace +node-defaults: {} +rawOutputDataConfig: {} +securityContext: {} +spec: + connections: + n0: + - end-node + n1: + - end-node + n2: + - end-node + start-node: + - n0 + - n1 + - n2 + edges: + downstream: + n0: + - end-node + n1: + - end-node + n2: + - end-node + start-node: + - n0 + - n1 + - n2 + upstream: + end-node: + - n0 + - n1 + - n2 + n0: + - start-node + n1: + - start-node + n2: + - start-node + id: ::core.containerization.use_secrets.my_secret_workflow + nodes: + end-node: + id: end-node + inputBindings: + - binding: + promise: + nodeId: n0 + var: o0 + var: o0 + - binding: + promise: + nodeId: n1 + var: o0 + var: o1 + - binding: + promise: + nodeId: n1 + var: o1 + var: o2 + - binding: + promise: + nodeId: n2 + var: o0 + var: o3 + - binding: + promise: + nodeId: n2 + var: o1 + var: o4 + kind: end + resources: {} + n0: + id: n0 + kind: task + name: secret_task + resources: {} + retry: + minAttempts: 1 + task: 'resource_type:TASK name:"core.containerization.use_secrets.secret_task" ' + n1: + id: n1 + kind: task + name: user_info_task + resources: {} + retry: + minAttempts: 1 + task: 'resource_type:TASK name:"core.containerization.use_secrets.user_info_task" ' + n2: + id: n2 + kind: task + name: secret_file_task + resources: {} + retry: + minAttempts: 1 + task: 'resource_type:TASK name:"core.containerization.use_secrets.secret_file_task" ' + start-node: + id: start-node + kind: start + resources: {} + outputBindings: + - binding: + promise: + nodeId: n0 + var: o0 + var: o0 + - binding: + promise: + nodeId: n1 + var: o0 + var: o1 + - binding: + promise: + nodeId: n1 + var: o1 + var: o2 + - binding: + promise: + nodeId: n2 + var: o0 + var: o3 + - binding: + promise: + nodeId: n2 + var: o1 + var: o4 + outputs: + variables: + o0: + type: + simple: STRING + o1: + type: + simple: STRING + o2: + type: + simple: STRING + o3: + type: + simple: STRING + o4: + type: + simple: STRING +status: + phase: 0 +tasks: + 'resource_type:TASK name:"core.containerization.use_secrets.secret_file_task" ': + container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.containerization.use_secrets + - task-name + - secret_file_task + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.containerization.use_secrets.secret_file_task + resourceType: TASK + interface: + inputs: {} + outputs: + variables: + o0: + type: + simple: STRING + o1: + type: + simple: STRING + metadata: + retries: {} + runtime: + flavor: python + type: FLYTE_SDK + version: 0.32.6 + securityContext: + secrets: + - group: user-info + key: user_secret + mountRequirement: ENV_VAR + type: python-task + 'resource_type:TASK name:"core.containerization.use_secrets.secret_task" ': + container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.containerization.use_secrets + - task-name + - secret_task + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.containerization.use_secrets.secret_task + resourceType: TASK + interface: + inputs: {} + outputs: + variables: + o0: + type: + simple: STRING + metadata: + retries: {} + runtime: + flavor: python + type: FLYTE_SDK + version: 0.32.6 + securityContext: + secrets: + - group: user-info + key: user_secret + type: python-task + 'resource_type:TASK name:"core.containerization.use_secrets.user_info_task" ': + container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.containerization.use_secrets + - task-name + - user_info_task + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.containerization.use_secrets.user_info_task + resourceType: TASK + interface: + inputs: {} + outputs: + variables: + o0: + type: + simple: STRING + o1: + type: + simple: STRING + metadata: + retries: {} + runtime: + flavor: python + type: FLYTE_SDK + version: 0.32.6 + securityContext: + secrets: + - group: user-info + key: username + - group: user-info + key: password + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/k8s/019_core.control_flow.chain_tasks.chain_tasks_wf_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/k8s/019_core.control_flow.chain_tasks.chain_tasks_wf_2_wf.yaml new file mode 100755 index 0000000000..c30ffc2724 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/k8s/019_core.control_flow.chain_tasks.chain_tasks_wf_2_wf.yaml @@ -0,0 +1,196 @@ +apiVersion: flyte.lyft.com/v1alpha1 +executionConfig: + Interruptible: null + MaxParallelism: 0 + RecoveryExecution: {} + TaskPluginImpls: null + TaskResources: + Limits: + CPU: "0" + EphemeralStorage: "0" + GPU: "0" + Memory: "0" + Storage: "0" + Requests: + CPU: "0" + EphemeralStorage: "0" + GPU: "0" + Memory: "0" + Storage: "0" +executionId: {} +inputs: {} +kind: flyteworkflow +metadata: + creationTimestamp: null + labels: + domain: domain + execution-id: name + project: hello + shard-key: "6" + workflow-name: core-control-flow-chain-tasks-chain-tasks-wf + name: name + namespace: namespace +node-defaults: {} +rawOutputDataConfig: {} +securityContext: {} +spec: + connections: + n0: + - n1 + n1: + - end-node + start-node: + - n0 + edges: + downstream: + n0: + - n1 + n1: + - end-node + start-node: + - n0 + upstream: + end-node: + - n1 + n0: + - start-node + n1: + - n0 + id: ::core.control_flow.chain_tasks.chain_tasks_wf + nodes: + end-node: + id: end-node + inputBindings: + - binding: + promise: + nodeId: n1 + var: o0 + var: o0 + kind: end + resources: {} + n0: + id: n0 + kind: task + name: write + resources: {} + retry: + minAttempts: 1 + task: 'resource_type:TASK name:"core.control_flow.chain_tasks.write" ' + n1: + id: n1 + kind: task + name: read + resources: {} + retry: + minAttempts: 1 + task: 'resource_type:TASK name:"core.control_flow.chain_tasks.read" ' + start-node: + id: start-node + kind: start + resources: {} + outputBindings: + - binding: + promise: + nodeId: n1 + var: o0 + var: o0 + outputs: + variables: + o0: + type: + structuredDatasetType: + format: parquet +status: + phase: 0 +tasks: + 'resource_type:TASK name:"core.control_flow.chain_tasks.read" ': + container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.control_flow.chain_tasks + - task-name + - read + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.control_flow.chain_tasks.read + resourceType: TASK + interface: + inputs: {} + outputs: + variables: + o0: + type: + structuredDatasetType: + format: parquet + metadata: + retries: {} + runtime: + flavor: python + type: FLYTE_SDK + version: 0.32.6 + type: python-task + 'resource_type:TASK name:"core.control_flow.chain_tasks.write" ': + container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.control_flow.chain_tasks + - task-name + - write + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.control_flow.chain_tasks.write + resourceType: TASK + interface: + inputs: {} + outputs: {} + metadata: + retries: {} + runtime: + flavor: python + type: FLYTE_SDK + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/k8s/022_core.control_flow.checkpoint.example_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/k8s/022_core.control_flow.checkpoint.example_2_wf.yaml new file mode 100755 index 0000000000..f0b17af3b8 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/k8s/022_core.control_flow.checkpoint.example_2_wf.yaml @@ -0,0 +1,153 @@ +apiVersion: flyte.lyft.com/v1alpha1 +executionConfig: + Interruptible: null + MaxParallelism: 0 + RecoveryExecution: {} + TaskPluginImpls: null + TaskResources: + Limits: + CPU: "0" + EphemeralStorage: "0" + GPU: "0" + Memory: "0" + Storage: "0" + Requests: + CPU: "0" + EphemeralStorage: "0" + GPU: "0" + Memory: "0" + Storage: "0" +executionId: {} +inputs: + literals: + n_iterations: + scalar: + primitive: + integer: "0" +kind: flyteworkflow +metadata: + creationTimestamp: null + labels: + domain: domain + execution-id: name + project: hello + shard-key: "6" + workflow-name: core-control-flow-checkpoint-example + name: name + namespace: namespace +node-defaults: {} +rawOutputDataConfig: {} +securityContext: {} +spec: + connections: + n0: + - end-node + start-node: + - n0 + edges: + downstream: + n0: + - end-node + start-node: + - n0 + upstream: + end-node: + - n0 + n0: + - start-node + id: ::core.control_flow.checkpoint.example + nodes: + end-node: + id: end-node + inputBindings: + - binding: + promise: + nodeId: n0 + var: o0 + var: o0 + kind: end + resources: {} + n0: + id: n0 + inputBindings: + - binding: + promise: + nodeId: start-node + var: n_iterations + var: n_iterations + kind: task + name: use_checkpoint + resources: {} + retry: + minAttempts: 4 + task: 'resource_type:TASK name:"core.control_flow.checkpoint.use_checkpoint" ' + start-node: + id: start-node + kind: start + resources: {} + outputBindings: + - binding: + promise: + nodeId: n0 + var: o0 + var: o0 + outputs: + variables: + o0: + type: + simple: INTEGER +status: + phase: 0 +tasks: + 'resource_type:TASK name:"core.control_flow.checkpoint.use_checkpoint" ': + container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.control_flow.checkpoint + - task-name + - use_checkpoint + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.control_flow.checkpoint.use_checkpoint + resourceType: TASK + interface: + inputs: + variables: + n_iterations: + type: + simple: INTEGER + outputs: + variables: + o0: + type: + simple: INTEGER + metadata: + retries: + retries: 3 + runtime: + flavor: python + type: FLYTE_SDK + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/k8s/026_core.control_flow.conditions.multiplier_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/k8s/026_core.control_flow.conditions.multiplier_2_wf.yaml new file mode 100755 index 0000000000..8e959bd0c8 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/k8s/026_core.control_flow.conditions.multiplier_2_wf.yaml @@ -0,0 +1,256 @@ +apiVersion: flyte.lyft.com/v1alpha1 +executionConfig: + Interruptible: null + MaxParallelism: 0 + RecoveryExecution: {} + TaskPluginImpls: null + TaskResources: + Limits: + CPU: "0" + EphemeralStorage: "0" + GPU: "0" + Memory: "0" + Storage: "0" + Requests: + CPU: "0" + EphemeralStorage: "0" + GPU: "0" + Memory: "0" + Storage: "0" +executionId: {} +inputs: + literals: + my_input: + scalar: + primitive: + floatValue: 0 +kind: flyteworkflow +metadata: + creationTimestamp: null + labels: + domain: domain + execution-id: name + project: hello + shard-key: "6" + workflow-name: core-control-flow-conditions-multiplier + name: name + namespace: namespace +node-defaults: {} +rawOutputDataConfig: {} +securityContext: {} +spec: + connections: + n0: + - end-node + start-node: + - n0 + edges: + downstream: + n0: + - end-node + start-node: + - n0 + upstream: + end-node: + - n0 + n0: + - start-node + n0-n0: + - start-node + n0-n1: + - start-node + id: ::core.control_flow.conditions.multiplier + nodes: + end-node: + id: end-node + inputBindings: + - binding: + promise: + nodeId: n0 + var: o0 + var: o0 + kind: end + resources: {} + n0: + branch: + else: n0-n1 + if: + condition: + conjunction: + leftExpression: + comparison: + leftValue: + var: .my_input + operator: GTE + rightValue: + primitive: + floatValue: 0.1 + rightExpression: + comparison: + leftValue: + var: .my_input + operator: LTE + rightValue: + primitive: + floatValue: 1 + then: n0-n0 + id: n0 + inputBindings: + - binding: + promise: + nodeId: start-node + var: my_input + var: .my_input + kind: branch + name: fractions + resources: {} + retry: + minAttempts: 1 + n0-n0: + id: n0-n0 + inputBindings: + - binding: + promise: + nodeId: start-node + var: my_input + var: "n" + kind: task + name: double + resources: {} + retry: + minAttempts: 1 + task: 'resource_type:TASK name:"core.control_flow.conditions.double" ' + n0-n1: + id: n0-n1 + inputBindings: + - binding: + promise: + nodeId: start-node + var: my_input + var: "n" + kind: task + name: square + resources: {} + retry: + minAttempts: 1 + task: 'resource_type:TASK name:"core.control_flow.conditions.square" ' + start-node: + id: start-node + kind: start + resources: {} + outputBindings: + - binding: + promise: + nodeId: n0 + var: o0 + var: o0 + outputs: + variables: + o0: + type: + simple: FLOAT +status: + phase: 0 +tasks: + 'resource_type:TASK name:"core.control_flow.conditions.double" ': + container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.control_flow.conditions + - task-name + - double + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.control_flow.conditions.double + resourceType: TASK + interface: + inputs: + variables: + "n": + type: + simple: FLOAT + outputs: + variables: + o0: + type: + simple: FLOAT + metadata: + retries: {} + runtime: + flavor: python + type: FLYTE_SDK + version: 0.32.6 + type: python-task + 'resource_type:TASK name:"core.control_flow.conditions.square" ': + container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.control_flow.conditions + - task-name + - square + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.control_flow.conditions.square + resourceType: TASK + interface: + inputs: + variables: + "n": + type: + simple: FLOAT + outputs: + variables: + o0: + type: + simple: FLOAT + metadata: + retries: {} + runtime: + flavor: python + type: FLYTE_SDK + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/k8s/028_core.control_flow.conditions.multiplier_2_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/k8s/028_core.control_flow.conditions.multiplier_2_2_wf.yaml new file mode 100755 index 0000000000..7b94c0b89f --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/k8s/028_core.control_flow.conditions.multiplier_2_2_wf.yaml @@ -0,0 +1,278 @@ +apiVersion: flyte.lyft.com/v1alpha1 +executionConfig: + Interruptible: null + MaxParallelism: 0 + RecoveryExecution: {} + TaskPluginImpls: null + TaskResources: + Limits: + CPU: "0" + EphemeralStorage: "0" + GPU: "0" + Memory: "0" + Storage: "0" + Requests: + CPU: "0" + EphemeralStorage: "0" + GPU: "0" + Memory: "0" + Storage: "0" +executionId: {} +inputs: + literals: + my_input: + scalar: + primitive: + floatValue: 0 +kind: flyteworkflow +metadata: + creationTimestamp: null + labels: + domain: domain + execution-id: name + project: hello + shard-key: "6" + workflow-name: core-control-flow-conditions-multiplier-2 + name: name + namespace: namespace +node-defaults: {} +rawOutputDataConfig: {} +securityContext: {} +spec: + connections: + n0: + - end-node + start-node: + - n0 + edges: + downstream: + n0: + - end-node + start-node: + - n0 + upstream: + end-node: + - n0 + n0: + - start-node + n0-n0: + - start-node + n0-n1: + - start-node + id: ::core.control_flow.conditions.multiplier_2 + nodes: + end-node: + id: end-node + inputBindings: + - binding: + promise: + nodeId: n0 + var: o0 + var: o0 + kind: end + resources: {} + n0: + branch: + elseFail: + failedNodeId: fractions + message: The input must be between 0 and 10 + elseIf: + - condition: + conjunction: + leftExpression: + comparison: + leftValue: + var: .my_input + operator: GT + rightValue: + primitive: + floatValue: 1 + rightExpression: + comparison: + leftValue: + var: .my_input + operator: LTE + rightValue: + primitive: + floatValue: 10 + then: n0-n1 + if: + condition: + conjunction: + leftExpression: + comparison: + leftValue: + var: .my_input + operator: GT + rightValue: + primitive: + floatValue: 0.1 + rightExpression: + comparison: + leftValue: + var: .my_input + operator: LT + rightValue: + primitive: + floatValue: 1 + then: n0-n0 + id: n0 + inputBindings: + - binding: + promise: + nodeId: start-node + var: my_input + var: .my_input + kind: branch + name: fractions + resources: {} + retry: + minAttempts: 1 + n0-n0: + id: n0-n0 + inputBindings: + - binding: + promise: + nodeId: start-node + var: my_input + var: "n" + kind: task + name: double + resources: {} + retry: + minAttempts: 1 + task: 'resource_type:TASK name:"core.control_flow.conditions.double" ' + n0-n1: + id: n0-n1 + inputBindings: + - binding: + promise: + nodeId: start-node + var: my_input + var: "n" + kind: task + name: square + resources: {} + retry: + minAttempts: 1 + task: 'resource_type:TASK name:"core.control_flow.conditions.square" ' + start-node: + id: start-node + kind: start + resources: {} + outputBindings: + - binding: + promise: + nodeId: n0 + var: o0 + var: o0 + outputs: + variables: + o0: + type: + simple: FLOAT +status: + phase: 0 +tasks: + 'resource_type:TASK name:"core.control_flow.conditions.double" ': + container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.control_flow.conditions + - task-name + - double + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.control_flow.conditions.double + resourceType: TASK + interface: + inputs: + variables: + "n": + type: + simple: FLOAT + outputs: + variables: + o0: + type: + simple: FLOAT + metadata: + retries: {} + runtime: + flavor: python + type: FLYTE_SDK + version: 0.32.6 + type: python-task + 'resource_type:TASK name:"core.control_flow.conditions.square" ': + container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.control_flow.conditions + - task-name + - square + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.control_flow.conditions.square + resourceType: TASK + interface: + inputs: + variables: + "n": + type: + simple: FLOAT + outputs: + variables: + o0: + type: + simple: FLOAT + metadata: + retries: {} + runtime: + flavor: python + type: FLYTE_SDK + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/k8s/030_core.control_flow.conditions.multiplier_3_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/k8s/030_core.control_flow.conditions.multiplier_3_2_wf.yaml new file mode 100755 index 0000000000..eb37b0a975 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/k8s/030_core.control_flow.conditions.multiplier_3_2_wf.yaml @@ -0,0 +1,298 @@ +apiVersion: flyte.lyft.com/v1alpha1 +executionConfig: + Interruptible: null + MaxParallelism: 0 + RecoveryExecution: {} + TaskPluginImpls: null + TaskResources: + Limits: + CPU: "0" + EphemeralStorage: "0" + GPU: "0" + Memory: "0" + Storage: "0" + Requests: + CPU: "0" + EphemeralStorage: "0" + GPU: "0" + Memory: "0" + Storage: "0" +executionId: {} +inputs: + literals: + my_input: + scalar: + primitive: + floatValue: 0 +kind: flyteworkflow +metadata: + creationTimestamp: null + labels: + domain: domain + execution-id: name + project: hello + shard-key: "6" + workflow-name: core-control-flow-conditions-multiplier-3 + name: name + namespace: namespace +node-defaults: {} +rawOutputDataConfig: {} +securityContext: {} +spec: + connections: + n0: + - n1 + n1: + - end-node + start-node: + - n0 + edges: + downstream: + n0: + - n1 + n1: + - end-node + start-node: + - n0 + upstream: + end-node: + - n1 + n0: + - start-node + n0-n0: + - start-node + n0-n1: + - start-node + n1: + - n0 + id: ::core.control_flow.conditions.multiplier_3 + nodes: + end-node: + id: end-node + inputBindings: + - binding: + promise: + nodeId: n1 + var: o0 + var: o0 + kind: end + resources: {} + n0: + branch: + elseFail: + failedNodeId: fractions + message: The input must be between 0 and 10 + elseIf: + - condition: + conjunction: + leftExpression: + comparison: + leftValue: + var: .my_input + operator: GT + rightValue: + primitive: + floatValue: 1 + rightExpression: + comparison: + leftValue: + var: .my_input + operator: LT + rightValue: + primitive: + floatValue: 10 + then: n0-n1 + if: + condition: + conjunction: + leftExpression: + comparison: + leftValue: + var: .my_input + operator: GT + rightValue: + primitive: + floatValue: 0.1 + rightExpression: + comparison: + leftValue: + var: .my_input + operator: LT + rightValue: + primitive: + floatValue: 1 + then: n0-n0 + id: n0 + inputBindings: + - binding: + promise: + nodeId: start-node + var: my_input + var: .my_input + kind: branch + name: fractions + resources: {} + retry: + minAttempts: 1 + n0-n0: + id: n0-n0 + inputBindings: + - binding: + promise: + nodeId: start-node + var: my_input + var: "n" + kind: task + name: double + resources: {} + retry: + minAttempts: 1 + task: 'resource_type:TASK name:"core.control_flow.conditions.double" ' + n0-n1: + id: n0-n1 + inputBindings: + - binding: + promise: + nodeId: start-node + var: my_input + var: "n" + kind: task + name: square + resources: {} + retry: + minAttempts: 1 + task: 'resource_type:TASK name:"core.control_flow.conditions.square" ' + n1: + id: n1 + inputBindings: + - binding: + promise: + nodeId: n0 + var: o0 + var: "n" + kind: task + name: double + resources: {} + retry: + minAttempts: 1 + task: 'resource_type:TASK name:"core.control_flow.conditions.double" ' + start-node: + id: start-node + kind: start + resources: {} + outputBindings: + - binding: + promise: + nodeId: n1 + var: o0 + var: o0 + outputs: + variables: + o0: + type: + simple: FLOAT +status: + phase: 0 +tasks: + 'resource_type:TASK name:"core.control_flow.conditions.double" ': + container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.control_flow.conditions + - task-name + - double + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.control_flow.conditions.double + resourceType: TASK + interface: + inputs: + variables: + "n": + type: + simple: FLOAT + outputs: + variables: + o0: + type: + simple: FLOAT + metadata: + retries: {} + runtime: + flavor: python + type: FLYTE_SDK + version: 0.32.6 + type: python-task + 'resource_type:TASK name:"core.control_flow.conditions.square" ': + container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.control_flow.conditions + - task-name + - square + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.control_flow.conditions.square + resourceType: TASK + interface: + inputs: + variables: + "n": + type: + simple: FLOAT + outputs: + variables: + o0: + type: + simple: FLOAT + metadata: + retries: {} + runtime: + flavor: python + type: FLYTE_SDK + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/k8s/035_core.control_flow.conditions.basic_boolean_wf_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/k8s/035_core.control_flow.conditions.basic_boolean_wf_2_wf.yaml new file mode 100755 index 0000000000..03836a7dcf --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/k8s/035_core.control_flow.conditions.basic_boolean_wf_2_wf.yaml @@ -0,0 +1,292 @@ +apiVersion: flyte.lyft.com/v1alpha1 +executionConfig: + Interruptible: null + MaxParallelism: 0 + RecoveryExecution: {} + TaskPluginImpls: null + TaskResources: + Limits: + CPU: "0" + EphemeralStorage: "0" + GPU: "0" + Memory: "0" + Storage: "0" + Requests: + CPU: "0" + EphemeralStorage: "0" + GPU: "0" + Memory: "0" + Storage: "0" +executionId: {} +inputs: + literals: + seed: + scalar: + primitive: + integer: "0" +kind: flyteworkflow +metadata: + creationTimestamp: null + labels: + domain: domain + execution-id: name + project: hello + shard-key: "6" + workflow-name: core-control-flow-conditions-basic-boolean-wf + name: name + namespace: namespace +node-defaults: {} +rawOutputDataConfig: {} +securityContext: {} +spec: + connections: + n0: + - n1 + n1: + - end-node + start-node: + - n0 + edges: + downstream: + n0: + - n1 + n1: + - end-node + start-node: + - n0 + upstream: + end-node: + - n1 + n0: + - start-node + n1: + - n0 + id: ::core.control_flow.conditions.basic_boolean_wf + nodes: + end-node: + id: end-node + inputBindings: + - binding: + promise: + nodeId: n1 + var: o0 + var: o0 + kind: end + resources: {} + n0: + id: n0 + inputBindings: + - binding: + promise: + nodeId: start-node + var: seed + var: seed + kind: task + name: coin_toss + resources: {} + retry: + minAttempts: 1 + task: 'resource_type:TASK name:"core.control_flow.conditions.coin_toss" ' + n1: + branch: + else: n1-n1 + if: + condition: + comparison: + leftValue: + var: n0.o0 + rightValue: + primitive: + boolean: true + then: n1-n0 + id: n1 + inputBindings: + - binding: + promise: + nodeId: n0 + var: o0 + var: n0.o0 + kind: branch + name: test + resources: {} + retry: + minAttempts: 1 + n1-n0: + id: n1-n0 + kind: task + name: success + resources: {} + retry: + minAttempts: 1 + task: 'resource_type:TASK name:"core.control_flow.conditions.success" ' + n1-n1: + id: n1-n1 + kind: task + name: failed + resources: {} + retry: + minAttempts: 1 + task: 'resource_type:TASK name:"core.control_flow.conditions.failed" ' + start-node: + id: start-node + kind: start + resources: {} + outputBindings: + - binding: + promise: + nodeId: n1 + var: o0 + var: o0 + outputs: + variables: + o0: + type: + simple: INTEGER +status: + phase: 0 +tasks: + 'resource_type:TASK name:"core.control_flow.conditions.coin_toss" ': + container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.control_flow.conditions + - task-name + - coin_toss + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.control_flow.conditions.coin_toss + resourceType: TASK + interface: + inputs: + variables: + seed: + type: + simple: INTEGER + outputs: + variables: + o0: + type: + simple: BOOLEAN + metadata: + retries: {} + runtime: + flavor: python + type: FLYTE_SDK + version: 0.32.6 + type: python-task + 'resource_type:TASK name:"core.control_flow.conditions.failed" ': + container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.control_flow.conditions + - task-name + - failed + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.control_flow.conditions.failed + resourceType: TASK + interface: + inputs: {} + outputs: + variables: + o0: + type: + simple: INTEGER + metadata: + retries: {} + runtime: + flavor: python + type: FLYTE_SDK + version: 0.32.6 + type: python-task + 'resource_type:TASK name:"core.control_flow.conditions.success" ': + container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.control_flow.conditions + - task-name + - success + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.control_flow.conditions.success + resourceType: TASK + interface: + inputs: {} + outputs: + variables: + o0: + type: + simple: INTEGER + metadata: + retries: {} + runtime: + flavor: python + type: FLYTE_SDK + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/k8s/037_core.control_flow.conditions.bool_input_wf_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/k8s/037_core.control_flow.conditions.bool_input_wf_2_wf.yaml new file mode 100755 index 0000000000..c3854d6790 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/k8s/037_core.control_flow.conditions.bool_input_wf_2_wf.yaml @@ -0,0 +1,221 @@ +apiVersion: flyte.lyft.com/v1alpha1 +executionConfig: + Interruptible: null + MaxParallelism: 0 + RecoveryExecution: {} + TaskPluginImpls: null + TaskResources: + Limits: + CPU: "0" + EphemeralStorage: "0" + GPU: "0" + Memory: "0" + Storage: "0" + Requests: + CPU: "0" + EphemeralStorage: "0" + GPU: "0" + Memory: "0" + Storage: "0" +executionId: {} +inputs: + literals: + b: + scalar: + primitive: + boolean: false +kind: flyteworkflow +metadata: + creationTimestamp: null + labels: + domain: domain + execution-id: name + project: hello + shard-key: "6" + workflow-name: core-control-flow-conditions-bool-input-wf + name: name + namespace: namespace +node-defaults: {} +rawOutputDataConfig: {} +securityContext: {} +spec: + connections: + n0: + - end-node + start-node: + - n0 + edges: + downstream: + n0: + - end-node + start-node: + - n0 + upstream: + end-node: + - n0 + n0: + - start-node + id: ::core.control_flow.conditions.bool_input_wf + nodes: + end-node: + id: end-node + inputBindings: + - binding: + promise: + nodeId: n0 + var: o0 + var: o0 + kind: end + resources: {} + n0: + branch: + else: n0-n1 + if: + condition: + comparison: + leftValue: + var: .b + rightValue: + primitive: + boolean: true + then: n0-n0 + id: n0 + inputBindings: + - binding: + promise: + nodeId: start-node + var: b + var: .b + kind: branch + name: test + resources: {} + retry: + minAttempts: 1 + n0-n0: + id: n0-n0 + kind: task + name: success + resources: {} + retry: + minAttempts: 1 + task: 'resource_type:TASK name:"core.control_flow.conditions.success" ' + n0-n1: + id: n0-n1 + kind: task + name: failed + resources: {} + retry: + minAttempts: 1 + task: 'resource_type:TASK name:"core.control_flow.conditions.failed" ' + start-node: + id: start-node + kind: start + resources: {} + outputBindings: + - binding: + promise: + nodeId: n0 + var: o0 + var: o0 + outputs: + variables: + o0: + type: + simple: INTEGER +status: + phase: 0 +tasks: + 'resource_type:TASK name:"core.control_flow.conditions.failed" ': + container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.control_flow.conditions + - task-name + - failed + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.control_flow.conditions.failed + resourceType: TASK + interface: + inputs: {} + outputs: + variables: + o0: + type: + simple: INTEGER + metadata: + retries: {} + runtime: + flavor: python + type: FLYTE_SDK + version: 0.32.6 + type: python-task + 'resource_type:TASK name:"core.control_flow.conditions.success" ': + container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.control_flow.conditions + - task-name + - success + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.control_flow.conditions.success + resourceType: TASK + interface: + inputs: {} + outputs: + variables: + o0: + type: + simple: INTEGER + metadata: + retries: {} + runtime: + flavor: python + type: FLYTE_SDK + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/k8s/039_core.control_flow.conditions.nested_conditions_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/k8s/039_core.control_flow.conditions.nested_conditions_2_wf.yaml new file mode 100755 index 0000000000..1b9ec7a14e --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/k8s/039_core.control_flow.conditions.nested_conditions_2_wf.yaml @@ -0,0 +1,357 @@ +apiVersion: flyte.lyft.com/v1alpha1 +executionConfig: + Interruptible: null + MaxParallelism: 0 + RecoveryExecution: {} + TaskPluginImpls: null + TaskResources: + Limits: + CPU: "0" + EphemeralStorage: "0" + GPU: "0" + Memory: "0" + Storage: "0" + Requests: + CPU: "0" + EphemeralStorage: "0" + GPU: "0" + Memory: "0" + Storage: "0" +executionId: {} +inputs: + literals: + my_input: + scalar: + primitive: + floatValue: 0 +kind: flyteworkflow +metadata: + creationTimestamp: null + labels: + domain: domain + execution-id: name + project: hello + shard-key: "6" + workflow-name: core-control-flow-conditions-nested-conditions + name: name + namespace: namespace +node-defaults: {} +rawOutputDataConfig: {} +securityContext: {} +spec: + connections: + n0: + - end-node + start-node: + - n0 + edges: + downstream: + n0: + - end-node + start-node: + - n0 + upstream: + end-node: + - n0 + n0: + - start-node + n0-n0: + - start-node + n0-n0-n0: + - start-node + n0-n0-n1: + - start-node + n0-n1: + - start-node + n0-n2: + - start-node + id: ::core.control_flow.conditions.nested_conditions + nodes: + end-node: + id: end-node + inputBindings: + - binding: + promise: + nodeId: n0 + var: o0 + var: o0 + kind: end + resources: {} + n0: + branch: + else: n0-n2 + elseIf: + - condition: + conjunction: + leftExpression: + comparison: + leftValue: + var: .my_input + operator: GT + rightValue: + primitive: + floatValue: 1 + rightExpression: + comparison: + leftValue: + var: .my_input + operator: LT + rightValue: + primitive: + floatValue: 10 + then: n0-n1 + if: + condition: + conjunction: + leftExpression: + comparison: + leftValue: + var: .my_input + operator: GT + rightValue: + primitive: + floatValue: 0.1 + rightExpression: + comparison: + leftValue: + var: .my_input + operator: LT + rightValue: + primitive: + floatValue: 1 + then: n0-n0 + id: n0 + inputBindings: + - binding: + promise: + nodeId: start-node + var: my_input + var: .my_input + kind: branch + name: fractions + resources: {} + retry: + minAttempts: 1 + n0-n0: + branch: + elseFail: + failedNodeId: inner_fractions + message: Only <0.7 allowed + elseIf: + - condition: + conjunction: + leftExpression: + comparison: + leftValue: + var: .my_input + operator: GT + rightValue: + primitive: + floatValue: 0.5 + rightExpression: + comparison: + leftValue: + var: .my_input + operator: LT + rightValue: + primitive: + floatValue: 0.7 + then: n0-n0-n1 + if: + condition: + comparison: + leftValue: + var: .my_input + operator: LT + rightValue: + primitive: + floatValue: 0.5 + then: n0-n0-n0 + id: n0-n0 + inputBindings: + - binding: + promise: + nodeId: start-node + var: my_input + var: .my_input + kind: branch + name: inner_fractions + resources: {} + retry: + minAttempts: 1 + n0-n0-n0: + id: n0-n0-n0 + inputBindings: + - binding: + promise: + nodeId: start-node + var: my_input + var: "n" + kind: task + name: double + resources: {} + retry: + minAttempts: 1 + task: 'resource_type:TASK name:"core.control_flow.conditions.double" ' + n0-n0-n1: + id: n0-n0-n1 + inputBindings: + - binding: + promise: + nodeId: start-node + var: my_input + var: "n" + kind: task + name: square + resources: {} + retry: + minAttempts: 1 + task: 'resource_type:TASK name:"core.control_flow.conditions.square" ' + n0-n1: + id: n0-n1 + inputBindings: + - binding: + promise: + nodeId: start-node + var: my_input + var: "n" + kind: task + name: square + resources: {} + retry: + minAttempts: 1 + task: 'resource_type:TASK name:"core.control_flow.conditions.square" ' + n0-n2: + id: n0-n2 + inputBindings: + - binding: + promise: + nodeId: start-node + var: my_input + var: "n" + kind: task + name: double + resources: {} + retry: + minAttempts: 1 + task: 'resource_type:TASK name:"core.control_flow.conditions.double" ' + start-node: + id: start-node + kind: start + resources: {} + outputBindings: + - binding: + promise: + nodeId: n0 + var: o0 + var: o0 + outputs: + variables: + o0: + type: + simple: FLOAT +status: + phase: 0 +tasks: + 'resource_type:TASK name:"core.control_flow.conditions.double" ': + container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.control_flow.conditions + - task-name + - double + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.control_flow.conditions.double + resourceType: TASK + interface: + inputs: + variables: + "n": + type: + simple: FLOAT + outputs: + variables: + o0: + type: + simple: FLOAT + metadata: + retries: {} + runtime: + flavor: python + type: FLYTE_SDK + version: 0.32.6 + type: python-task + 'resource_type:TASK name:"core.control_flow.conditions.square" ': + container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.control_flow.conditions + - task-name + - square + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.control_flow.conditions.square + resourceType: TASK + interface: + inputs: + variables: + "n": + type: + simple: FLOAT + outputs: + variables: + o0: + type: + simple: FLOAT + metadata: + retries: {} + runtime: + flavor: python + type: FLYTE_SDK + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/k8s/042_core.control_flow.conditions.consume_outputs_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/k8s/042_core.control_flow.conditions.consume_outputs_2_wf.yaml new file mode 100755 index 0000000000..a2ff6638cf --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/k8s/042_core.control_flow.conditions.consume_outputs_2_wf.yaml @@ -0,0 +1,399 @@ +apiVersion: flyte.lyft.com/v1alpha1 +executionConfig: + Interruptible: null + MaxParallelism: 0 + RecoveryExecution: {} + TaskPluginImpls: null + TaskResources: + Limits: + CPU: "0" + EphemeralStorage: "0" + GPU: "0" + Memory: "0" + Storage: "0" + Requests: + CPU: "0" + EphemeralStorage: "0" + GPU: "0" + Memory: "0" + Storage: "0" +executionId: {} +inputs: + literals: + my_input: + scalar: + primitive: + floatValue: 0 + seed: + scalar: + primitive: + integer: "0" +kind: flyteworkflow +metadata: + creationTimestamp: null + labels: + domain: domain + execution-id: name + project: hello + shard-key: "6" + workflow-name: core-control-flow-conditions-consume-outputs + name: name + namespace: namespace +node-defaults: {} +rawOutputDataConfig: {} +securityContext: {} +spec: + connections: + n0: + - n1 + n1: + - n2 + n2: + - end-node + start-node: + - n0 + edges: + downstream: + n0: + - n1 + n1: + - n2 + n2: + - end-node + start-node: + - n0 + upstream: + end-node: + - n2 + n0: + - start-node + n1: + - n0 + n1-n0: + - start-node + n1-n1: + - start-node + n2: + - n1 + id: ::core.control_flow.conditions.consume_outputs + nodes: + end-node: + id: end-node + inputBindings: + - binding: + promise: + nodeId: n2 + var: o0 + var: o0 + kind: end + resources: {} + n0: + id: n0 + inputBindings: + - binding: + promise: + nodeId: start-node + var: seed + var: seed + kind: task + name: coin_toss + resources: {} + retry: + minAttempts: 1 + task: 'resource_type:TASK name:"core.control_flow.conditions.coin_toss" ' + n1: + branch: + else: n1-n1 + if: + condition: + comparison: + leftValue: + var: n0.o0 + rightValue: + primitive: + boolean: true + then: n1-n0 + id: n1 + inputBindings: + - binding: + promise: + nodeId: n0 + var: o0 + var: n0.o0 + kind: branch + name: double_or_square + resources: {} + retry: + minAttempts: 1 + n1-n0: + id: n1-n0 + inputBindings: + - binding: + promise: + nodeId: start-node + var: my_input + var: "n" + kind: task + name: square + resources: {} + retry: + minAttempts: 1 + task: 'resource_type:TASK name:"core.control_flow.conditions.square" ' + n1-n1: + id: n1-n1 + inputBindings: + - binding: + promise: + nodeId: start-node + var: my_input + var: a + - binding: + promise: + nodeId: start-node + var: my_input + var: b + kind: task + name: calc_sum + resources: {} + retry: + minAttempts: 1 + task: 'resource_type:TASK name:"core.control_flow.conditions.calc_sum" ' + n2: + id: n2 + inputBindings: + - binding: + promise: + nodeId: n1 + var: o0 + var: "n" + kind: task + name: double + resources: {} + retry: + minAttempts: 1 + task: 'resource_type:TASK name:"core.control_flow.conditions.double" ' + start-node: + id: start-node + kind: start + resources: {} + outputBindings: + - binding: + promise: + nodeId: n2 + var: o0 + var: o0 + outputs: + variables: + o0: + type: + simple: FLOAT +status: + phase: 0 +tasks: + 'resource_type:TASK name:"core.control_flow.conditions.calc_sum" ': + container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.control_flow.conditions + - task-name + - calc_sum + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.control_flow.conditions.calc_sum + resourceType: TASK + interface: + inputs: + variables: + a: + type: + simple: FLOAT + b: + type: + simple: FLOAT + outputs: + variables: + o0: + type: + simple: FLOAT + metadata: + retries: {} + runtime: + flavor: python + type: FLYTE_SDK + version: 0.32.6 + type: python-task + 'resource_type:TASK name:"core.control_flow.conditions.coin_toss" ': + container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.control_flow.conditions + - task-name + - coin_toss + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.control_flow.conditions.coin_toss + resourceType: TASK + interface: + inputs: + variables: + seed: + type: + simple: INTEGER + outputs: + variables: + o0: + type: + simple: BOOLEAN + metadata: + retries: {} + runtime: + flavor: python + type: FLYTE_SDK + version: 0.32.6 + type: python-task + 'resource_type:TASK name:"core.control_flow.conditions.double" ': + container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.control_flow.conditions + - task-name + - double + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.control_flow.conditions.double + resourceType: TASK + interface: + inputs: + variables: + "n": + type: + simple: FLOAT + outputs: + variables: + o0: + type: + simple: FLOAT + metadata: + retries: {} + runtime: + flavor: python + type: FLYTE_SDK + version: 0.32.6 + type: python-task + 'resource_type:TASK name:"core.control_flow.conditions.square" ': + container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.control_flow.conditions + - task-name + - square + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.control_flow.conditions.square + resourceType: TASK + interface: + inputs: + variables: + "n": + type: + simple: FLOAT + outputs: + variables: + o0: + type: + simple: FLOAT + metadata: + retries: {} + runtime: + flavor: python + type: FLYTE_SDK + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/k8s/048_core.control_flow.dynamics.wf_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/k8s/048_core.control_flow.dynamics.wf_2_wf.yaml new file mode 100755 index 0000000000..45edd83041 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/k8s/048_core.control_flow.dynamics.wf_2_wf.yaml @@ -0,0 +1,167 @@ +apiVersion: flyte.lyft.com/v1alpha1 +executionConfig: + Interruptible: null + MaxParallelism: 0 + RecoveryExecution: {} + TaskPluginImpls: null + TaskResources: + Limits: + CPU: "0" + EphemeralStorage: "0" + GPU: "0" + Memory: "0" + Storage: "0" + Requests: + CPU: "0" + EphemeralStorage: "0" + GPU: "0" + Memory: "0" + Storage: "0" +executionId: {} +inputs: + literals: + s1: + scalar: + primitive: + stringValue: "" + s2: + scalar: + primitive: + stringValue: "" +kind: flyteworkflow +metadata: + creationTimestamp: null + labels: + domain: domain + execution-id: name + project: hello + shard-key: "6" + workflow-name: core-control-flow-dynamics-wf + name: name + namespace: namespace +node-defaults: {} +rawOutputDataConfig: {} +securityContext: {} +spec: + connections: + n0: + - end-node + start-node: + - n0 + edges: + downstream: + n0: + - end-node + start-node: + - n0 + upstream: + end-node: + - n0 + n0: + - start-node + id: ::core.control_flow.dynamics.wf + nodes: + end-node: + id: end-node + inputBindings: + - binding: + promise: + nodeId: n0 + var: o0 + var: o0 + kind: end + resources: {} + n0: + id: n0 + inputBindings: + - binding: + promise: + nodeId: start-node + var: s1 + var: s1 + - binding: + promise: + nodeId: start-node + var: s2 + var: s2 + kind: task + name: count_characters + resources: {} + retry: + minAttempts: 1 + task: 'resource_type:TASK name:"core.control_flow.dynamics.count_characters" ' + start-node: + id: start-node + kind: start + resources: {} + outputBindings: + - binding: + promise: + nodeId: n0 + var: o0 + var: o0 + outputs: + variables: + o0: + type: + simple: INTEGER +status: + phase: 0 +tasks: + 'resource_type:TASK name:"core.control_flow.dynamics.count_characters" ': + container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.control_flow.dynamics + - task-name + - count_characters + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + env: + - key: _F_SS_C + value: H4sIAAAAAAAC/8VRXW7DIAy+ysTzWtKtUuiuMk2RAZN6pZCBE6mrevcBXdXuBHsIsr8f85mcBR1hxMHE4GgUb09nYdHB7HloREMCHGtxY8Tzk3BfoSLj3qQ1Ren8iTGm8VqYGA+6fFXIUKcKExOulFa4AbXdmR1YZ0CpbruDrt/qTWfBvvTK9V3X9+JSjO36XLzv/xHgo0aYUvxEw8UeZu8LYOMRKNz7BVOm+ABgWO7NdOJ9DAMFxjQlLGcNIuPEcilCqSnIq+a1bVSTH4iHhRLP4ItkSDHyX5MnfTOtlczEuJrAHOpbyduENg0yDxkTgadv4JKydMwUxtz+KQbQHm2pHfiMdTnMhb5KLaWHrSlzIj03xkfTJL/05fIDQ+X0kUICAAA= + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.control_flow.dynamics.count_characters + resourceType: TASK + interface: + inputs: + variables: + s1: + type: + simple: STRING + s2: + type: + simple: STRING + outputs: + variables: + o0: + type: + simple: INTEGER + metadata: + retries: {} + runtime: + flavor: python + type: FLYTE_SDK + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/k8s/053_core.control_flow.map_task.my_map_workflow_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/k8s/053_core.control_flow.map_task.my_map_workflow_2_wf.yaml new file mode 100755 index 0000000000..552d1b806c --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/k8s/053_core.control_flow.map_task.my_map_workflow_2_wf.yaml @@ -0,0 +1,235 @@ +apiVersion: flyte.lyft.com/v1alpha1 +executionConfig: + Interruptible: null + MaxParallelism: 0 + RecoveryExecution: {} + TaskPluginImpls: null + TaskResources: + Limits: + CPU: "0" + EphemeralStorage: "0" + GPU: "0" + Memory: "0" + Storage: "0" + Requests: + CPU: "0" + EphemeralStorage: "0" + GPU: "0" + Memory: "0" + Storage: "0" +executionId: {} +inputs: + literals: + a: + collection: + literals: + - scalar: + primitive: + integer: "0" +kind: flyteworkflow +metadata: + creationTimestamp: null + labels: + domain: domain + execution-id: name + project: hello + shard-key: "6" + workflow-name: core-control-flow-map-task-my-map-workflow + name: name + namespace: namespace +node-defaults: {} +rawOutputDataConfig: {} +securityContext: {} +spec: + connections: + n0: + - n1 + n1: + - end-node + start-node: + - n0 + edges: + downstream: + n0: + - n1 + n1: + - end-node + start-node: + - n0 + upstream: + end-node: + - n1 + n0: + - start-node + n1: + - n0 + id: ::core.control_flow.map_task.my_map_workflow + nodes: + end-node: + id: end-node + inputBindings: + - binding: + promise: + nodeId: n1 + var: o0 + var: o0 + kind: end + resources: {} + n0: + id: n0 + inputBindings: + - binding: + promise: + nodeId: start-node + var: a + var: a + kind: task + name: mapper_a_mappable_task_0 + resources: + limits: + memory: 500Mi + requests: + memory: 300Mi + retry: + minAttempts: 2 + task: 'resource_type:TASK name:"core.control_flow.map_task.mapper_a_mappable_task_0" ' + n1: + id: n1 + inputBindings: + - binding: + promise: + nodeId: n0 + var: o0 + var: b + kind: task + name: coalesce + resources: {} + retry: + minAttempts: 1 + task: 'resource_type:TASK name:"core.control_flow.map_task.coalesce" ' + start-node: + id: start-node + kind: start + resources: {} + outputBindings: + - binding: + promise: + nodeId: n1 + var: o0 + var: o0 + outputs: + variables: + o0: + type: + simple: STRING +status: + phase: 0 +tasks: + 'resource_type:TASK name:"core.control_flow.map_task.coalesce" ': + container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.control_flow.map_task + - task-name + - coalesce + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.control_flow.map_task.coalesce + resourceType: TASK + interface: + inputs: + variables: + b: + type: + collectionType: + simple: STRING + outputs: + variables: + o0: + type: + simple: STRING + metadata: + retries: {} + runtime: + flavor: python + type: FLYTE_SDK + version: 0.32.6 + type: python-task + 'resource_type:TASK name:"core.control_flow.map_task.mapper_a_mappable_task_0" ': + container: + args: + - pyflyte-map-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.control_flow.map_task + - task-name + - a_mappable_task + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + custom: + minSuccessRatio: 1 + id: + name: core.control_flow.map_task.mapper_a_mappable_task_0 + resourceType: TASK + interface: + inputs: + variables: + a: + type: + collectionType: + simple: INTEGER + outputs: + variables: + o0: + type: + collectionType: + simple: STRING + metadata: + retries: {} + runtime: + flavor: python + type: FLYTE_SDK + version: 0.32.6 + taskTypeVersion: 1 + type: container_array diff --git a/pkg/compiler/test/testdata/snacks-core/k8s/059_core.control_flow.merge_sort.merge_sort_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/k8s/059_core.control_flow.merge_sort.merge_sort_2_wf.yaml new file mode 100755 index 0000000000..fa333434da --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/k8s/059_core.control_flow.merge_sort.merge_sort_2_wf.yaml @@ -0,0 +1,276 @@ +apiVersion: flyte.lyft.com/v1alpha1 +executionConfig: + Interruptible: null + MaxParallelism: 0 + RecoveryExecution: {} + TaskPluginImpls: null + TaskResources: + Limits: + CPU: "0" + EphemeralStorage: "0" + GPU: "0" + Memory: "0" + Storage: "0" + Requests: + CPU: "0" + EphemeralStorage: "0" + GPU: "0" + Memory: "0" + Storage: "0" +executionId: {} +inputs: + literals: + numbers: + collection: + literals: + - scalar: + primitive: + integer: "0" + numbers_count: + scalar: + primitive: + integer: "0" + run_local_at_count: + scalar: + primitive: + integer: "0" +kind: flyteworkflow +metadata: + creationTimestamp: null + labels: + domain: domain + execution-id: name + project: hello + shard-key: "6" + workflow-name: core-control-flow-merge-sort-merge-sort + name: name + namespace: namespace +node-defaults: {} +rawOutputDataConfig: {} +securityContext: {} +spec: + connections: + n0: + - end-node + start-node: + - n0 + edges: + downstream: + n0: + - end-node + start-node: + - n0 + upstream: + end-node: + - n0 + n0: + - start-node + n0-n0: + - start-node + n0-n1: + - start-node + id: ::core.control_flow.merge_sort.merge_sort + nodes: + end-node: + id: end-node + inputBindings: + - binding: + promise: + nodeId: n0 + var: o0 + var: o0 + kind: end + resources: {} + n0: + branch: + else: n0-n1 + if: + condition: + comparison: + leftValue: + var: .numbers_count + operator: LTE + rightValue: + var: .run_local_at_count + then: n0-n0 + id: n0 + inputBindings: + - binding: + promise: + nodeId: start-node + var: numbers_count + var: .numbers_count + - binding: + promise: + nodeId: start-node + var: run_local_at_count + var: .run_local_at_count + kind: branch + name: terminal_case + resources: {} + retry: + minAttempts: 1 + n0-n0: + id: n0-n0 + inputBindings: + - binding: + promise: + nodeId: start-node + var: numbers + var: numbers + kind: task + name: sort_locally + resources: {} + retry: + minAttempts: 1 + task: 'resource_type:TASK name:"core.control_flow.merge_sort.sort_locally" ' + n0-n1: + id: n0-n1 + inputBindings: + - binding: + promise: + nodeId: start-node + var: numbers + var: numbers + - binding: + promise: + nodeId: start-node + var: run_local_at_count + var: run_local_at_count + kind: task + name: merge_sort_remotely + resources: {} + retry: + minAttempts: 1 + task: 'resource_type:TASK name:"core.control_flow.merge_sort.merge_sort_remotely" ' + start-node: + id: start-node + kind: start + resources: {} + outputBindings: + - binding: + promise: + nodeId: n0 + var: o0 + var: o0 + outputs: + variables: + o0: + type: + collectionType: + simple: INTEGER +status: + phase: 0 +tasks: + 'resource_type:TASK name:"core.control_flow.merge_sort.merge_sort_remotely" ': + container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.control_flow.merge_sort + - task-name + - merge_sort_remotely + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + env: + - key: _F_SS_C + value: H4sIAAAAAAAC/8VRXW7DIAy+ysTzWtKtUuiuMk2RAZN6pZCBE6mrevcBXdXuBHsIsr8f85mcBR1hxMHE4GgUb09nYdHB7HloREMCHGtxY8Tzk3BfoSLj3qQ1Ren8iTGm8VqYGA+6fFXIUKcKExOulFa4AbXdmR1YZ0CpbruDrt/qTWfBvvTK9V3X9+JSjO36XLzv/xHgo0aYUvxEw8UeZu8LYOMRKNz7BVOm+ABgWO7NdOJ9DAMFxjQlLGcNIuPEcilCqSnIq+a1bVSTH4iHhRLP4ItkSDHyX5MnfTOtlczEuJrAHOpbyduENg0yDxkTgadv4JKydMwUxtz+KQbQHm2pHfiMdTnMhb5KLaWHrSlzIj03xkfTJL/05fIDQ+X0kUICAAA= + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.control_flow.merge_sort.merge_sort_remotely + resourceType: TASK + interface: + inputs: + variables: + numbers: + type: + collectionType: + simple: INTEGER + run_local_at_count: + type: + simple: INTEGER + outputs: + variables: + o0: + type: + collectionType: + simple: INTEGER + metadata: + retries: {} + runtime: + flavor: python + type: FLYTE_SDK + version: 0.32.6 + type: python-task + 'resource_type:TASK name:"core.control_flow.merge_sort.sort_locally" ': + container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.control_flow.merge_sort + - task-name + - sort_locally + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.control_flow.merge_sort.sort_locally + resourceType: TASK + interface: + inputs: + variables: + numbers: + type: + collectionType: + simple: INTEGER + outputs: + variables: + o0: + type: + collectionType: + simple: INTEGER + metadata: + retries: {} + runtime: + flavor: python + type: FLYTE_SDK + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/k8s/062_core.control_flow.subworkflows.my_subwf_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/k8s/062_core.control_flow.subworkflows.my_subwf_2_wf.yaml new file mode 100755 index 0000000000..633da9a819 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/k8s/062_core.control_flow.subworkflows.my_subwf_2_wf.yaml @@ -0,0 +1,191 @@ +apiVersion: flyte.lyft.com/v1alpha1 +executionConfig: + Interruptible: null + MaxParallelism: 0 + RecoveryExecution: {} + TaskPluginImpls: null + TaskResources: + Limits: + CPU: "0" + EphemeralStorage: "0" + GPU: "0" + Memory: "0" + Storage: "0" + Requests: + CPU: "0" + EphemeralStorage: "0" + GPU: "0" + Memory: "0" + Storage: "0" +executionId: {} +inputs: + literals: + a: + scalar: + primitive: + integer: "0" +kind: flyteworkflow +metadata: + creationTimestamp: null + labels: + domain: domain + execution-id: name + project: hello + shard-key: "6" + workflow-name: core-control-flow-subworkflows-my-subwf + name: name + namespace: namespace +node-defaults: {} +rawOutputDataConfig: {} +securityContext: {} +spec: + connections: + n0: + - end-node + - n1 + n1: + - end-node + start-node: + - n0 + edges: + downstream: + n0: + - end-node + - n1 + n1: + - end-node + start-node: + - n0 + upstream: + end-node: + - n0 + - n1 + n0: + - start-node + n1: + - n0 + id: ::core.control_flow.subworkflows.my_subwf + nodes: + end-node: + id: end-node + inputBindings: + - binding: + promise: + nodeId: n0 + var: c + var: o0 + - binding: + promise: + nodeId: n1 + var: c + var: o1 + kind: end + resources: {} + n0: + id: n0 + inputBindings: + - binding: + promise: + nodeId: start-node + var: a + var: a + kind: task + name: t1 + resources: {} + retry: + minAttempts: 1 + task: 'resource_type:TASK name:"core.control_flow.subworkflows.t1" ' + n1: + id: n1 + inputBindings: + - binding: + promise: + nodeId: n0 + var: t1_int_output + var: a + kind: task + name: t1 + resources: {} + retry: + minAttempts: 1 + task: 'resource_type:TASK name:"core.control_flow.subworkflows.t1" ' + start-node: + id: start-node + kind: start + resources: {} + outputBindings: + - binding: + promise: + nodeId: n0 + var: c + var: o0 + - binding: + promise: + nodeId: n1 + var: c + var: o1 + outputs: + variables: + o0: + type: + simple: STRING + o1: + type: + simple: STRING +status: + phase: 0 +tasks: + 'resource_type:TASK name:"core.control_flow.subworkflows.t1" ': + container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.control_flow.subworkflows + - task-name + - t1 + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.control_flow.subworkflows.t1 + resourceType: TASK + interface: + inputs: + variables: + a: + type: + simple: INTEGER + outputs: + variables: + c: + type: + simple: STRING + t1_int_output: + type: + simple: INTEGER + metadata: + retries: {} + runtime: + flavor: python + type: FLYTE_SDK + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/k8s/069_core.control_flow.subworkflows.ext_workflow_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/k8s/069_core.control_flow.subworkflows.ext_workflow_2_wf.yaml new file mode 100755 index 0000000000..98f73f45c7 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/k8s/069_core.control_flow.subworkflows.ext_workflow_2_wf.yaml @@ -0,0 +1,152 @@ +apiVersion: flyte.lyft.com/v1alpha1 +executionConfig: + Interruptible: null + MaxParallelism: 0 + RecoveryExecution: {} + TaskPluginImpls: null + TaskResources: + Limits: + CPU: "0" + EphemeralStorage: "0" + GPU: "0" + Memory: "0" + Storage: "0" + Requests: + CPU: "0" + EphemeralStorage: "0" + GPU: "0" + Memory: "0" + Storage: "0" +executionId: {} +inputs: + literals: + my_input: + scalar: + primitive: + stringValue: "" +kind: flyteworkflow +metadata: + creationTimestamp: null + labels: + domain: domain + execution-id: name + project: hello + shard-key: "6" + workflow-name: core-control-flow-subworkflows-ext-workflow + name: name + namespace: namespace +node-defaults: {} +rawOutputDataConfig: {} +securityContext: {} +spec: + connections: + n0: + - end-node + start-node: + - n0 + edges: + downstream: + n0: + - end-node + start-node: + - n0 + upstream: + end-node: + - n0 + n0: + - start-node + id: ::core.control_flow.subworkflows.ext_workflow + nodes: + end-node: + id: end-node + inputBindings: + - binding: + promise: + nodeId: n0 + var: o0 + var: o0 + kind: end + resources: {} + n0: + id: n0 + inputBindings: + - binding: + promise: + nodeId: start-node + var: my_input + var: input_string1 + kind: task + name: count_freq_words + resources: {} + retry: + minAttempts: 1 + task: 'resource_type:TASK name:"core.control_flow.subworkflows.count_freq_words" ' + start-node: + id: start-node + kind: start + resources: {} + outputBindings: + - binding: + promise: + nodeId: n0 + var: o0 + var: o0 + outputs: + variables: + o0: + type: + simple: STRUCT +status: + phase: 0 +tasks: + 'resource_type:TASK name:"core.control_flow.subworkflows.count_freq_words" ': + container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.control_flow.subworkflows + - task-name + - count_freq_words + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.control_flow.subworkflows.count_freq_words + resourceType: TASK + interface: + inputs: + variables: + input_string1: + type: + simple: STRING + outputs: + variables: + o0: + type: + simple: STRUCT + metadata: + retries: {} + runtime: + flavor: python + type: FLYTE_SDK + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/k8s/077_core.extend_flyte.custom_task_plugin.my_workflow_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/k8s/077_core.extend_flyte.custom_task_plugin.my_workflow_2_wf.yaml new file mode 100755 index 0000000000..91a5e738d7 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/k8s/077_core.extend_flyte.custom_task_plugin.my_workflow_2_wf.yaml @@ -0,0 +1,198 @@ +apiVersion: flyte.lyft.com/v1alpha1 +executionConfig: + Interruptible: null + MaxParallelism: 0 + RecoveryExecution: {} + TaskPluginImpls: null + TaskResources: + Limits: + CPU: "0" + EphemeralStorage: "0" + GPU: "0" + Memory: "0" + Storage: "0" + Requests: + CPU: "0" + EphemeralStorage: "0" + GPU: "0" + Memory: "0" + Storage: "0" +executionId: {} +inputs: + literals: + path: + scalar: + primitive: + stringValue: "" +kind: flyteworkflow +metadata: + creationTimestamp: null + labels: + domain: domain + execution-id: name + project: hello + shard-key: "6" + workflow-name: core-extend-flyte-custom-task-plugin-my-workflow + name: name + namespace: namespace +node-defaults: {} +rawOutputDataConfig: {} +securityContext: {} +spec: + connections: + n0: + - n1 + n1: + - end-node + start-node: + - n0 + edges: + downstream: + n0: + - n1 + n1: + - end-node + start-node: + - n0 + upstream: + end-node: + - n1 + n0: + - start-node + n1: + - n0 + id: ::core.extend_flyte.custom_task_plugin.my_workflow + nodes: + end-node: + id: end-node + inputBindings: + - binding: + promise: + nodeId: n1 + var: o0 + var: o0 + kind: end + resources: {} + n0: + activeDeadline: 40m0s + executionDeadline: 20m0s + id: n0 + inputBindings: + - binding: + promise: + nodeId: start-node + var: path + var: path + kind: task + name: my-objectstore-sensor + resources: {} + retry: + minAttempts: 11 + task: 'resource_type:TASK name:"my-objectstore-sensor" ' + n1: + id: n1 + inputBindings: + - binding: + promise: + nodeId: n0 + var: path + var: path + kind: task + name: print_file + resources: {} + retry: + minAttempts: 1 + task: 'resource_type:TASK name:"core.extend_flyte.custom_task_plugin.print_file" ' + start-node: + id: start-node + kind: start + resources: {} + outputBindings: + - binding: + promise: + nodeId: n1 + var: o0 + var: o0 + outputs: + variables: + o0: + type: + simple: STRING +status: + phase: 0 +tasks: + 'resource_type:TASK name:"core.extend_flyte.custom_task_plugin.print_file" ': + container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.extend_flyte.custom_task_plugin + - task-name + - print_file + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.extend_flyte.custom_task_plugin.print_file + resourceType: TASK + interface: + inputs: + variables: + path: + type: + simple: STRING + outputs: + variables: + o0: + type: + simple: STRING + metadata: + retries: {} + runtime: + flavor: python + type: FLYTE_SDK + version: 0.32.6 + type: python-task + 'resource_type:TASK name:"my-objectstore-sensor" ': + id: + name: my-objectstore-sensor + resourceType: TASK + interface: + inputs: + variables: + path: + type: + simple: STRING + outputs: + variables: + path: + type: + simple: STRING + metadata: + retries: + retries: 10 + runtime: + flavor: python + type: FLYTE_SDK + version: 0.32.6 + timeout: 1200s + type: object-store-sensor diff --git a/pkg/compiler/test/testdata/snacks-core/k8s/081_core.extend_flyte.custom_types.wf_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/k8s/081_core.extend_flyte.custom_types.wf_2_wf.yaml new file mode 100755 index 0000000000..92daf86892 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/k8s/081_core.extend_flyte.custom_types.wf_2_wf.yaml @@ -0,0 +1,212 @@ +apiVersion: flyte.lyft.com/v1alpha1 +executionConfig: + Interruptible: null + MaxParallelism: 0 + RecoveryExecution: {} + TaskPluginImpls: null + TaskResources: + Limits: + CPU: "0" + EphemeralStorage: "0" + GPU: "0" + Memory: "0" + Storage: "0" + Requests: + CPU: "0" + EphemeralStorage: "0" + GPU: "0" + Memory: "0" + Storage: "0" +executionId: {} +inputs: {} +kind: flyteworkflow +metadata: + creationTimestamp: null + labels: + domain: domain + execution-id: name + project: hello + shard-key: "6" + workflow-name: core-extend-flyte-custom-types-wf + name: name + namespace: namespace +node-defaults: {} +rawOutputDataConfig: {} +securityContext: {} +spec: + connections: + n0: + - n1 + n1: + - end-node + start-node: + - n0 + edges: + downstream: + n0: + - n1 + n1: + - end-node + start-node: + - n0 + upstream: + end-node: + - n1 + n0: + - start-node + n1: + - n0 + id: ::core.extend_flyte.custom_types.wf + nodes: + end-node: + id: end-node + inputBindings: + - binding: + promise: + nodeId: n1 + var: o0 + var: o0 + kind: end + resources: {} + n0: + id: n0 + kind: task + name: generate + resources: {} + retry: + minAttempts: 1 + task: 'resource_type:TASK name:"core.extend_flyte.custom_types.generate" ' + n1: + id: n1 + inputBindings: + - binding: + promise: + nodeId: n0 + var: o0 + var: d + kind: task + name: consume + resources: {} + retry: + minAttempts: 1 + task: 'resource_type:TASK name:"core.extend_flyte.custom_types.consume" ' + start-node: + id: start-node + kind: start + resources: {} + outputBindings: + - binding: + promise: + nodeId: n1 + var: o0 + var: o0 + outputs: + variables: + o0: + type: + simple: STRING +status: + phase: 0 +tasks: + 'resource_type:TASK name:"core.extend_flyte.custom_types.consume" ': + container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.extend_flyte.custom_types + - task-name + - consume + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.extend_flyte.custom_types.consume + resourceType: TASK + interface: + inputs: + variables: + d: + type: + blob: + dimensionality: MULTIPART + format: binary + outputs: + variables: + o0: + type: + simple: STRING + metadata: + retries: {} + runtime: + flavor: python + type: FLYTE_SDK + version: 0.32.6 + type: python-task + 'resource_type:TASK name:"core.extend_flyte.custom_types.generate" ': + container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.extend_flyte.custom_types + - task-name + - generate + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.extend_flyte.custom_types.generate + resourceType: TASK + interface: + inputs: {} + outputs: + variables: + o0: + type: + blob: + dimensionality: MULTIPART + format: binary + metadata: + retries: {} + runtime: + flavor: python + type: FLYTE_SDK + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/k8s/085_core.flyte_basics.basic_workflow.my_wf_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/k8s/085_core.flyte_basics.basic_workflow.my_wf_2_wf.yaml new file mode 100755 index 0000000000..6d52f77461 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/k8s/085_core.flyte_basics.basic_workflow.my_wf_2_wf.yaml @@ -0,0 +1,257 @@ +apiVersion: flyte.lyft.com/v1alpha1 +executionConfig: + Interruptible: null + MaxParallelism: 0 + RecoveryExecution: {} + TaskPluginImpls: null + TaskResources: + Limits: + CPU: "0" + EphemeralStorage: "0" + GPU: "0" + Memory: "0" + Storage: "0" + Requests: + CPU: "0" + EphemeralStorage: "0" + GPU: "0" + Memory: "0" + Storage: "0" +executionId: {} +inputs: + literals: + a: + scalar: + primitive: + integer: "0" + b: + scalar: + primitive: + stringValue: "" +kind: flyteworkflow +metadata: + creationTimestamp: null + labels: + domain: domain + execution-id: name + project: hello + shard-key: "6" + workflow-name: core-flyte-basics-basic-workflow-my-wf + name: name + namespace: namespace +node-defaults: {} +rawOutputDataConfig: {} +securityContext: {} +spec: + connections: + n0: + - end-node + - n1 + n1: + - end-node + start-node: + - n0 + - n1 + edges: + downstream: + n0: + - end-node + - n1 + n1: + - end-node + start-node: + - n0 + - n1 + upstream: + end-node: + - n0 + - n1 + n0: + - start-node + n1: + - n0 + - start-node + id: ::core.flyte_basics.basic_workflow.my_wf + nodes: + end-node: + id: end-node + inputBindings: + - binding: + promise: + nodeId: n0 + var: t1_int_output + var: o0 + - binding: + promise: + nodeId: n1 + var: o0 + var: o1 + kind: end + resources: {} + n0: + id: n0 + inputBindings: + - binding: + promise: + nodeId: start-node + var: a + var: a + kind: task + name: t1 + resources: {} + retry: + minAttempts: 1 + task: 'resource_type:TASK name:"core.flyte_basics.basic_workflow.t1" ' + n1: + id: n1 + inputBindings: + - binding: + promise: + nodeId: n0 + var: c + var: a + - binding: + promise: + nodeId: start-node + var: b + var: b + kind: task + name: t2 + resources: {} + retry: + minAttempts: 1 + task: 'resource_type:TASK name:"core.flyte_basics.basic_workflow.t2" ' + start-node: + id: start-node + kind: start + resources: {} + outputBindings: + - binding: + promise: + nodeId: n0 + var: t1_int_output + var: o0 + - binding: + promise: + nodeId: n1 + var: o0 + var: o1 + outputs: + variables: + o0: + type: + simple: INTEGER + o1: + type: + simple: STRING +status: + phase: 0 +tasks: + 'resource_type:TASK name:"core.flyte_basics.basic_workflow.t1" ': + container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.basic_workflow + - task-name + - t1 + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.flyte_basics.basic_workflow.t1 + resourceType: TASK + interface: + inputs: + variables: + a: + type: + simple: INTEGER + outputs: + variables: + c: + type: + simple: STRING + t1_int_output: + type: + simple: INTEGER + metadata: + retries: {} + runtime: + flavor: python + type: FLYTE_SDK + version: 0.32.6 + type: python-task + 'resource_type:TASK name:"core.flyte_basics.basic_workflow.t2" ': + container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.basic_workflow + - task-name + - t2 + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.flyte_basics.basic_workflow.t2 + resourceType: TASK + interface: + inputs: + variables: + a: + type: + simple: STRING + b: + type: + simple: STRING + outputs: + variables: + o0: + type: + simple: STRING + metadata: + retries: {} + runtime: + flavor: python + type: FLYTE_SDK + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/k8s/089_core.flyte_basics.decorating_tasks.wf_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/k8s/089_core.flyte_basics.decorating_tasks.wf_2_wf.yaml new file mode 100755 index 0000000000..3920835e6c --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/k8s/089_core.flyte_basics.decorating_tasks.wf_2_wf.yaml @@ -0,0 +1,223 @@ +apiVersion: flyte.lyft.com/v1alpha1 +executionConfig: + Interruptible: null + MaxParallelism: 0 + RecoveryExecution: {} + TaskPluginImpls: null + TaskResources: + Limits: + CPU: "0" + EphemeralStorage: "0" + GPU: "0" + Memory: "0" + Storage: "0" + Requests: + CPU: "0" + EphemeralStorage: "0" + GPU: "0" + Memory: "0" + Storage: "0" +executionId: {} +inputs: + literals: + x: + scalar: + primitive: + integer: "0" +kind: flyteworkflow +metadata: + creationTimestamp: null + labels: + domain: domain + execution-id: name + project: hello + shard-key: "6" + workflow-name: core-flyte-basics-decorating-tasks-wf + name: name + namespace: namespace +node-defaults: {} +rawOutputDataConfig: {} +securityContext: {} +spec: + connections: + n0: + - n1 + n1: + - end-node + start-node: + - n0 + edges: + downstream: + n0: + - n1 + n1: + - end-node + start-node: + - n0 + upstream: + end-node: + - n1 + n0: + - start-node + n1: + - n0 + id: ::core.flyte_basics.decorating_tasks.wf + nodes: + end-node: + id: end-node + inputBindings: + - binding: + promise: + nodeId: n1 + var: o0 + var: o0 + kind: end + resources: {} + n0: + id: n0 + inputBindings: + - binding: + promise: + nodeId: start-node + var: x + var: x + kind: task + name: t1 + resources: {} + retry: + minAttempts: 1 + task: 'resource_type:TASK name:"core.flyte_basics.decorating_tasks.t1" ' + n1: + id: n1 + inputBindings: + - binding: + promise: + nodeId: n0 + var: o0 + var: x + kind: task + name: t2 + resources: {} + retry: + minAttempts: 1 + task: 'resource_type:TASK name:"core.flyte_basics.decorating_tasks.t2" ' + start-node: + id: start-node + kind: start + resources: {} + outputBindings: + - binding: + promise: + nodeId: n1 + var: o0 + var: o0 + outputs: + variables: + o0: + type: + simple: INTEGER +status: + phase: 0 +tasks: + 'resource_type:TASK name:"core.flyte_basics.decorating_tasks.t1" ': + container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.decorating_tasks + - task-name + - t1 + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.flyte_basics.decorating_tasks.t1 + resourceType: TASK + interface: + inputs: + variables: + x: + type: + simple: INTEGER + outputs: + variables: + o0: + type: + simple: INTEGER + metadata: + retries: {} + runtime: + flavor: python + type: FLYTE_SDK + version: 0.32.6 + type: python-task + 'resource_type:TASK name:"core.flyte_basics.decorating_tasks.t2" ': + container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.decorating_tasks + - task-name + - t2 + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.flyte_basics.decorating_tasks.t2 + resourceType: TASK + interface: + inputs: + variables: + x: + type: + simple: INTEGER + outputs: + variables: + o0: + type: + simple: INTEGER + metadata: + retries: {} + runtime: + flavor: python + type: FLYTE_SDK + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/k8s/095_core.flyte_basics.decorating_workflows.wf_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/k8s/095_core.flyte_basics.decorating_workflows.wf_2_wf.yaml new file mode 100755 index 0000000000..f94c0ea9ac --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/k8s/095_core.flyte_basics.decorating_workflows.wf_2_wf.yaml @@ -0,0 +1,343 @@ +apiVersion: flyte.lyft.com/v1alpha1 +executionConfig: + Interruptible: null + MaxParallelism: 0 + RecoveryExecution: {} + TaskPluginImpls: null + TaskResources: + Limits: + CPU: "0" + EphemeralStorage: "0" + GPU: "0" + Memory: "0" + Storage: "0" + Requests: + CPU: "0" + EphemeralStorage: "0" + GPU: "0" + Memory: "0" + Storage: "0" +executionId: {} +inputs: + literals: + x: + scalar: + primitive: + floatValue: 0 +kind: flyteworkflow +metadata: + creationTimestamp: null + labels: + domain: domain + execution-id: name + project: hello + shard-key: "6" + workflow-name: core-flyte-basics-decorating-workflows-wf + name: name + namespace: namespace +node-defaults: {} +rawOutputDataConfig: {} +securityContext: {} +spec: + connections: + n0: + - n1 + n1: + - n2 + n2: + - end-node + - n3 + n3: + - end-node + start-node: + - n0 + - n1 + edges: + downstream: + n0: + - n1 + n1: + - n2 + n2: + - end-node + - n3 + n3: + - end-node + start-node: + - n0 + - n1 + upstream: + end-node: + - n2 + - n3 + n0: + - start-node + n1: + - n0 + - start-node + n2: + - n1 + n3: + - n2 + id: ::core.flyte_basics.decorating_workflows.wf + nodes: + end-node: + id: end-node + inputBindings: + - binding: + promise: + nodeId: n2 + var: o0 + var: o0 + kind: end + resources: {} + n0: + id: n0 + kind: task + name: setup + resources: {} + retry: + minAttempts: 1 + task: 'resource_type:TASK name:"core.flyte_basics.decorating_workflows.setup" ' + n1: + id: n1 + inputBindings: + - binding: + promise: + nodeId: start-node + var: x + var: x + kind: task + name: t1 + resources: {} + retry: + minAttempts: 1 + task: 'resource_type:TASK name:"core.flyte_basics.decorating_workflows.t1" ' + n2: + id: n2 + inputBindings: + - binding: + promise: + nodeId: n1 + var: o0 + var: x + kind: task + name: t2 + resources: {} + retry: + minAttempts: 1 + task: 'resource_type:TASK name:"core.flyte_basics.decorating_workflows.t2" ' + n3: + id: n3 + kind: task + name: teardown + resources: {} + retry: + minAttempts: 1 + task: 'resource_type:TASK name:"core.flyte_basics.decorating_workflows.teardown" ' + start-node: + id: start-node + kind: start + resources: {} + outputBindings: + - binding: + promise: + nodeId: n2 + var: o0 + var: o0 + outputs: + variables: + o0: + type: + simple: FLOAT +status: + phase: 0 +tasks: + 'resource_type:TASK name:"core.flyte_basics.decorating_workflows.setup" ': + container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.decorating_workflows + - task-name + - setup + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.flyte_basics.decorating_workflows.setup + resourceType: TASK + interface: + inputs: {} + outputs: {} + metadata: + retries: {} + runtime: + flavor: python + type: FLYTE_SDK + version: 0.32.6 + type: python-task + 'resource_type:TASK name:"core.flyte_basics.decorating_workflows.t1" ': + container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.decorating_workflows + - task-name + - t1 + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.flyte_basics.decorating_workflows.t1 + resourceType: TASK + interface: + inputs: + variables: + x: + type: + simple: FLOAT + outputs: + variables: + o0: + type: + simple: FLOAT + metadata: + retries: {} + runtime: + flavor: python + type: FLYTE_SDK + version: 0.32.6 + type: python-task + 'resource_type:TASK name:"core.flyte_basics.decorating_workflows.t2" ': + container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.decorating_workflows + - task-name + - t2 + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.flyte_basics.decorating_workflows.t2 + resourceType: TASK + interface: + inputs: + variables: + x: + type: + simple: FLOAT + outputs: + variables: + o0: + type: + simple: FLOAT + metadata: + retries: {} + runtime: + flavor: python + type: FLYTE_SDK + version: 0.32.6 + type: python-task + 'resource_type:TASK name:"core.flyte_basics.decorating_workflows.teardown" ': + container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.decorating_workflows + - task-name + - teardown + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.flyte_basics.decorating_workflows.teardown + resourceType: TASK + interface: + inputs: {} + outputs: {} + metadata: + retries: {} + runtime: + flavor: python + type: FLYTE_SDK + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/k8s/098_core.flyte_basics.documented_workflow.sphinx_docstring_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/k8s/098_core.flyte_basics.documented_workflow.sphinx_docstring_2_wf.yaml new file mode 100755 index 0000000000..b71c5a23b6 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/k8s/098_core.flyte_basics.documented_workflow.sphinx_docstring_2_wf.yaml @@ -0,0 +1,168 @@ +apiVersion: flyte.lyft.com/v1alpha1 +executionConfig: + Interruptible: null + MaxParallelism: 0 + RecoveryExecution: {} + TaskPluginImpls: null + TaskResources: + Limits: + CPU: "0" + EphemeralStorage: "0" + GPU: "0" + Memory: "0" + Storage: "0" + Requests: + CPU: "0" + EphemeralStorage: "0" + GPU: "0" + Memory: "0" + Storage: "0" +executionId: {} +inputs: + literals: + data: + scalar: + generic: {} + df: + scalar: + structuredDataset: + metadata: + structuredDatasetType: + format: parquet +kind: flyteworkflow +metadata: + creationTimestamp: null + labels: + domain: domain + execution-id: name + project: hello + shard-key: "6" + workflow-name: core-flyte-basics-documented-workflow-sphinx-docstring + name: name + namespace: namespace +node-defaults: {} +rawOutputDataConfig: {} +securityContext: {} +spec: + connections: + n0: + - end-node + start-node: + - n0 + edges: + downstream: + n0: + - end-node + start-node: + - n0 + upstream: + end-node: + - n0 + n0: + - start-node + id: ::core.flyte_basics.documented_workflow.sphinx_docstring + nodes: + end-node: + id: end-node + inputBindings: + - binding: + promise: + nodeId: n0 + var: o0 + var: o0 + kind: end + resources: {} + n0: + id: n0 + inputBindings: + - binding: + promise: + nodeId: start-node + var: data + var: data + - binding: + promise: + nodeId: start-node + var: df + var: df + kind: task + name: add_data + resources: {} + retry: + minAttempts: 1 + task: 'resource_type:TASK name:"core.flyte_basics.documented_workflow.add_data" ' + start-node: + id: start-node + kind: start + resources: {} + outputBindings: + - binding: + promise: + nodeId: n0 + var: o0 + var: o0 + outputs: + variables: + o0: + type: + structuredDatasetType: + format: parquet +status: + phase: 0 +tasks: + 'resource_type:TASK name:"core.flyte_basics.documented_workflow.add_data" ': + container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.documented_workflow + - task-name + - add_data + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.flyte_basics.documented_workflow.add_data + resourceType: TASK + interface: + inputs: + variables: + data: + type: + simple: STRUCT + df: + type: + structuredDatasetType: + format: parquet + outputs: + variables: + o0: + type: + structuredDatasetType: + format: parquet + metadata: + retries: {} + runtime: + flavor: python + type: FLYTE_SDK + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/k8s/100_core.flyte_basics.documented_workflow.numpy_docstring_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/k8s/100_core.flyte_basics.documented_workflow.numpy_docstring_2_wf.yaml new file mode 100755 index 0000000000..614fe07435 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/k8s/100_core.flyte_basics.documented_workflow.numpy_docstring_2_wf.yaml @@ -0,0 +1,168 @@ +apiVersion: flyte.lyft.com/v1alpha1 +executionConfig: + Interruptible: null + MaxParallelism: 0 + RecoveryExecution: {} + TaskPluginImpls: null + TaskResources: + Limits: + CPU: "0" + EphemeralStorage: "0" + GPU: "0" + Memory: "0" + Storage: "0" + Requests: + CPU: "0" + EphemeralStorage: "0" + GPU: "0" + Memory: "0" + Storage: "0" +executionId: {} +inputs: + literals: + data: + scalar: + generic: {} + df: + scalar: + structuredDataset: + metadata: + structuredDatasetType: + format: parquet +kind: flyteworkflow +metadata: + creationTimestamp: null + labels: + domain: domain + execution-id: name + project: hello + shard-key: "6" + workflow-name: core-flyte-basics-documented-workflow-numpy-docstring + name: name + namespace: namespace +node-defaults: {} +rawOutputDataConfig: {} +securityContext: {} +spec: + connections: + n0: + - end-node + start-node: + - n0 + edges: + downstream: + n0: + - end-node + start-node: + - n0 + upstream: + end-node: + - n0 + n0: + - start-node + id: ::core.flyte_basics.documented_workflow.numpy_docstring + nodes: + end-node: + id: end-node + inputBindings: + - binding: + promise: + nodeId: n0 + var: o0 + var: o0 + kind: end + resources: {} + n0: + id: n0 + inputBindings: + - binding: + promise: + nodeId: start-node + var: data + var: data + - binding: + promise: + nodeId: start-node + var: df + var: df + kind: task + name: add_data + resources: {} + retry: + minAttempts: 1 + task: 'resource_type:TASK name:"core.flyte_basics.documented_workflow.add_data" ' + start-node: + id: start-node + kind: start + resources: {} + outputBindings: + - binding: + promise: + nodeId: n0 + var: o0 + var: o0 + outputs: + variables: + o0: + type: + structuredDatasetType: + format: parquet +status: + phase: 0 +tasks: + 'resource_type:TASK name:"core.flyte_basics.documented_workflow.add_data" ': + container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.documented_workflow + - task-name + - add_data + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.flyte_basics.documented_workflow.add_data + resourceType: TASK + interface: + inputs: + variables: + data: + type: + simple: STRUCT + df: + type: + structuredDatasetType: + format: parquet + outputs: + variables: + o0: + type: + structuredDatasetType: + format: parquet + metadata: + retries: {} + runtime: + flavor: python + type: FLYTE_SDK + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/k8s/102_core.flyte_basics.documented_workflow.google_docstring_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/k8s/102_core.flyte_basics.documented_workflow.google_docstring_2_wf.yaml new file mode 100755 index 0000000000..4ed4f0e682 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/k8s/102_core.flyte_basics.documented_workflow.google_docstring_2_wf.yaml @@ -0,0 +1,168 @@ +apiVersion: flyte.lyft.com/v1alpha1 +executionConfig: + Interruptible: null + MaxParallelism: 0 + RecoveryExecution: {} + TaskPluginImpls: null + TaskResources: + Limits: + CPU: "0" + EphemeralStorage: "0" + GPU: "0" + Memory: "0" + Storage: "0" + Requests: + CPU: "0" + EphemeralStorage: "0" + GPU: "0" + Memory: "0" + Storage: "0" +executionId: {} +inputs: + literals: + data: + scalar: + generic: {} + df: + scalar: + structuredDataset: + metadata: + structuredDatasetType: + format: parquet +kind: flyteworkflow +metadata: + creationTimestamp: null + labels: + domain: domain + execution-id: name + project: hello + shard-key: "6" + workflow-name: core-flyte-basics-documented-workflow-google-docstring + name: name + namespace: namespace +node-defaults: {} +rawOutputDataConfig: {} +securityContext: {} +spec: + connections: + n0: + - end-node + start-node: + - n0 + edges: + downstream: + n0: + - end-node + start-node: + - n0 + upstream: + end-node: + - n0 + n0: + - start-node + id: ::core.flyte_basics.documented_workflow.google_docstring + nodes: + end-node: + id: end-node + inputBindings: + - binding: + promise: + nodeId: n0 + var: o0 + var: o0 + kind: end + resources: {} + n0: + id: n0 + inputBindings: + - binding: + promise: + nodeId: start-node + var: data + var: data + - binding: + promise: + nodeId: start-node + var: df + var: df + kind: task + name: add_data + resources: {} + retry: + minAttempts: 1 + task: 'resource_type:TASK name:"core.flyte_basics.documented_workflow.add_data" ' + start-node: + id: start-node + kind: start + resources: {} + outputBindings: + - binding: + promise: + nodeId: n0 + var: o0 + var: o0 + outputs: + variables: + o0: + type: + structuredDatasetType: + format: parquet +status: + phase: 0 +tasks: + 'resource_type:TASK name:"core.flyte_basics.documented_workflow.add_data" ': + container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.documented_workflow + - task-name + - add_data + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.flyte_basics.documented_workflow.add_data + resourceType: TASK + interface: + inputs: + variables: + data: + type: + simple: STRUCT + df: + type: + structuredDatasetType: + format: parquet + outputs: + variables: + o0: + type: + structuredDatasetType: + format: parquet + metadata: + retries: {} + runtime: + flavor: python + type: FLYTE_SDK + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/k8s/105_core.flyte_basics.files.normalize_csv_file_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/k8s/105_core.flyte_basics.files.normalize_csv_file_2_wf.yaml new file mode 100755 index 0000000000..eb6d09c687 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/k8s/105_core.flyte_basics.files.normalize_csv_file_2_wf.yaml @@ -0,0 +1,196 @@ +apiVersion: flyte.lyft.com/v1alpha1 +executionConfig: + Interruptible: null + MaxParallelism: 0 + RecoveryExecution: {} + TaskPluginImpls: null + TaskResources: + Limits: + CPU: "0" + EphemeralStorage: "0" + GPU: "0" + Memory: "0" + Storage: "0" + Requests: + CPU: "0" + EphemeralStorage: "0" + GPU: "0" + Memory: "0" + Storage: "0" +executionId: {} +inputs: + literals: + column_names: + collection: + literals: + - scalar: + primitive: + stringValue: "" + columns_to_normalize: + collection: + literals: + - scalar: + primitive: + stringValue: "" + csv_url: + scalar: + blob: + metadata: + type: {} + uri: /tmp/somepath + output_location: + scalar: + primitive: + stringValue: "" +kind: flyteworkflow +metadata: + creationTimestamp: null + labels: + domain: domain + execution-id: name + project: hello + shard-key: "6" + workflow-name: core-flyte-basics-files-normalize-csv-file + name: name + namespace: namespace +node-defaults: {} +rawOutputDataConfig: {} +securityContext: {} +spec: + connections: + n0: + - end-node + start-node: + - n0 + edges: + downstream: + n0: + - end-node + start-node: + - n0 + upstream: + end-node: + - n0 + n0: + - start-node + id: ::core.flyte_basics.files.normalize_csv_file + nodes: + end-node: + id: end-node + inputBindings: + - binding: + promise: + nodeId: n0 + var: o0 + var: o0 + kind: end + resources: {} + n0: + id: n0 + inputBindings: + - binding: + promise: + nodeId: start-node + var: column_names + var: column_names + - binding: + promise: + nodeId: start-node + var: columns_to_normalize + var: columns_to_normalize + - binding: + promise: + nodeId: start-node + var: csv_url + var: csv_url + - binding: + promise: + nodeId: start-node + var: output_location + var: output_location + kind: task + name: normalize_columns + resources: {} + retry: + minAttempts: 1 + task: 'resource_type:TASK name:"core.flyte_basics.files.normalize_columns" ' + start-node: + id: start-node + kind: start + resources: {} + outputBindings: + - binding: + promise: + nodeId: n0 + var: o0 + var: o0 + outputs: + variables: + o0: + type: + blob: {} +status: + phase: 0 +tasks: + 'resource_type:TASK name:"core.flyte_basics.files.normalize_columns" ': + container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.files + - task-name + - normalize_columns + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.flyte_basics.files.normalize_columns + resourceType: TASK + interface: + inputs: + variables: + column_names: + type: + collectionType: + simple: STRING + columns_to_normalize: + type: + collectionType: + simple: STRING + csv_url: + type: + blob: {} + output_location: + type: + simple: STRING + outputs: + variables: + o0: + type: + blob: {} + metadata: + retries: {} + runtime: + flavor: python + type: FLYTE_SDK + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/k8s/109_core.flyte_basics.folders.download_and_normalize_csv_files_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/k8s/109_core.flyte_basics.folders.download_and_normalize_csv_files_2_wf.yaml new file mode 100755 index 0000000000..8603a881d8 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/k8s/109_core.flyte_basics.folders.download_and_normalize_csv_files_2_wf.yaml @@ -0,0 +1,269 @@ +apiVersion: flyte.lyft.com/v1alpha1 +executionConfig: + Interruptible: null + MaxParallelism: 0 + RecoveryExecution: {} + TaskPluginImpls: null + TaskResources: + Limits: + CPU: "0" + EphemeralStorage: "0" + GPU: "0" + Memory: "0" + Storage: "0" + Requests: + CPU: "0" + EphemeralStorage: "0" + GPU: "0" + Memory: "0" + Storage: "0" +executionId: {} +inputs: + literals: + columns_metadata: + collection: + literals: + - collection: + literals: + - scalar: + primitive: + stringValue: "" + columns_to_normalize_metadata: + collection: + literals: + - collection: + literals: + - scalar: + primitive: + stringValue: "" + csv_urls: + collection: + literals: + - scalar: + primitive: + stringValue: "" +kind: flyteworkflow +metadata: + creationTimestamp: null + labels: + domain: domain + execution-id: name + project: hello + shard-key: "6" + workflow-name: core-flyte-basics-folders-download-and-normalize-csv-files + name: name + namespace: namespace +node-defaults: {} +rawOutputDataConfig: {} +securityContext: {} +spec: + connections: + n0: + - n1 + n1: + - end-node + start-node: + - n0 + - n1 + edges: + downstream: + n0: + - n1 + n1: + - end-node + start-node: + - n0 + - n1 + upstream: + end-node: + - n1 + n0: + - start-node + n1: + - n0 + - start-node + id: ::core.flyte_basics.folders.download_and_normalize_csv_files + nodes: + end-node: + id: end-node + inputBindings: + - binding: + promise: + nodeId: n1 + var: o0 + var: o0 + kind: end + resources: {} + n0: + id: n0 + inputBindings: + - binding: + promise: + nodeId: start-node + var: csv_urls + var: csv_urls + kind: task + name: download_files + resources: {} + retry: + minAttempts: 1 + task: 'resource_type:TASK name:"core.flyte_basics.folders.download_files" ' + n1: + id: n1 + inputBindings: + - binding: + promise: + nodeId: start-node + var: columns_metadata + var: columns_metadata + - binding: + promise: + nodeId: start-node + var: columns_to_normalize_metadata + var: columns_to_normalize_metadata + - binding: + promise: + nodeId: n0 + var: o0 + var: csv_files_dir + kind: task + name: normalize_all_files + resources: {} + retry: + minAttempts: 1 + task: 'resource_type:TASK name:"core.flyte_basics.folders.normalize_all_files" ' + start-node: + id: start-node + kind: start + resources: {} + outputBindings: + - binding: + promise: + nodeId: n1 + var: o0 + var: o0 + outputs: + variables: + o0: + type: + blob: + dimensionality: MULTIPART +status: + phase: 0 +tasks: + 'resource_type:TASK name:"core.flyte_basics.folders.download_files" ': + container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.folders + - task-name + - download_files + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.flyte_basics.folders.download_files + resourceType: TASK + interface: + inputs: + variables: + csv_urls: + type: + collectionType: + simple: STRING + outputs: + variables: + o0: + type: + blob: + dimensionality: MULTIPART + metadata: + retries: {} + runtime: + flavor: python + type: FLYTE_SDK + version: 0.32.6 + type: python-task + 'resource_type:TASK name:"core.flyte_basics.folders.normalize_all_files" ': + container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.folders + - task-name + - normalize_all_files + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.flyte_basics.folders.normalize_all_files + resourceType: TASK + interface: + inputs: + variables: + columns_metadata: + type: + collectionType: + collectionType: + simple: STRING + columns_to_normalize_metadata: + type: + collectionType: + collectionType: + simple: STRING + csv_files_dir: + type: + blob: + dimensionality: MULTIPART + outputs: + variables: + o0: + type: + blob: + dimensionality: MULTIPART + metadata: + retries: {} + runtime: + flavor: python + type: FLYTE_SDK + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/k8s/112_core.flyte_basics.hello_world.my_wf_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/k8s/112_core.flyte_basics.hello_world.my_wf_2_wf.yaml new file mode 100755 index 0000000000..57fbfabf7b --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/k8s/112_core.flyte_basics.hello_world.my_wf_2_wf.yaml @@ -0,0 +1,137 @@ +apiVersion: flyte.lyft.com/v1alpha1 +executionConfig: + Interruptible: null + MaxParallelism: 0 + RecoveryExecution: {} + TaskPluginImpls: null + TaskResources: + Limits: + CPU: "0" + EphemeralStorage: "0" + GPU: "0" + Memory: "0" + Storage: "0" + Requests: + CPU: "0" + EphemeralStorage: "0" + GPU: "0" + Memory: "0" + Storage: "0" +executionId: {} +inputs: {} +kind: flyteworkflow +metadata: + creationTimestamp: null + labels: + domain: domain + execution-id: name + project: hello + shard-key: "6" + workflow-name: core-flyte-basics-hello-world-my-wf + name: name + namespace: namespace +node-defaults: {} +rawOutputDataConfig: {} +securityContext: {} +spec: + connections: + n0: + - end-node + start-node: + - n0 + edges: + downstream: + n0: + - end-node + start-node: + - n0 + upstream: + end-node: + - n0 + n0: + - start-node + id: ::core.flyte_basics.hello_world.my_wf + nodes: + end-node: + id: end-node + inputBindings: + - binding: + promise: + nodeId: n0 + var: o0 + var: o0 + kind: end + resources: {} + n0: + id: n0 + kind: task + name: say_hello + resources: {} + retry: + minAttempts: 1 + task: 'resource_type:TASK name:"core.flyte_basics.hello_world.say_hello" ' + start-node: + id: start-node + kind: start + resources: {} + outputBindings: + - binding: + promise: + nodeId: n0 + var: o0 + var: o0 + outputs: + variables: + o0: + type: + simple: STRING +status: + phase: 0 +tasks: + 'resource_type:TASK name:"core.flyte_basics.hello_world.say_hello" ': + container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.hello_world + - task-name + - say_hello + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.flyte_basics.hello_world.say_hello + resourceType: TASK + interface: + inputs: {} + outputs: + variables: + o0: + type: + simple: STRING + metadata: + retries: {} + runtime: + flavor: python + type: FLYTE_SDK + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/k8s/117_my.imperative.workflow.example_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/k8s/117_my.imperative.workflow.example_2_wf.yaml new file mode 100755 index 0000000000..7bf3321b7e --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/k8s/117_my.imperative.workflow.example_2_wf.yaml @@ -0,0 +1,320 @@ +apiVersion: flyte.lyft.com/v1alpha1 +executionConfig: + Interruptible: null + MaxParallelism: 0 + RecoveryExecution: {} + TaskPluginImpls: null + TaskResources: + Limits: + CPU: "0" + EphemeralStorage: "0" + GPU: "0" + Memory: "0" + Storage: "0" + Requests: + CPU: "0" + EphemeralStorage: "0" + GPU: "0" + Memory: "0" + Storage: "0" +executionId: {} +inputs: + literals: + in1: + scalar: + primitive: + stringValue: "" + in2: + scalar: + primitive: + stringValue: "" +kind: flyteworkflow +metadata: + creationTimestamp: null + labels: + domain: domain + execution-id: name + project: hello + shard-key: "6" + workflow-name: my-imperative-workflow-example + name: name + namespace: namespace +node-defaults: {} +rawOutputDataConfig: {} +securityContext: {} +spec: + connections: + n0: + - end-node + n1: + - end-node + n2: + - end-node + start-node: + - n0 + - n1 + - n2 + edges: + downstream: + n0: + - end-node + n1: + - end-node + n2: + - end-node + start-node: + - n0 + - n1 + - n2 + upstream: + end-node: + - n0 + - n1 + - n2 + n0: + - start-node + n1: + - start-node + n2: + - start-node + id: ::my.imperative.workflow.example + nodes: + end-node: + id: end-node + inputBindings: + - binding: + promise: + nodeId: n0 + var: o0 + var: output_from_t1 + - binding: + collection: + bindings: + - promise: + nodeId: n0 + var: o0 + - promise: + nodeId: n2 + var: o0 + var: output_list + kind: end + resources: {} + n0: + id: n0 + inputBindings: + - binding: + promise: + nodeId: start-node + var: in1 + var: a + kind: task + name: t1 + resources: {} + retry: + minAttempts: 1 + task: 'resource_type:TASK name:"core.flyte_basics.imperative_wf_style.t1" ' + n1: + id: n1 + kind: task + name: t2 + resources: {} + retry: + minAttempts: 1 + task: 'resource_type:TASK name:"core.flyte_basics.imperative_wf_style.t2" ' + n2: + id: n2 + inputBindings: + - binding: + collection: + bindings: + - promise: + nodeId: start-node + var: in1 + - promise: + nodeId: start-node + var: in2 + var: a + kind: task + name: t3 + resources: {} + retry: + minAttempts: 1 + task: 'resource_type:TASK name:"core.flyte_basics.imperative_wf_style.t3" ' + start-node: + id: start-node + kind: start + resources: {} + outputBindings: + - binding: + promise: + nodeId: n0 + var: o0 + var: output_from_t1 + - binding: + collection: + bindings: + - promise: + nodeId: n0 + var: o0 + - promise: + nodeId: n2 + var: o0 + var: output_list + outputs: + variables: + output_from_t1: + type: + simple: STRING + output_list: + type: + collectionType: + simple: STRING +status: + phase: 0 +tasks: + 'resource_type:TASK name:"core.flyte_basics.imperative_wf_style.t1" ': + container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.imperative_wf_style + - task-name + - t1 + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.flyte_basics.imperative_wf_style.t1 + resourceType: TASK + interface: + inputs: + variables: + a: + type: + simple: STRING + outputs: + variables: + o0: + type: + simple: STRING + metadata: + retries: {} + runtime: + flavor: python + type: FLYTE_SDK + version: 0.32.6 + type: python-task + 'resource_type:TASK name:"core.flyte_basics.imperative_wf_style.t2" ': + container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.imperative_wf_style + - task-name + - t2 + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.flyte_basics.imperative_wf_style.t2 + resourceType: TASK + interface: + inputs: {} + outputs: {} + metadata: + retries: {} + runtime: + flavor: python + type: FLYTE_SDK + version: 0.32.6 + type: python-task + 'resource_type:TASK name:"core.flyte_basics.imperative_wf_style.t3" ': + container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.imperative_wf_style + - task-name + - t3 + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.flyte_basics.imperative_wf_style.t3 + resourceType: TASK + interface: + inputs: + variables: + a: + type: + collectionType: + simple: STRING + outputs: + variables: + o0: + type: + simple: STRING + metadata: + retries: {} + runtime: + flavor: python + type: FLYTE_SDK + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/k8s/120_core.flyte_basics.lp.my_wf_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/k8s/120_core.flyte_basics.lp.my_wf_2_wf.yaml new file mode 100755 index 0000000000..84454acd07 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/k8s/120_core.flyte_basics.lp.my_wf_2_wf.yaml @@ -0,0 +1,152 @@ +apiVersion: flyte.lyft.com/v1alpha1 +executionConfig: + Interruptible: null + MaxParallelism: 0 + RecoveryExecution: {} + TaskPluginImpls: null + TaskResources: + Limits: + CPU: "0" + EphemeralStorage: "0" + GPU: "0" + Memory: "0" + Storage: "0" + Requests: + CPU: "0" + EphemeralStorage: "0" + GPU: "0" + Memory: "0" + Storage: "0" +executionId: {} +inputs: + literals: + val: + scalar: + primitive: + integer: "0" +kind: flyteworkflow +metadata: + creationTimestamp: null + labels: + domain: domain + execution-id: name + project: hello + shard-key: "6" + workflow-name: core-flyte-basics-lp-my-wf + name: name + namespace: namespace +node-defaults: {} +rawOutputDataConfig: {} +securityContext: {} +spec: + connections: + n0: + - end-node + start-node: + - n0 + edges: + downstream: + n0: + - end-node + start-node: + - n0 + upstream: + end-node: + - n0 + n0: + - start-node + id: ::core.flyte_basics.lp.my_wf + nodes: + end-node: + id: end-node + inputBindings: + - binding: + promise: + nodeId: n0 + var: o0 + var: o0 + kind: end + resources: {} + n0: + id: n0 + inputBindings: + - binding: + promise: + nodeId: start-node + var: val + var: val + kind: task + name: square + resources: {} + retry: + minAttempts: 1 + task: 'resource_type:TASK name:"core.flyte_basics.lp.square" ' + start-node: + id: start-node + kind: start + resources: {} + outputBindings: + - binding: + promise: + nodeId: n0 + var: o0 + var: o0 + outputs: + variables: + o0: + type: + simple: INTEGER +status: + phase: 0 +tasks: + 'resource_type:TASK name:"core.flyte_basics.lp.square" ': + container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.lp + - task-name + - square + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.flyte_basics.lp.square + resourceType: TASK + interface: + inputs: + variables: + val: + type: + simple: INTEGER + outputs: + variables: + o0: + type: + simple: INTEGER + metadata: + retries: {} + runtime: + flavor: python + type: FLYTE_SDK + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/k8s/125_core.flyte_basics.lp.go_greet_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/k8s/125_core.flyte_basics.lp.go_greet_2_wf.yaml new file mode 100755 index 0000000000..1e3316bff4 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/k8s/125_core.flyte_basics.lp.go_greet_2_wf.yaml @@ -0,0 +1,176 @@ +apiVersion: flyte.lyft.com/v1alpha1 +executionConfig: + Interruptible: null + MaxParallelism: 0 + RecoveryExecution: {} + TaskPluginImpls: null + TaskResources: + Limits: + CPU: "0" + EphemeralStorage: "0" + GPU: "0" + Memory: "0" + Storage: "0" + Requests: + CPU: "0" + EphemeralStorage: "0" + GPU: "0" + Memory: "0" + Storage: "0" +executionId: {} +inputs: + literals: + am: + scalar: + primitive: + boolean: false + day_of_week: + scalar: + primitive: + stringValue: "" + number: + scalar: + primitive: + integer: "0" +kind: flyteworkflow +metadata: + creationTimestamp: null + labels: + domain: domain + execution-id: name + project: hello + shard-key: "6" + workflow-name: core-flyte-basics-lp-go-greet + name: name + namespace: namespace +node-defaults: {} +rawOutputDataConfig: {} +securityContext: {} +spec: + connections: + n0: + - end-node + start-node: + - n0 + edges: + downstream: + n0: + - end-node + start-node: + - n0 + upstream: + end-node: + - n0 + n0: + - start-node + id: ::core.flyte_basics.lp.go_greet + nodes: + end-node: + id: end-node + inputBindings: + - binding: + promise: + nodeId: n0 + var: o0 + var: o0 + kind: end + resources: {} + n0: + id: n0 + inputBindings: + - binding: + promise: + nodeId: start-node + var: am + var: am + - binding: + promise: + nodeId: start-node + var: day_of_week + var: day_of_week + - binding: + promise: + nodeId: start-node + var: number + var: number + kind: task + name: greet + resources: {} + retry: + minAttempts: 1 + task: 'resource_type:TASK name:"core.flyte_basics.lp.greet" ' + start-node: + id: start-node + kind: start + resources: {} + outputBindings: + - binding: + promise: + nodeId: n0 + var: o0 + var: o0 + outputs: + variables: + o0: + type: + simple: STRING +status: + phase: 0 +tasks: + 'resource_type:TASK name:"core.flyte_basics.lp.greet" ': + container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.lp + - task-name + - greet + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.flyte_basics.lp.greet + resourceType: TASK + interface: + inputs: + variables: + am: + type: + simple: BOOLEAN + day_of_week: + type: + simple: STRING + number: + type: + simple: INTEGER + outputs: + variables: + o0: + type: + simple: STRING + metadata: + retries: {} + runtime: + flavor: python + type: FLYTE_SDK + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/k8s/129_core.flyte_basics.named_outputs.my_wf_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/k8s/129_core.flyte_basics.named_outputs.my_wf_2_wf.yaml new file mode 100755 index 0000000000..cb8d7ba31d --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/k8s/129_core.flyte_basics.named_outputs.my_wf_2_wf.yaml @@ -0,0 +1,167 @@ +apiVersion: flyte.lyft.com/v1alpha1 +executionConfig: + Interruptible: null + MaxParallelism: 0 + RecoveryExecution: {} + TaskPluginImpls: null + TaskResources: + Limits: + CPU: "0" + EphemeralStorage: "0" + GPU: "0" + Memory: "0" + Storage: "0" + Requests: + CPU: "0" + EphemeralStorage: "0" + GPU: "0" + Memory: "0" + Storage: "0" +executionId: {} +inputs: {} +kind: flyteworkflow +metadata: + creationTimestamp: null + labels: + domain: domain + execution-id: name + project: hello + shard-key: "6" + workflow-name: core-flyte-basics-named-outputs-my-wf + name: name + namespace: namespace +node-defaults: {} +rawOutputDataConfig: {} +securityContext: {} +spec: + connections: + n0: + - end-node + n1: + - end-node + start-node: + - n0 + - n1 + edges: + downstream: + n0: + - end-node + n1: + - end-node + start-node: + - n0 + - n1 + upstream: + end-node: + - n0 + - n1 + n0: + - start-node + n1: + - start-node + id: ::core.flyte_basics.named_outputs.my_wf + nodes: + end-node: + id: end-node + inputBindings: + - binding: + promise: + nodeId: n0 + var: greet + var: greet1 + - binding: + promise: + nodeId: n1 + var: greet + var: greet2 + kind: end + resources: {} + n0: + id: n0 + kind: task + name: say_hello + resources: {} + retry: + minAttempts: 1 + task: 'resource_type:TASK name:"core.flyte_basics.named_outputs.say_hello" ' + n1: + id: n1 + kind: task + name: say_hello + resources: {} + retry: + minAttempts: 1 + task: 'resource_type:TASK name:"core.flyte_basics.named_outputs.say_hello" ' + start-node: + id: start-node + kind: start + resources: {} + outputBindings: + - binding: + promise: + nodeId: n0 + var: greet + var: greet1 + - binding: + promise: + nodeId: n1 + var: greet + var: greet2 + outputs: + variables: + greet1: + type: + simple: STRING + greet2: + type: + simple: STRING +status: + phase: 0 +tasks: + 'resource_type:TASK name:"core.flyte_basics.named_outputs.say_hello" ': + container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.named_outputs + - task-name + - say_hello + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.flyte_basics.named_outputs.say_hello + resourceType: TASK + interface: + inputs: {} + outputs: + variables: + greet: + type: + simple: STRING + metadata: + retries: {} + runtime: + flavor: python + type: FLYTE_SDK + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/k8s/140_core.flyte_basics.shell_task.wf_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/k8s/140_core.flyte_basics.shell_task.wf_2_wf.yaml new file mode 100755 index 0000000000..35c747caca --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/k8s/140_core.flyte_basics.shell_task.wf_2_wf.yaml @@ -0,0 +1,386 @@ +apiVersion: flyte.lyft.com/v1alpha1 +executionConfig: + Interruptible: null + MaxParallelism: 0 + RecoveryExecution: {} + TaskPluginImpls: null + TaskResources: + Limits: + CPU: "0" + EphemeralStorage: "0" + GPU: "0" + Memory: "0" + Storage: "0" + Requests: + CPU: "0" + EphemeralStorage: "0" + GPU: "0" + Memory: "0" + Storage: "0" +executionId: {} +inputs: {} +kind: flyteworkflow +metadata: + creationTimestamp: null + labels: + domain: domain + execution-id: name + project: hello + shard-key: "6" + workflow-name: core-flyte-basics-shell-task-wf + name: name + namespace: namespace +node-defaults: {} +rawOutputDataConfig: {} +securityContext: {} +spec: + connections: + n0: + - n1 + - n2 + - n3 + n1: + - n2 + n2: + - n3 + n3: + - end-node + start-node: + - n0 + edges: + downstream: + n0: + - n1 + - n2 + - n3 + n1: + - n2 + n2: + - n3 + n3: + - end-node + start-node: + - n0 + upstream: + end-node: + - n3 + n0: + - start-node + n1: + - n0 + n2: + - n0 + - n1 + n3: + - n0 + - n2 + id: ::core.flyte_basics.shell_task.wf + nodes: + end-node: + id: end-node + inputBindings: + - binding: + promise: + nodeId: n3 + var: k + var: o0 + kind: end + resources: {} + n0: + id: n0 + kind: task + name: create_entities + resources: {} + retry: + minAttempts: 1 + task: 'resource_type:TASK name:"core.flyte_basics.shell_task.create_entities" ' + n1: + id: n1 + inputBindings: + - binding: + promise: + nodeId: n0 + var: o0 + var: x + kind: task + name: task_1 + resources: {} + retry: + minAttempts: 1 + task: 'resource_type:TASK name:"task_1" ' + n2: + id: n2 + inputBindings: + - binding: + promise: + nodeId: n1 + var: i + var: x + - binding: + promise: + nodeId: n0 + var: o1 + var: "y" + kind: task + name: task_2 + resources: {} + retry: + minAttempts: 1 + task: 'resource_type:TASK name:"task_2" ' + n3: + id: n3 + inputBindings: + - binding: + promise: + nodeId: n0 + var: o0 + var: x + - binding: + promise: + nodeId: n0 + var: o1 + var: "y" + - binding: + promise: + nodeId: n2 + var: j + var: z + kind: task + name: task_3 + resources: {} + retry: + minAttempts: 1 + task: 'resource_type:TASK name:"task_3" ' + start-node: + id: start-node + kind: start + resources: {} + outputBindings: + - binding: + promise: + nodeId: n3 + var: k + var: o0 + outputs: + variables: + o0: + type: + blob: {} +status: + phase: 0 +tasks: + 'resource_type:TASK name:"core.flyte_basics.shell_task.create_entities" ': + container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.shell_task + - task-name + - create_entities + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.flyte_basics.shell_task.create_entities + resourceType: TASK + interface: + inputs: {} + outputs: + variables: + o0: + type: + blob: {} + o1: + type: + blob: + dimensionality: MULTIPART + metadata: + retries: {} + runtime: + flavor: python + type: FLYTE_SDK + version: 0.32.6 + type: python-task + 'resource_type:TASK name:"task_1" ': + container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.shell_task + - task-name + - t1 + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: task_1 + resourceType: TASK + interface: + inputs: + variables: + x: + type: + blob: {} + outputs: + variables: + i: + type: + blob: {} + metadata: + retries: {} + runtime: + flavor: python + type: FLYTE_SDK + version: 0.32.6 + type: python-task + 'resource_type:TASK name:"task_2" ': + container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.shell_task + - task-name + - t2 + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: task_2 + resourceType: TASK + interface: + inputs: + variables: + x: + type: + blob: {} + "y": + type: + blob: + dimensionality: MULTIPART + outputs: + variables: + j: + type: + blob: {} + metadata: + retries: {} + runtime: + flavor: python + type: FLYTE_SDK + version: 0.32.6 + type: python-task + 'resource_type:TASK name:"task_3" ': + container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.shell_task + - task-name + - t3 + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: task_3 + resourceType: TASK + interface: + inputs: + variables: + x: + type: + blob: {} + "y": + type: + blob: + dimensionality: MULTIPART + z: + type: + blob: {} + outputs: + variables: + k: + type: + blob: {} + metadata: + retries: {} + runtime: + flavor: python + type: FLYTE_SDK + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/k8s/147_core.flyte_basics.task_cache.cached_dataframe_wf_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/k8s/147_core.flyte_basics.task_cache.cached_dataframe_wf_2_wf.yaml new file mode 100755 index 0000000000..6f40a115fc --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/k8s/147_core.flyte_basics.task_cache.cached_dataframe_wf_2_wf.yaml @@ -0,0 +1,300 @@ +apiVersion: flyte.lyft.com/v1alpha1 +executionConfig: + Interruptible: null + MaxParallelism: 0 + RecoveryExecution: {} + TaskPluginImpls: null + TaskResources: + Limits: + CPU: "0" + EphemeralStorage: "0" + GPU: "0" + Memory: "0" + Storage: "0" + Requests: + CPU: "0" + EphemeralStorage: "0" + GPU: "0" + Memory: "0" + Storage: "0" +executionId: {} +inputs: {} +kind: flyteworkflow +metadata: + creationTimestamp: null + labels: + domain: domain + execution-id: name + project: hello + shard-key: "6" + workflow-name: core-flyte-basics-task-cache-cached-dataframe-wf + name: name + namespace: namespace +node-defaults: {} +rawOutputDataConfig: {} +securityContext: {} +spec: + connections: + n0: + - n1 + - n2 + n1: + - n2 + - n3 + n2: + - n3 + n3: + - end-node + start-node: + - n0 + edges: + downstream: + n0: + - n1 + - n2 + n1: + - n2 + - n3 + n2: + - n3 + n3: + - end-node + start-node: + - n0 + upstream: + end-node: + - n3 + n0: + - start-node + n1: + - n0 + n2: + - n0 + - n1 + n3: + - n1 + - n2 + id: ::core.flyte_basics.task_cache.cached_dataframe_wf + nodes: + end-node: + id: end-node + kind: end + resources: {} + n0: + id: n0 + kind: task + name: uncached_data_reading_task + resources: {} + retry: + minAttempts: 1 + task: 'resource_type:TASK name:"core.flyte_basics.task_cache.uncached_data_reading_task" ' + n1: + id: n1 + inputBindings: + - binding: + promise: + nodeId: n0 + var: o0 + var: df + kind: task + name: cached_data_processing_task + resources: {} + retry: + minAttempts: 1 + task: 'resource_type:TASK name:"core.flyte_basics.task_cache.cached_data_processing_task" ' + n2: + id: n2 + inputBindings: + - binding: + promise: + nodeId: n0 + var: o0 + var: df + kind: task + name: cached_data_processing_task + resources: {} + retry: + minAttempts: 1 + task: 'resource_type:TASK name:"core.flyte_basics.task_cache.cached_data_processing_task" ' + n3: + id: n3 + inputBindings: + - binding: + promise: + nodeId: n1 + var: o0 + var: df1 + - binding: + promise: + nodeId: n2 + var: o0 + var: df2 + kind: task + name: compare_dataframes + resources: {} + retry: + minAttempts: 1 + task: 'resource_type:TASK name:"core.flyte_basics.task_cache.compare_dataframes" ' + start-node: + id: start-node + kind: start + resources: {} + outputs: {} +status: + phase: 0 +tasks: + 'resource_type:TASK name:"core.flyte_basics.task_cache.cached_data_processing_task" ': + container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.task_cache + - task-name + - cached_data_processing_task + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.flyte_basics.task_cache.cached_data_processing_task + resourceType: TASK + interface: + inputs: + variables: + df: + type: + structuredDatasetType: + format: parquet + outputs: + variables: + o0: + type: + structuredDatasetType: + format: parquet + metadata: + discoverable: true + discoveryVersion: "1.0" + retries: {} + runtime: + flavor: python + type: FLYTE_SDK + version: 0.32.6 + type: python-task + 'resource_type:TASK name:"core.flyte_basics.task_cache.compare_dataframes" ': + container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.task_cache + - task-name + - compare_dataframes + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.flyte_basics.task_cache.compare_dataframes + resourceType: TASK + interface: + inputs: + variables: + df1: + type: + structuredDatasetType: + format: parquet + df2: + type: + structuredDatasetType: + format: parquet + outputs: {} + metadata: + retries: {} + runtime: + flavor: python + type: FLYTE_SDK + version: 0.32.6 + type: python-task + 'resource_type:TASK name:"core.flyte_basics.task_cache.uncached_data_reading_task" ': + container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.flyte_basics.task_cache + - task-name + - uncached_data_reading_task + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.flyte_basics.task_cache.uncached_data_reading_task + resourceType: TASK + interface: + inputs: {} + outputs: + variables: + o0: + type: + structuredDatasetType: + format: parquet + metadata: + retries: {} + runtime: + flavor: python + type: FLYTE_SDK + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/k8s/151_core.scheduled_workflows.lp_schedules.date_formatter_wf_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/k8s/151_core.scheduled_workflows.lp_schedules.date_formatter_wf_2_wf.yaml new file mode 100755 index 0000000000..27376484c9 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/k8s/151_core.scheduled_workflows.lp_schedules.date_formatter_wf_2_wf.yaml @@ -0,0 +1,136 @@ +apiVersion: flyte.lyft.com/v1alpha1 +executionConfig: + Interruptible: null + MaxParallelism: 0 + RecoveryExecution: {} + TaskPluginImpls: null + TaskResources: + Limits: + CPU: "0" + EphemeralStorage: "0" + GPU: "0" + Memory: "0" + Storage: "0" + Requests: + CPU: "0" + EphemeralStorage: "0" + GPU: "0" + Memory: "0" + Storage: "0" +executionId: {} +inputs: + literals: + kickoff_time: + scalar: + primitive: + datetime: "1970-01-01T00:00:00.000010Z" +kind: flyteworkflow +metadata: + creationTimestamp: null + labels: + domain: domain + execution-id: name + project: hello + shard-key: "6" + workflow-name: core-scheduled-workflows-lp-schedules-date-formatter-wf + name: name + namespace: namespace +node-defaults: {} +rawOutputDataConfig: {} +securityContext: {} +spec: + connections: + n0: + - end-node + start-node: + - n0 + edges: + downstream: + n0: + - end-node + start-node: + - n0 + upstream: + end-node: + - n0 + n0: + - start-node + id: ::core.scheduled_workflows.lp_schedules.date_formatter_wf + nodes: + end-node: + id: end-node + kind: end + resources: {} + n0: + id: n0 + inputBindings: + - binding: + promise: + nodeId: start-node + var: kickoff_time + var: run_date + kind: task + name: format_date + resources: {} + retry: + minAttempts: 1 + task: 'resource_type:TASK name:"core.scheduled_workflows.lp_schedules.format_date" ' + start-node: + id: start-node + kind: start + resources: {} + outputs: {} +status: + phase: 0 +tasks: + 'resource_type:TASK name:"core.scheduled_workflows.lp_schedules.format_date" ': + container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.scheduled_workflows.lp_schedules + - task-name + - format_date + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.scheduled_workflows.lp_schedules.format_date + resourceType: TASK + interface: + inputs: + variables: + run_date: + type: + simple: DATETIME + outputs: + variables: + o0: + type: + simple: STRING + metadata: + retries: {} + runtime: + flavor: python + type: FLYTE_SDK + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/k8s/155_core.scheduled_workflows.lp_schedules.positive_wf_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/k8s/155_core.scheduled_workflows.lp_schedules.positive_wf_2_wf.yaml new file mode 100755 index 0000000000..8aca784d78 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/k8s/155_core.scheduled_workflows.lp_schedules.positive_wf_2_wf.yaml @@ -0,0 +1,136 @@ +apiVersion: flyte.lyft.com/v1alpha1 +executionConfig: + Interruptible: null + MaxParallelism: 0 + RecoveryExecution: {} + TaskPluginImpls: null + TaskResources: + Limits: + CPU: "0" + EphemeralStorage: "0" + GPU: "0" + Memory: "0" + Storage: "0" + Requests: + CPU: "0" + EphemeralStorage: "0" + GPU: "0" + Memory: "0" + Storage: "0" +executionId: {} +inputs: + literals: + name: + scalar: + primitive: + stringValue: "" +kind: flyteworkflow +metadata: + creationTimestamp: null + labels: + domain: domain + execution-id: name + project: hello + shard-key: "6" + workflow-name: core-scheduled-workflows-lp-schedules-positive-wf + name: name + namespace: namespace +node-defaults: {} +rawOutputDataConfig: {} +securityContext: {} +spec: + connections: + n0: + - end-node + start-node: + - n0 + edges: + downstream: + n0: + - end-node + start-node: + - n0 + upstream: + end-node: + - n0 + n0: + - start-node + id: ::core.scheduled_workflows.lp_schedules.positive_wf + nodes: + end-node: + id: end-node + kind: end + resources: {} + n0: + id: n0 + inputBindings: + - binding: + promise: + nodeId: start-node + var: name + var: name + kind: task + name: be_positive + resources: {} + retry: + minAttempts: 1 + task: 'resource_type:TASK name:"core.scheduled_workflows.lp_schedules.be_positive" ' + start-node: + id: start-node + kind: start + resources: {} + outputs: {} +status: + phase: 0 +tasks: + 'resource_type:TASK name:"core.scheduled_workflows.lp_schedules.be_positive" ': + container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.scheduled_workflows.lp_schedules + - task-name + - be_positive + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.scheduled_workflows.lp_schedules.be_positive + resourceType: TASK + interface: + inputs: + variables: + name: + type: + simple: STRING + outputs: + variables: + o0: + type: + simple: STRING + metadata: + retries: {} + runtime: + flavor: python + type: FLYTE_SDK + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/k8s/162_core.type_system.custom_objects.wf_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/k8s/162_core.type_system.custom_objects.wf_2_wf.yaml new file mode 100755 index 0000000000..57dd92a743 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/k8s/162_core.type_system.custom_objects.wf_2_wf.yaml @@ -0,0 +1,405 @@ +apiVersion: flyte.lyft.com/v1alpha1 +executionConfig: + Interruptible: null + MaxParallelism: 0 + RecoveryExecution: {} + TaskPluginImpls: null + TaskResources: + Limits: + CPU: "0" + EphemeralStorage: "0" + GPU: "0" + Memory: "0" + Storage: "0" + Requests: + CPU: "0" + EphemeralStorage: "0" + GPU: "0" + Memory: "0" + Storage: "0" +executionId: {} +inputs: + literals: + x: + scalar: + primitive: + integer: "0" + "y": + scalar: + primitive: + integer: "0" +kind: flyteworkflow +metadata: + creationTimestamp: null + labels: + domain: domain + execution-id: name + project: hello + shard-key: "6" + workflow-name: core-type-system-custom-objects-wf + name: name + namespace: namespace +node-defaults: {} +rawOutputDataConfig: {} +securityContext: {} +spec: + connections: + n0: + - end-node + - n1 + n1: + - end-node + n2: + - n4 + n3: + - n4 + n4: + - end-node + start-node: + - n0 + - n2 + - n3 + edges: + downstream: + n0: + - end-node + - n1 + n1: + - end-node + n2: + - n4 + n3: + - n4 + n4: + - end-node + start-node: + - n0 + - n2 + - n3 + upstream: + end-node: + - n0 + - n1 + - n4 + n0: + - start-node + n1: + - n0 + n2: + - start-node + n3: + - start-node + n4: + - n2 + - n3 + id: ::core.type_system.custom_objects.wf + nodes: + end-node: + id: end-node + inputBindings: + - binding: + promise: + nodeId: n4 + var: o0 + var: o0 + - binding: + promise: + nodeId: n0 + var: o0 + var: o1 + kind: end + resources: {} + n0: + id: n0 + kind: task + name: upload_result + resources: {} + retry: + minAttempts: 1 + task: 'resource_type:TASK name:"core.type_system.custom_objects.upload_result" ' + n1: + id: n1 + inputBindings: + - binding: + promise: + nodeId: n0 + var: o0 + var: res + kind: task + name: download_result + resources: {} + retry: + minAttempts: 1 + task: 'resource_type:TASK name:"core.type_system.custom_objects.download_result" ' + n2: + id: n2 + inputBindings: + - binding: + promise: + nodeId: start-node + var: x + var: x + kind: task + name: stringify + resources: {} + retry: + minAttempts: 1 + task: 'resource_type:TASK name:"core.type_system.custom_objects.stringify" ' + n3: + id: n3 + inputBindings: + - binding: + promise: + nodeId: start-node + var: "y" + var: x + kind: task + name: stringify + resources: {} + retry: + minAttempts: 1 + task: 'resource_type:TASK name:"core.type_system.custom_objects.stringify" ' + n4: + id: n4 + inputBindings: + - binding: + promise: + nodeId: n2 + var: o0 + var: x + - binding: + promise: + nodeId: n3 + var: o0 + var: "y" + kind: task + name: add + resources: {} + retry: + minAttempts: 1 + task: 'resource_type:TASK name:"core.type_system.custom_objects.add" ' + start-node: + id: start-node + kind: start + resources: {} + outputBindings: + - binding: + promise: + nodeId: n4 + var: o0 + var: o0 + - binding: + promise: + nodeId: n0 + var: o0 + var: o1 + outputs: + variables: + o0: + type: + simple: STRUCT + o1: + type: + simple: STRUCT +status: + phase: 0 +tasks: + 'resource_type:TASK name:"core.type_system.custom_objects.add" ': + container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.type_system.custom_objects + - task-name + - add + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.type_system.custom_objects.add + resourceType: TASK + interface: + inputs: + variables: + x: + type: + simple: STRUCT + "y": + type: + simple: STRUCT + outputs: + variables: + o0: + type: + simple: STRUCT + metadata: + retries: {} + runtime: + flavor: python + type: FLYTE_SDK + version: 0.32.6 + type: python-task + 'resource_type:TASK name:"core.type_system.custom_objects.download_result" ': + container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.type_system.custom_objects + - task-name + - download_result + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.type_system.custom_objects.download_result + resourceType: TASK + interface: + inputs: + variables: + res: + type: + simple: STRUCT + outputs: {} + metadata: + retries: {} + runtime: + flavor: python + type: FLYTE_SDK + version: 0.32.6 + type: python-task + 'resource_type:TASK name:"core.type_system.custom_objects.stringify" ': + container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.type_system.custom_objects + - task-name + - stringify + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.type_system.custom_objects.stringify + resourceType: TASK + interface: + inputs: + variables: + x: + type: + simple: INTEGER + outputs: + variables: + o0: + type: + simple: STRUCT + metadata: + retries: {} + runtime: + flavor: python + type: FLYTE_SDK + version: 0.32.6 + type: python-task + 'resource_type:TASK name:"core.type_system.custom_objects.upload_result" ': + container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.type_system.custom_objects + - task-name + - upload_result + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.type_system.custom_objects.upload_result + resourceType: TASK + interface: + inputs: {} + outputs: + variables: + o0: + type: + simple: STRUCT + metadata: + retries: {} + runtime: + flavor: python + type: FLYTE_SDK + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/k8s/166_core.type_system.enums.enum_wf_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/k8s/166_core.type_system.enums.enum_wf_2_wf.yaml new file mode 100755 index 0000000000..260c1c1013 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/k8s/166_core.type_system.enums.enum_wf_2_wf.yaml @@ -0,0 +1,251 @@ +apiVersion: flyte.lyft.com/v1alpha1 +executionConfig: + Interruptible: null + MaxParallelism: 0 + RecoveryExecution: {} + TaskPluginImpls: null + TaskResources: + Limits: + CPU: "0" + EphemeralStorage: "0" + GPU: "0" + Memory: "0" + Storage: "0" + Requests: + CPU: "0" + EphemeralStorage: "0" + GPU: "0" + Memory: "0" + Storage: "0" +executionId: {} +inputs: + literals: + c: + scalar: + primitive: + stringValue: red +kind: flyteworkflow +metadata: + creationTimestamp: null + labels: + domain: domain + execution-id: name + project: hello + shard-key: "6" + workflow-name: core-type-system-enums-enum-wf + name: name + namespace: namespace +node-defaults: {} +rawOutputDataConfig: {} +securityContext: {} +spec: + connections: + n0: + - end-node + - n1 + n1: + - end-node + start-node: + - n0 + edges: + downstream: + n0: + - end-node + - n1 + n1: + - end-node + start-node: + - n0 + upstream: + end-node: + - n0 + - n1 + n0: + - start-node + n1: + - n0 + id: ::core.type_system.enums.enum_wf + nodes: + end-node: + id: end-node + inputBindings: + - binding: + promise: + nodeId: n1 + var: o0 + var: o0 + - binding: + promise: + nodeId: n0 + var: o0 + var: o1 + kind: end + resources: {} + n0: + id: n0 + inputBindings: + - binding: + promise: + nodeId: start-node + var: c + var: c + kind: task + name: enum_stringify + resources: {} + retry: + minAttempts: 1 + task: 'resource_type:TASK name:"core.type_system.enums.enum_stringify" ' + n1: + id: n1 + inputBindings: + - binding: + promise: + nodeId: n0 + var: o0 + var: c + kind: task + name: string_to_enum + resources: {} + retry: + minAttempts: 1 + task: 'resource_type:TASK name:"core.type_system.enums.string_to_enum" ' + start-node: + id: start-node + kind: start + resources: {} + outputBindings: + - binding: + promise: + nodeId: n1 + var: o0 + var: o0 + - binding: + promise: + nodeId: n0 + var: o0 + var: o1 + outputs: + variables: + o0: + type: + enumType: + values: + - red + - green + - blue + o1: + type: + simple: STRING +status: + phase: 0 +tasks: + 'resource_type:TASK name:"core.type_system.enums.enum_stringify" ': + container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.type_system.enums + - task-name + - enum_stringify + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.type_system.enums.enum_stringify + resourceType: TASK + interface: + inputs: + variables: + c: + type: + enumType: + values: + - red + - green + - blue + outputs: + variables: + o0: + type: + simple: STRING + metadata: + retries: {} + runtime: + flavor: python + type: FLYTE_SDK + version: 0.32.6 + type: python-task + 'resource_type:TASK name:"core.type_system.enums.string_to_enum" ': + container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.type_system.enums + - task-name + - string_to_enum + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.type_system.enums.string_to_enum + resourceType: TASK + interface: + inputs: + variables: + c: + type: + simple: STRING + outputs: + variables: + o0: + type: + enumType: + values: + - red + - green + - blue + metadata: + retries: {} + runtime: + flavor: python + type: FLYTE_SDK + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/k8s/169_core.type_system.flyte_pickle.welcome_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/k8s/169_core.type_system.flyte_pickle.welcome_2_wf.yaml new file mode 100755 index 0000000000..17529df047 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/k8s/169_core.type_system.flyte_pickle.welcome_2_wf.yaml @@ -0,0 +1,154 @@ +apiVersion: flyte.lyft.com/v1alpha1 +executionConfig: + Interruptible: null + MaxParallelism: 0 + RecoveryExecution: {} + TaskPluginImpls: null + TaskResources: + Limits: + CPU: "0" + EphemeralStorage: "0" + GPU: "0" + Memory: "0" + Storage: "0" + Requests: + CPU: "0" + EphemeralStorage: "0" + GPU: "0" + Memory: "0" + Storage: "0" +executionId: {} +inputs: + literals: + name: + scalar: + primitive: + stringValue: "" +kind: flyteworkflow +metadata: + creationTimestamp: null + labels: + domain: domain + execution-id: name + project: hello + shard-key: "6" + workflow-name: core-type-system-flyte-pickle-welcome + name: name + namespace: namespace +node-defaults: {} +rawOutputDataConfig: {} +securityContext: {} +spec: + connections: + n0: + - end-node + start-node: + - n0 + edges: + downstream: + n0: + - end-node + start-node: + - n0 + upstream: + end-node: + - n0 + n0: + - start-node + id: ::core.type_system.flyte_pickle.welcome + nodes: + end-node: + id: end-node + inputBindings: + - binding: + promise: + nodeId: n0 + var: o0 + var: o0 + kind: end + resources: {} + n0: + id: n0 + inputBindings: + - binding: + promise: + nodeId: start-node + var: name + var: name + kind: task + name: greet + resources: {} + retry: + minAttempts: 1 + task: 'resource_type:TASK name:"core.type_system.flyte_pickle.greet" ' + start-node: + id: start-node + kind: start + resources: {} + outputBindings: + - binding: + promise: + nodeId: n0 + var: o0 + var: o0 + outputs: + variables: + o0: + type: + blob: + format: PythonPickle +status: + phase: 0 +tasks: + 'resource_type:TASK name:"core.type_system.flyte_pickle.greet" ': + container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.type_system.flyte_pickle + - task-name + - greet + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.type_system.flyte_pickle.greet + resourceType: TASK + interface: + inputs: + variables: + name: + type: + simple: STRING + outputs: + variables: + o0: + type: + blob: + format: PythonPickle + metadata: + retries: {} + runtime: + flavor: python + type: FLYTE_SDK + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/k8s/173_core.type_system.schema.df_wf_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/k8s/173_core.type_system.schema.df_wf_2_wf.yaml new file mode 100755 index 0000000000..1d7f73d86a --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/k8s/173_core.type_system.schema.df_wf_2_wf.yaml @@ -0,0 +1,227 @@ +apiVersion: flyte.lyft.com/v1alpha1 +executionConfig: + Interruptible: null + MaxParallelism: 0 + RecoveryExecution: {} + TaskPluginImpls: null + TaskResources: + Limits: + CPU: "0" + EphemeralStorage: "0" + GPU: "0" + Memory: "0" + Storage: "0" + Requests: + CPU: "0" + EphemeralStorage: "0" + GPU: "0" + Memory: "0" + Storage: "0" +executionId: {} +inputs: + literals: + a: + scalar: + primitive: + integer: "0" +kind: flyteworkflow +metadata: + creationTimestamp: null + labels: + domain: domain + execution-id: name + project: hello + shard-key: "6" + workflow-name: core-type-system-schema-df-wf + name: name + namespace: namespace +node-defaults: {} +rawOutputDataConfig: {} +securityContext: {} +spec: + connections: + n0: + - n1 + n1: + - end-node + start-node: + - n0 + edges: + downstream: + n0: + - n1 + n1: + - end-node + start-node: + - n0 + upstream: + end-node: + - n1 + n0: + - start-node + n1: + - n0 + id: ::core.type_system.schema.df_wf + nodes: + end-node: + id: end-node + inputBindings: + - binding: + promise: + nodeId: n1 + var: o0 + var: o0 + kind: end + resources: {} + n0: + id: n0 + inputBindings: + - binding: + promise: + nodeId: start-node + var: a + var: a + kind: task + name: get_df + resources: {} + retry: + minAttempts: 1 + task: 'resource_type:TASK name:"core.type_system.schema.get_df" ' + n1: + id: n1 + inputBindings: + - binding: + promise: + nodeId: n0 + var: o0 + var: df + kind: task + name: add_df + resources: {} + retry: + minAttempts: 1 + task: 'resource_type:TASK name:"core.type_system.schema.add_df" ' + start-node: + id: start-node + kind: start + resources: {} + outputBindings: + - binding: + promise: + nodeId: n1 + var: o0 + var: o0 + outputs: + variables: + o0: + type: + structuredDatasetType: + format: parquet +status: + phase: 0 +tasks: + 'resource_type:TASK name:"core.type_system.schema.add_df" ': + container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.type_system.schema + - task-name + - add_df + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.type_system.schema.add_df + resourceType: TASK + interface: + inputs: + variables: + df: + type: + structuredDatasetType: + format: parquet + outputs: + variables: + o0: + type: + structuredDatasetType: + format: parquet + metadata: + retries: {} + runtime: + flavor: python + type: FLYTE_SDK + version: 0.32.6 + type: python-task + 'resource_type:TASK name:"core.type_system.schema.get_df" ': + container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.type_system.schema + - task-name + - get_df + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.type_system.schema.get_df + resourceType: TASK + interface: + inputs: + variables: + a: + type: + simple: INTEGER + outputs: + variables: + o0: + type: + structuredDatasetType: + format: parquet + metadata: + retries: {} + runtime: + flavor: python + type: FLYTE_SDK + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/k8s/179_core.type_system.structured_dataset.pandas_compatibility_wf_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/k8s/179_core.type_system.structured_dataset.pandas_compatibility_wf_2_wf.yaml new file mode 100755 index 0000000000..548a07c958 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/k8s/179_core.type_system.structured_dataset.pandas_compatibility_wf_2_wf.yaml @@ -0,0 +1,330 @@ +apiVersion: flyte.lyft.com/v1alpha1 +executionConfig: + Interruptible: null + MaxParallelism: 0 + RecoveryExecution: {} + TaskPluginImpls: null + TaskResources: + Limits: + CPU: "0" + EphemeralStorage: "0" + GPU: "0" + Memory: "0" + Storage: "0" + Requests: + CPU: "0" + EphemeralStorage: "0" + GPU: "0" + Memory: "0" + Storage: "0" +executionId: {} +inputs: + literals: + a: + scalar: + primitive: + integer: "0" +kind: flyteworkflow +metadata: + creationTimestamp: null + labels: + domain: domain + execution-id: name + project: hello + shard-key: "6" + workflow-name: core-type-system-structured-dataset-pandas-compatibility-wf + name: name + namespace: namespace +node-defaults: {} +rawOutputDataConfig: {} +securityContext: {} +spec: + connections: + n0: + - n1 + n1: + - n2 + n2: + - end-node + start-node: + - n0 + edges: + downstream: + n0: + - n1 + n1: + - n2 + n2: + - end-node + start-node: + - n0 + upstream: + end-node: + - n2 + n0: + - start-node + n1: + - n0 + n2: + - n1 + id: ::core.type_system.structured_dataset.pandas_compatibility_wf + nodes: + end-node: + id: end-node + inputBindings: + - binding: + promise: + nodeId: n2 + var: o0 + var: o0 + kind: end + resources: {} + n0: + id: n0 + inputBindings: + - binding: + promise: + nodeId: start-node + var: a + var: a + kind: task + name: get_df + resources: {} + retry: + minAttempts: 1 + task: 'resource_type:TASK name:"core.type_system.structured_dataset.get_df" ' + n1: + id: n1 + inputBindings: + - binding: + promise: + nodeId: n0 + var: o0 + var: df + kind: task + name: get_subset_df + resources: {} + retry: + minAttempts: 1 + task: 'resource_type:TASK name:"core.type_system.structured_dataset.get_subset_df" ' + n2: + id: n2 + inputBindings: + - binding: + promise: + nodeId: n1 + var: o0 + var: ds + kind: task + name: to_numpy + resources: {} + retry: + minAttempts: 1 + task: 'resource_type:TASK name:"core.type_system.structured_dataset.to_numpy" ' + start-node: + id: start-node + kind: start + resources: {} + outputBindings: + - binding: + promise: + nodeId: n2 + var: o0 + var: o0 + outputs: + variables: + o0: + type: + structuredDatasetType: + columns: + - literalType: + simple: INTEGER + name: Age + format: parquet +status: + phase: 0 +tasks: + 'resource_type:TASK name:"core.type_system.structured_dataset.get_df" ': + container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.type_system.structured_dataset + - task-name + - get_df + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.type_system.structured_dataset.get_df + resourceType: TASK + interface: + inputs: + variables: + a: + type: + simple: INTEGER + outputs: + variables: + o0: + type: + structuredDatasetType: + columns: + - literalType: + simple: STRING + name: Name + - literalType: + simple: INTEGER + name: Age + - literalType: + simple: INTEGER + name: Height + format: parquet + metadata: + retries: {} + runtime: + flavor: python + type: FLYTE_SDK + version: 0.32.6 + type: python-task + 'resource_type:TASK name:"core.type_system.structured_dataset.get_subset_df" ': + container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.type_system.structured_dataset + - task-name + - get_subset_df + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.type_system.structured_dataset.get_subset_df + resourceType: TASK + interface: + inputs: + variables: + df: + type: + structuredDatasetType: + columns: + - literalType: + simple: INTEGER + name: Age + format: parquet + outputs: + variables: + o0: + type: + structuredDatasetType: + columns: + - literalType: + simple: INTEGER + name: Age + format: parquet + metadata: + retries: {} + runtime: + flavor: python + type: FLYTE_SDK + version: 0.32.6 + type: python-task + 'resource_type:TASK name:"core.type_system.structured_dataset.to_numpy" ': + container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.type_system.structured_dataset + - task-name + - to_numpy + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.type_system.structured_dataset.to_numpy + resourceType: TASK + interface: + inputs: + variables: + ds: + type: + structuredDatasetType: + columns: + - literalType: + simple: INTEGER + name: Age + format: parquet + outputs: + variables: + o0: + type: + structuredDatasetType: + columns: + - literalType: + simple: INTEGER + name: Age + format: parquet + metadata: + retries: {} + runtime: + flavor: python + type: FLYTE_SDK + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/k8s/181_core.type_system.structured_dataset.schema_compatibility_wf_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/k8s/181_core.type_system.structured_dataset.schema_compatibility_wf_2_wf.yaml new file mode 100755 index 0000000000..ceb764fa92 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/k8s/181_core.type_system.structured_dataset.schema_compatibility_wf_2_wf.yaml @@ -0,0 +1,324 @@ +apiVersion: flyte.lyft.com/v1alpha1 +executionConfig: + Interruptible: null + MaxParallelism: 0 + RecoveryExecution: {} + TaskPluginImpls: null + TaskResources: + Limits: + CPU: "0" + EphemeralStorage: "0" + GPU: "0" + Memory: "0" + Storage: "0" + Requests: + CPU: "0" + EphemeralStorage: "0" + GPU: "0" + Memory: "0" + Storage: "0" +executionId: {} +inputs: + literals: + a: + scalar: + primitive: + integer: "0" +kind: flyteworkflow +metadata: + creationTimestamp: null + labels: + domain: domain + execution-id: name + project: hello + shard-key: "6" + workflow-name: core-type-system-structured-dataset-schema-compatibility-wf + name: name + namespace: namespace +node-defaults: {} +rawOutputDataConfig: {} +securityContext: {} +spec: + connections: + n0: + - n1 + n1: + - n2 + n2: + - end-node + start-node: + - n0 + edges: + downstream: + n0: + - n1 + n1: + - n2 + n2: + - end-node + start-node: + - n0 + upstream: + end-node: + - n2 + n0: + - start-node + n1: + - n0 + n2: + - n1 + id: ::core.type_system.structured_dataset.schema_compatibility_wf + nodes: + end-node: + id: end-node + inputBindings: + - binding: + promise: + nodeId: n2 + var: o0 + var: o0 + kind: end + resources: {} + n0: + id: n0 + inputBindings: + - binding: + promise: + nodeId: start-node + var: a + var: a + kind: task + name: get_schema_df + resources: {} + retry: + minAttempts: 1 + task: 'resource_type:TASK name:"core.type_system.structured_dataset.get_schema_df" ' + n1: + id: n1 + inputBindings: + - binding: + promise: + nodeId: n0 + var: o0 + var: df + kind: task + name: get_subset_df + resources: {} + retry: + minAttempts: 1 + task: 'resource_type:TASK name:"core.type_system.structured_dataset.get_subset_df" ' + n2: + id: n2 + inputBindings: + - binding: + promise: + nodeId: n1 + var: o0 + var: ds + kind: task + name: to_numpy + resources: {} + retry: + minAttempts: 1 + task: 'resource_type:TASK name:"core.type_system.structured_dataset.to_numpy" ' + start-node: + id: start-node + kind: start + resources: {} + outputBindings: + - binding: + promise: + nodeId: n2 + var: o0 + var: o0 + outputs: + variables: + o0: + type: + structuredDatasetType: + columns: + - literalType: + simple: INTEGER + name: Age + format: parquet +status: + phase: 0 +tasks: + 'resource_type:TASK name:"core.type_system.structured_dataset.get_schema_df" ': + container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.type_system.structured_dataset + - task-name + - get_schema_df + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.type_system.structured_dataset.get_schema_df + resourceType: TASK + interface: + inputs: + variables: + a: + type: + simple: INTEGER + outputs: + variables: + o0: + type: + schema: + columns: + - name: Name + type: STRING + - name: Age + - name: Height + metadata: + retries: {} + runtime: + flavor: python + type: FLYTE_SDK + version: 0.32.6 + type: python-task + 'resource_type:TASK name:"core.type_system.structured_dataset.get_subset_df" ': + container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.type_system.structured_dataset + - task-name + - get_subset_df + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.type_system.structured_dataset.get_subset_df + resourceType: TASK + interface: + inputs: + variables: + df: + type: + structuredDatasetType: + columns: + - literalType: + simple: INTEGER + name: Age + format: parquet + outputs: + variables: + o0: + type: + structuredDatasetType: + columns: + - literalType: + simple: INTEGER + name: Age + format: parquet + metadata: + retries: {} + runtime: + flavor: python + type: FLYTE_SDK + version: 0.32.6 + type: python-task + 'resource_type:TASK name:"core.type_system.structured_dataset.to_numpy" ': + container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.type_system.structured_dataset + - task-name + - to_numpy + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.type_system.structured_dataset.to_numpy + resourceType: TASK + interface: + inputs: + variables: + ds: + type: + structuredDatasetType: + columns: + - literalType: + simple: INTEGER + name: Age + format: parquet + outputs: + variables: + o0: + type: + structuredDatasetType: + columns: + - literalType: + simple: INTEGER + name: Age + format: parquet + metadata: + retries: {} + runtime: + flavor: python + type: FLYTE_SDK + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/test/testdata/snacks-core/k8s/185_core.type_system.typed_schema.wf_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/k8s/185_core.type_system.typed_schema.wf_2_wf.yaml new file mode 100755 index 0000000000..cd06f04544 --- /dev/null +++ b/pkg/compiler/test/testdata/snacks-core/k8s/185_core.type_system.typed_schema.wf_2_wf.yaml @@ -0,0 +1,220 @@ +apiVersion: flyte.lyft.com/v1alpha1 +executionConfig: + Interruptible: null + MaxParallelism: 0 + RecoveryExecution: {} + TaskPluginImpls: null + TaskResources: + Limits: + CPU: "0" + EphemeralStorage: "0" + GPU: "0" + Memory: "0" + Storage: "0" + Requests: + CPU: "0" + EphemeralStorage: "0" + GPU: "0" + Memory: "0" + Storage: "0" +executionId: {} +inputs: {} +kind: flyteworkflow +metadata: + creationTimestamp: null + labels: + domain: domain + execution-id: name + project: hello + shard-key: "6" + workflow-name: core-type-system-typed-schema-wf + name: name + namespace: namespace +node-defaults: {} +rawOutputDataConfig: {} +securityContext: {} +spec: + connections: + n0: + - n1 + n1: + - end-node + start-node: + - n0 + edges: + downstream: + n0: + - n1 + n1: + - end-node + start-node: + - n0 + upstream: + end-node: + - n1 + n0: + - start-node + n1: + - n0 + id: ::core.type_system.typed_schema.wf + nodes: + end-node: + id: end-node + inputBindings: + - binding: + promise: + nodeId: n1 + var: o0 + var: o0 + kind: end + resources: {} + n0: + id: n0 + kind: task + name: t1 + resources: {} + retry: + minAttempts: 1 + task: 'resource_type:TASK name:"core.type_system.typed_schema.t1" ' + n1: + id: n1 + inputBindings: + - binding: + promise: + nodeId: n0 + var: o0 + var: schema + kind: task + name: t2 + resources: {} + retry: + minAttempts: 1 + task: 'resource_type:TASK name:"core.type_system.typed_schema.t2" ' + start-node: + id: start-node + kind: start + resources: {} + outputBindings: + - binding: + promise: + nodeId: n1 + var: o0 + var: o0 + outputs: + variables: + o0: + type: + schema: + columns: + - name: x +status: + phase: 0 +tasks: + 'resource_type:TASK name:"core.type_system.typed_schema.t1" ': + container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.type_system.typed_schema + - task-name + - t1 + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.type_system.typed_schema.t1 + resourceType: TASK + interface: + inputs: {} + outputs: + variables: + o0: + type: + schema: + columns: + - name: x + - name: "y" + type: STRING + metadata: + retries: {} + runtime: + flavor: python + type: FLYTE_SDK + version: 0.32.6 + type: python-task + 'resource_type:TASK name:"core.type_system.typed_schema.t2" ': + container: + args: + - pyflyte-execute + - --inputs + - '{{.input}}' + - --output-prefix + - '{{.outputPrefix}}' + - --raw-output-data-prefix + - '{{.rawOutputDataPrefix}}' + - --checkpoint-path + - '{{.checkpointOutputPrefix}}' + - --prev-checkpoint + - '{{.prevCheckpointPrefix}}' + - --resolver + - flytekit.core.python_auto_container.default_task_resolver + - -- + - task-module + - core.type_system.typed_schema + - task-name + - t2 + config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 + image: ghcr.io/flyteorg/flytecookbook:core-8b8e1a849c9adfca88049a074b10dad278f70077 + resources: {} + id: + name: core.type_system.typed_schema.t2 + resourceType: TASK + interface: + inputs: + variables: + schema: + type: + schema: + columns: + - name: x + - name: "y" + type: STRING + outputs: + variables: + o0: + type: + schema: + columns: + - name: x + metadata: + retries: {} + runtime: + flavor: python + type: FLYTE_SDK + version: 0.32.6 + type: python-task diff --git a/pkg/compiler/transformers/k8s/node.go b/pkg/compiler/transformers/k8s/node.go index 33e883d0a4..aece64a9aa 100644 --- a/pkg/compiler/transformers/k8s/node.go +++ b/pkg/compiler/transformers/k8s/node.go @@ -221,28 +221,6 @@ func buildTasks(tasks []*core.CompiledTask, errs errors.CompileErrors) map[commo errs.Collect(errors.NewValueCollisionError(taskID, "Id", taskID)) } - // We don't want Annotation data available at task runtime for performance. - if flyteTask.Template != nil && - flyteTask.Template.Interface != nil { - - if flyteTask.Template.Interface.Inputs != nil { - for _, v := range flyteTask.Template.Interface.Inputs.Variables { - if v.Type != nil && v.Type.Annotation != nil { - v.Type.Annotation = nil - } - } - } - - if flyteTask.Template.Interface.Outputs != nil { - for _, v := range flyteTask.Template.Interface.Outputs.Variables { - if v.Type != nil && v.Type.Annotation != nil { - v.Type.Annotation = nil - } - } - } - - } - res[taskID] = &v1alpha1.TaskSpec{TaskTemplate: flyteTask.Template} } } diff --git a/pkg/compiler/transformers/k8s/node_test.go b/pkg/compiler/transformers/k8s/node_test.go index c5bb803020..a5b7ef275e 100644 --- a/pkg/compiler/transformers/k8s/node_test.go +++ b/pkg/compiler/transformers/k8s/node_test.go @@ -4,7 +4,6 @@ import ( "testing" "github.com/flyteorg/flytepropeller/pkg/apis/flyteworkflow/v1alpha1" - "google.golang.org/protobuf/types/known/structpb" "k8s.io/apimachinery/pkg/api/resource" "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core" @@ -215,20 +214,7 @@ func TestBuildTasks(t *testing.T) { withoutAnnotations := make(map[string]*core.Variable) withoutAnnotations["a"] = &core.Variable{ - Type: &core.LiteralType{ - Annotation: &core.TypeAnnotation{}, - }, - } - - randomData, _ := structpb.NewStruct(map[string]interface{}{ - "foo": "bar", - }) - - withAnnotations := make(map[string]*core.Variable) - withAnnotations["a"] = &core.Variable{ - Type: &core.LiteralType{ - Annotation: &core.TypeAnnotation{Annotations: randomData}, - }, + Type: &core.LiteralType{}, } tasks := []*core.CompiledTask{ @@ -237,7 +223,7 @@ func TestBuildTasks(t *testing.T) { Id: &core.Identifier{Name: "annotatedInput"}, Interface: &core.TypedInterface{ Inputs: &core.VariableMap{ - Variables: withAnnotations, + Variables: withoutAnnotations, }, }, }, @@ -257,7 +243,7 @@ func TestBuildTasks(t *testing.T) { Id: &core.Identifier{Name: "annotatedOutput"}, Interface: &core.TypedInterface{ Outputs: &core.VariableMap{ - Variables: withAnnotations, + Variables: withoutAnnotations, }, }, }, diff --git a/pkg/compiler/transformers/k8s/utils.go b/pkg/compiler/transformers/k8s/utils.go index e0a0098dab..1cf9a521cc 100644 --- a/pkg/compiler/transformers/k8s/utils.go +++ b/pkg/compiler/transformers/k8s/utils.go @@ -89,3 +89,67 @@ func toBindingValueArray(bindings []*core.Binding) []*v1alpha1.Binding { func minInt(i, j int) int { return int(math.Min(float64(i), float64(j))) } + +// StripTypeMetadata strips the type metadata from the given type. +func StripTypeMetadata(t *core.LiteralType) *core.LiteralType { + if t == nil { + return nil + } + + c := *t + c.Metadata = nil + c.Structure = nil + c.Annotation = nil + + switch underlyingType := c.Type.(type) { + case *core.LiteralType_UnionType: + variants := make([]*core.LiteralType, 0, len(c.GetUnionType().Variants)) + for _, variant := range c.GetUnionType().Variants { + variants = append(variants, StripTypeMetadata(variant)) + } + + underlyingType.UnionType.Variants = variants + case *core.LiteralType_MapValueType: + underlyingType.MapValueType = StripTypeMetadata(c.GetMapValueType()) + case *core.LiteralType_CollectionType: + underlyingType.CollectionType = StripTypeMetadata(c.GetCollectionType()) + case *core.LiteralType_StructuredDatasetType: + columns := make([]*core.StructuredDatasetType_DatasetColumn, 0, len(c.GetStructuredDatasetType().Columns)) + for _, column := range c.GetStructuredDatasetType().Columns { + columns = append(columns, &core.StructuredDatasetType_DatasetColumn{ + Name: column.Name, + LiteralType: StripTypeMetadata(column.LiteralType), + }) + } + + underlyingType.StructuredDatasetType.Columns = columns + } + + return &c +} + +func StripInterfaceTypeMetadata(iface *core.TypedInterface) *core.TypedInterface { + if iface == nil { + return nil + } + + newIface := *iface + + if iface.Inputs != nil { + for name, i := range iface.Inputs.Variables { + i.Type = StripTypeMetadata(i.Type) + i.Description = "" + newIface.Inputs.Variables[name] = i + } + } + + if iface.Outputs != nil { + for name, i := range iface.Outputs.Variables { + i.Type = StripTypeMetadata(i.Type) + i.Description = "" + iface.Outputs.Variables[name] = i + } + } + + return &newIface +} diff --git a/pkg/compiler/transformers/k8s/utils_test.go b/pkg/compiler/transformers/k8s/utils_test.go index 8f04737002..7c48602b3c 100644 --- a/pkg/compiler/transformers/k8s/utils_test.go +++ b/pkg/compiler/transformers/k8s/utils_test.go @@ -3,6 +3,9 @@ package k8s import ( "testing" + "github.com/go-test/deep" + _struct "github.com/golang/protobuf/ptypes/struct" + "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core" "github.com/stretchr/testify/assert" ) @@ -56,3 +59,238 @@ func TestComputeRetryStrategy(t *testing.T) { } } + +func TestStripTypeMetadata(t *testing.T) { + + tests := []struct { + name string + args *core.LiteralType + want *core.LiteralType + }{ + { + name: "nil", + args: nil, + want: nil, + }, + { + name: "simple", + args: &core.LiteralType{ + Type: &core.LiteralType_Simple{ + Simple: core.SimpleType_INTEGER, + }, + Metadata: &_struct.Struct{ + Fields: map[string]*_struct.Value{ + "foo": { + Kind: &_struct.Value_StringValue{ + StringValue: "bar", + }, + }, + }, + }, + }, + want: &core.LiteralType{ + Type: &core.LiteralType_Simple{ + Simple: core.SimpleType_INTEGER, + }, + }, + }, + { + name: "collection", + args: &core.LiteralType{ + Type: &core.LiteralType_CollectionType{ + CollectionType: &core.LiteralType{ + Type: &core.LiteralType_Simple{ + Simple: core.SimpleType_INTEGER, + }, + }, + }, + Metadata: &_struct.Struct{ + Fields: map[string]*_struct.Value{ + "foo": { + Kind: &_struct.Value_StringValue{ + StringValue: "bar", + }, + }, + }, + }, + }, + want: &core.LiteralType{ + Type: &core.LiteralType_CollectionType{ + CollectionType: &core.LiteralType{ + Type: &core.LiteralType_Simple{ + Simple: core.SimpleType_INTEGER, + }, + }, + }, + }, + }, + { + name: "map", + args: &core.LiteralType{ + Type: &core.LiteralType_MapValueType{ + MapValueType: &core.LiteralType{ + Type: &core.LiteralType_Simple{ + Simple: core.SimpleType_INTEGER, + }, + }, + }, + Metadata: &_struct.Struct{ + Fields: map[string]*_struct.Value{ + "foo": { + Kind: &_struct.Value_StringValue{ + StringValue: "bar", + }, + }, + }, + }, + }, + want: &core.LiteralType{ + Type: &core.LiteralType_MapValueType{ + MapValueType: &core.LiteralType{ + Type: &core.LiteralType_Simple{ + Simple: core.SimpleType_INTEGER, + }, + }, + }, + }, + }, + { + name: "union", + args: &core.LiteralType{ + Type: &core.LiteralType_UnionType{ + UnionType: &core.UnionType{ + Variants: []*core.LiteralType{ + { + Type: &core.LiteralType_Simple{ + Simple: core.SimpleType_INTEGER, + }, + }, + { + Type: &core.LiteralType_Simple{ + Simple: core.SimpleType_STRING, + }, + Metadata: &_struct.Struct{ + Fields: map[string]*_struct.Value{ + "foo": { + Kind: &_struct.Value_StringValue{ + StringValue: "bar", + }, + }, + }, + }, + }, + }, + }, + }, + }, + want: &core.LiteralType{ + Type: &core.LiteralType_UnionType{ + UnionType: &core.UnionType{ + Variants: []*core.LiteralType{ + { + Type: &core.LiteralType_Simple{ + Simple: core.SimpleType_INTEGER, + }, + }, + { + Type: &core.LiteralType_Simple{ + Simple: core.SimpleType_STRING, + }, + }, + }, + }, + }, + }, + }, + { + name: "StructuredDataSet", + args: &core.LiteralType{ + Type: &core.LiteralType_StructuredDatasetType{ + StructuredDatasetType: &core.StructuredDatasetType{ + Columns: []*core.StructuredDatasetType_DatasetColumn{ + { + Name: "column1", + LiteralType: &core.LiteralType{ + Type: &core.LiteralType_Simple{ + Simple: core.SimpleType_STRING, + }, + Metadata: &_struct.Struct{ + Fields: map[string]*_struct.Value{ + "foo": { + Kind: &_struct.Value_StringValue{ + StringValue: "bar", + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + want: &core.LiteralType{ + Type: &core.LiteralType_StructuredDatasetType{ + StructuredDatasetType: &core.StructuredDatasetType{ + Columns: []*core.StructuredDatasetType_DatasetColumn{ + { + Name: "column1", + LiteralType: &core.LiteralType{ + Type: &core.LiteralType_Simple{ + Simple: core.SimpleType_STRING, + }, + }, + }, + }, + }, + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if diff := deep.Equal(StripTypeMetadata(tt.args), tt.want); diff != nil { + assert.Fail(t, "actual != expected", "Diff: %v", diff) + } + }) + } +} + +func TestStripInterfaceTypeMetadata(t *testing.T) { + t.Run("nil", func(t *testing.T) { + assert.Nil(t, StripInterfaceTypeMetadata(nil)) + }) + + t.Run("populated", func(t *testing.T) { + vars := &core.VariableMap{ + Variables: map[string]*core.Variable{ + "a": { + Type: &core.LiteralType{ + Type: &core.LiteralType_Simple{ + Simple: core.SimpleType_INTEGER, + }, + Metadata: &_struct.Struct{ + Fields: map[string]*_struct.Value{ + "foo": { + Kind: &_struct.Value_StringValue{ + StringValue: "bar", + }, + }, + }, + }, + }, + }, + }, + } + + i := &core.TypedInterface{ + Inputs: vars, + Outputs: vars, + } + + stripped := StripInterfaceTypeMetadata(i) + assert.Nil(t, stripped.Inputs.Variables["a"].Type.Metadata) + assert.Nil(t, stripped.Outputs.Variables["a"].Type.Metadata) + }) +} diff --git a/pkg/compiler/transformers/k8s/workflow.go b/pkg/compiler/transformers/k8s/workflow.go index 95e3db718d..32ccc0bf00 100644 --- a/pkg/compiler/transformers/k8s/workflow.go +++ b/pkg/compiler/transformers/k8s/workflow.go @@ -82,6 +82,8 @@ func WorkflowNameFromID(id string) string { func buildFlyteWorkflowSpec(wf *core.CompiledWorkflow, tasks []*core.CompiledTask, errs errors.CompileErrors) ( spec *v1alpha1.WorkflowSpec, err error) { + wf.Template.Interface = StripInterfaceTypeMetadata(wf.Template.Interface) + var failureN *v1alpha1.NodeSpec if n := wf.Template.GetFailureNode(); n != nil { nodes, ok := buildNodeSpec(n, tasks, errs.NewScope()) @@ -156,7 +158,7 @@ func generateName(wfID *core.Identifier, execID *core.WorkflowExecutionIdentifie } } -// Builds v1alpha1.FlyteWorkflow resource. Returned error, if not nil, is of type errors.CompilerErrors. +// BuildFlyteWorkflow builds v1alpha1.FlyteWorkflow resource. Returned error, if not nil, is of type errors.CompilerErrors. func BuildFlyteWorkflow(wfClosure *core.CompiledWorkflowClosure, inputs *core.LiteralMap, executionID *core.WorkflowExecutionIdentifier, namespace string) (*v1alpha1.FlyteWorkflow, error) { @@ -166,11 +168,16 @@ func BuildFlyteWorkflow(wfClosure *core.CompiledWorkflowClosure, inputs *core.Li return nil, errs } + for _, t := range wfClosure.Tasks { + t.Template.Interface = StripInterfaceTypeMetadata(t.Template.Interface) + } + primarySpec, err := buildFlyteWorkflowSpec(wfClosure.Primary, wfClosure.Tasks, errs.NewScope()) if err != nil { errs.Collect(errors.NewWorkflowBuildError(err)) return nil, errs } + subwfs := make(map[v1alpha1.WorkflowID]*v1alpha1.WorkflowSpec, len(wfClosure.SubWorkflows)) for _, subWf := range wfClosure.SubWorkflows { spec, err := buildFlyteWorkflowSpec(subWf, wfClosure.Tasks, errs.NewScope()) diff --git a/pkg/compiler/validators/interface.go b/pkg/compiler/validators/interface.go index aa0524319c..3f05ecf5ac 100644 --- a/pkg/compiler/validators/interface.go +++ b/pkg/compiler/validators/interface.go @@ -8,7 +8,7 @@ import ( "github.com/flyteorg/flytepropeller/pkg/compiler/errors" ) -// Validate interface has its required attributes set +// ValidateInterface validates interface has its required attributes set func ValidateInterface(nodeID c.NodeID, iface *core.TypedInterface, errs errors.CompileErrors) ( typedInterface *core.TypedInterface, ok bool) { @@ -32,7 +32,7 @@ func ValidateInterface(nodeID c.NodeID, iface *core.TypedInterface, errs errors. return iface, !errs.HasErrors() } -// Validates underlying interface of a node and returns the effective Typed Interface. +// ValidateUnderlyingInterface validates the underlying interface of a node and returns the effective Typed Interface. func ValidateUnderlyingInterface(w c.WorkflowBuilder, node c.NodeBuilder, errs errors.CompileErrors) (iface *core.TypedInterface, ok bool) { if node.GetInterface() != nil { return node.GetInterface(), true diff --git a/pkg/compiler/validators/utils.go b/pkg/compiler/validators/utils.go index f22302b87d..03e4636ce1 100644 --- a/pkg/compiler/validators/utils.go +++ b/pkg/compiler/validators/utils.go @@ -34,7 +34,7 @@ func findVariableByName(vars *core.VariableMap, name string) (variable *core.Var func literalTypeForScalar(scalar *core.Scalar) *core.LiteralType { // TODO: Should we just pass the type information with the value? That way we don't have to guess? var literalType *core.LiteralType - switch scalar.GetValue().(type) { + switch v := scalar.GetValue().(type) { case *core.Scalar_Primitive: literalType = literalTypeForPrimitive(scalar.GetPrimitive()) case *core.Scalar_Blob: @@ -52,6 +52,12 @@ func literalTypeForScalar(scalar *core.Scalar) *core.LiteralType { }, } case *core.Scalar_StructuredDataset: + if v.StructuredDataset == nil || v.StructuredDataset.Metadata == nil { + return &core.LiteralType{ + Type: &core.LiteralType_StructuredDatasetType{}, + } + } + literalType = &core.LiteralType{ Type: &core.LiteralType_StructuredDatasetType{ StructuredDatasetType: scalar.GetStructuredDataset().GetMetadata().StructuredDatasetType, diff --git a/pkg/compiler/validators/utils_test.go b/pkg/compiler/validators/utils_test.go index 004bc227a5..3557ba0ec8 100644 --- a/pkg/compiler/validators/utils_test.go +++ b/pkg/compiler/validators/utils_test.go @@ -4,29 +4,28 @@ import ( "testing" "github.com/flyteorg/flyteidl/clients/go/coreutils" - - flyte "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core" + "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core" "github.com/stretchr/testify/assert" ) func TestLiteralTypeForLiterals(t *testing.T) { t.Run("empty", func(t *testing.T) { lt := literalTypeForLiterals(nil) - assert.Equal(t, flyte.SimpleType_NONE.String(), lt.GetSimple().String()) + assert.Equal(t, core.SimpleType_NONE.String(), lt.GetSimple().String()) }) t.Run("homogenous", func(t *testing.T) { - lt := literalTypeForLiterals([]*flyte.Literal{ + lt := literalTypeForLiterals([]*core.Literal{ coreutils.MustMakeLiteral(5), coreutils.MustMakeLiteral(0), coreutils.MustMakeLiteral(5), }) - assert.Equal(t, flyte.SimpleType_INTEGER.String(), lt.GetSimple().String()) + assert.Equal(t, core.SimpleType_INTEGER.String(), lt.GetSimple().String()) }) t.Run("non-homogenous", func(t *testing.T) { - lt := literalTypeForLiterals([]*flyte.Literal{ + lt := literalTypeForLiterals([]*core.Literal{ coreutils.MustMakeLiteral("hello"), coreutils.MustMakeLiteral(5), coreutils.MustMakeLiteral("world"), @@ -35,12 +34,12 @@ func TestLiteralTypeForLiterals(t *testing.T) { }) assert.Len(t, lt.GetUnionType().Variants, 2) - assert.Equal(t, flyte.SimpleType_INTEGER.String(), lt.GetUnionType().Variants[0].GetSimple().String()) - assert.Equal(t, flyte.SimpleType_STRING.String(), lt.GetUnionType().Variants[1].GetSimple().String()) + assert.Equal(t, core.SimpleType_INTEGER.String(), lt.GetUnionType().Variants[0].GetSimple().String()) + assert.Equal(t, core.SimpleType_STRING.String(), lt.GetUnionType().Variants[1].GetSimple().String()) }) t.Run("non-homogenous ensure ordering", func(t *testing.T) { - lt := literalTypeForLiterals([]*flyte.Literal{ + lt := literalTypeForLiterals([]*core.Literal{ coreutils.MustMakeLiteral(5), coreutils.MustMakeLiteral("world"), coreutils.MustMakeLiteral(0), @@ -48,32 +47,32 @@ func TestLiteralTypeForLiterals(t *testing.T) { }) assert.Len(t, lt.GetUnionType().Variants, 2) - assert.Equal(t, flyte.SimpleType_INTEGER.String(), lt.GetUnionType().Variants[0].GetSimple().String()) - assert.Equal(t, flyte.SimpleType_STRING.String(), lt.GetUnionType().Variants[1].GetSimple().String()) + assert.Equal(t, core.SimpleType_INTEGER.String(), lt.GetUnionType().Variants[0].GetSimple().String()) + assert.Equal(t, core.SimpleType_STRING.String(), lt.GetUnionType().Variants[1].GetSimple().String()) }) } func TestJoinVariableMapsUniqueKeys(t *testing.T) { - intType := &flyte.LiteralType{ - Type: &flyte.LiteralType_Simple{ - Simple: flyte.SimpleType_INTEGER, + intType := &core.LiteralType{ + Type: &core.LiteralType_Simple{ + Simple: core.SimpleType_INTEGER, }, } - strType := &flyte.LiteralType{ - Type: &flyte.LiteralType_Simple{ - Simple: flyte.SimpleType_STRING, + strType := &core.LiteralType{ + Type: &core.LiteralType_Simple{ + Simple: core.SimpleType_STRING, }, } t.Run("Simple", func(t *testing.T) { - m1 := map[string]*flyte.Variable{ + m1 := map[string]*core.Variable{ "x": { Type: intType, }, } - m2 := map[string]*flyte.Variable{ + m2 := map[string]*core.Variable{ "y": { Type: intType, }, @@ -85,13 +84,13 @@ func TestJoinVariableMapsUniqueKeys(t *testing.T) { }) t.Run("No type collision", func(t *testing.T) { - m1 := map[string]*flyte.Variable{ + m1 := map[string]*core.Variable{ "x": { Type: intType, }, } - m2 := map[string]*flyte.Variable{ + m2 := map[string]*core.Variable{ "x": { Type: intType, }, @@ -103,13 +102,13 @@ func TestJoinVariableMapsUniqueKeys(t *testing.T) { }) t.Run("Type collision", func(t *testing.T) { - m1 := map[string]*flyte.Variable{ + m1 := map[string]*core.Variable{ "x": { Type: intType, }, } - m2 := map[string]*flyte.Variable{ + m2 := map[string]*core.Variable{ "x": { Type: strType, }, diff --git a/pkg/compiler/workflow_compiler.go b/pkg/compiler/workflow_compiler.go index 9b0c49dc79..332871215e 100755 --- a/pkg/compiler/workflow_compiler.go +++ b/pkg/compiler/workflow_compiler.go @@ -1,4 +1,4 @@ -// This package provides compiler services for flyte workflows. It performs static analysis on the Workflow and produces +// Package compiler provides compiler services for flyte workflows. It performs static analysis on the Workflow and produces // CompilerErrors for any detected issue. A flyte workflow should only be considered valid for execution if it passed through // the compiler first. The intended usage for the compiler is as follows: // 1) Call GetRequirements(...) and load/retrieve all tasks/workflows referenced in the response. @@ -283,8 +283,8 @@ func (w workflowBuilder) validateAllRequirements(errs errors.CompileErrors) bool return !errs.HasErrors() } -// Compiles a flyte workflow a and all of its dependencies into a single executable Workflow. Refer to GetRequirements() -// to obtain a list of launchplan and Task ids to load/compile first. +// CompileWorkflow compiles a flyte workflow a and all of its dependencies into a single executable Workflow. Refer to +// GetRequirements() to obtain a list of launchplan and Task ids to load/compile first. // Returns an executable Workflow (if no errors are found) or a list of errors that must be addressed before the Workflow // can be executed. Cast the error to errors.CompileErrors to inspect individual errors. func CompileWorkflow(primaryWf *core.WorkflowTemplate, subworkflows []*core.WorkflowTemplate, tasks []*core.CompiledTask, @@ -343,8 +343,10 @@ func CompileWorkflow(primaryWf *core.WorkflowTemplate, subworkflows []*core.Work compiledTasks = append(compiledTasks, &core.CompiledTask{Template: t.GetCoreTask()}) } + coreWf := validatedWf.GetCoreWorkflow() + return &core.CompiledWorkflowClosure{ - Primary: validatedWf.GetCoreWorkflow(), + Primary: coreWf, Tasks: compiledTasks, SubWorkflows: compiledSubWorkflows, }, nil