Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(v2): V2 create run api #6689

Merged
merged 19 commits into from
Oct 29, 2021
Merged

feat(v2): V2 create run api #6689

Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
19 commits
Select commit Hold shift + click to select a range
03583c6
added draft of create v2 pipeline run
capri-xiyue Oct 5, 2021
e3d38eb
fixed broken UT and added UT for parsing template
capri-xiyue Oct 5, 2021
8fb6cff
modified run apis to support v2 IR spec
capri-xiyue Oct 6, 2021
322f540
remove temporary patch
capri-xiyue Oct 7, 2021
cd802a7
fixed dependency
capri-xiyue Oct 7, 2021
e53140c
fixed build failure
capri-xiyue Oct 7, 2021
d30e149
finished draft
capri-xiyue Oct 20, 2021
6487705
finished create job and run
capri-xiyue Oct 20, 2021
802956f
resolve master conflicts
capri-xiyue Oct 20, 2021
0db6d60
refactor template and fixed broken UT
capri-xiyue Oct 26, 2021
4dc3c42
updated go license
capri-xiyue Oct 27, 2021
de09d1b
resolve conflicts
capri-xiyue Oct 27, 2021
5ed912b
fixed build failure
capri-xiyue Oct 27, 2021
504621e
fixed build
capri-xiyue Oct 27, 2021
654d915
added UT
capri-xiyue Oct 28, 2021
c731c25
modified UT
capri-xiyue Oct 28, 2021
ddc0c8d
fixed merge conflict
capri-xiyue Oct 28, 2021
7c89e74
fixed build failure
capri-xiyue Oct 28, 2021
d68b8ac
fixed license
capri-xiyue Oct 28, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
77 changes: 53 additions & 24 deletions backend/src/apiserver/resource/model_converter.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ package resource

import (
"encoding/json"
"fmt"

"github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1"
api "github.com/kubeflow/pipelines/backend/api/go_client"
Expand Down Expand Up @@ -53,9 +54,9 @@ func (r *ResourceManager) ToModelRunMetric(metric *api.RunMetric, runUUID string
}
}

// The input run might not contain workflowSpecManifest, but instead a pipeline ID.
// The caller would retrieve workflowSpecManifest and pass in.
func (r *ResourceManager) ToModelRunDetail(run *api.Run, runId string, workflow *util.Workflow, workflowSpecManifest string) (*model.RunDetail, error) {
// The input run might not contain workflowSpecManifest and pipelineSpecManifest, but instead a pipeline ID.
// The caller would retrieve manifest and pass in.
func (r *ResourceManager) ToModelRunDetail(run *api.Run, runId string, workflow *util.Workflow, manifest string, templateType util.TemplateType) (*model.RunDetail, error) {
params, err := toModelParameters(run.GetPipelineSpec().GetParameters())
if err != nil {
return nil, util.Wrap(err, "Unable to parse the parameter.")
Expand All @@ -77,28 +78,56 @@ func (r *ResourceManager) ToModelRunDetail(run *api.Run, runId string, workflow
return nil, util.Wrap(err, "Error getting the experiment UUID")
}

return &model.RunDetail{
Run: model.Run{
UUID: runId,
ExperimentUUID: experimentUUID,
DisplayName: run.Name,
Name: workflow.Name,
Namespace: workflow.Namespace,
ServiceAccount: workflow.Spec.ServiceAccountName,
Conditions: workflow.Condition(),
Description: run.Description,
ResourceReferences: resourceReferences,
PipelineSpec: model.PipelineSpec{
PipelineId: run.GetPipelineSpec().GetPipelineId(),
PipelineName: pipelineName,
WorkflowSpecManifest: workflowSpecManifest,
Parameters: params,
if templateType == util.V1 {
return &model.RunDetail{
Run: model.Run{
UUID: runId,
ExperimentUUID: experimentUUID,
DisplayName: run.Name,
Name: workflow.Name,
Namespace: workflow.Namespace,
ServiceAccount: workflow.Spec.ServiceAccountName,
Conditions: workflow.Condition(),
Description: run.Description,
ResourceReferences: resourceReferences,
PipelineSpec: model.PipelineSpec{
PipelineId: run.GetPipelineSpec().GetPipelineId(),
PipelineName: pipelineName,
WorkflowSpecManifest: manifest,
Parameters: params,
},
},
},
PipelineRuntime: model.PipelineRuntime{
WorkflowRuntimeManifest: workflow.ToStringForStore(),
},
}, nil
PipelineRuntime: model.PipelineRuntime{
WorkflowRuntimeManifest: workflow.ToStringForStore(),
},
}, nil
} else if templateType == util.V2 {
return &model.RunDetail{
Run: model.Run{
UUID: runId,
ExperimentUUID: experimentUUID,
DisplayName: run.Name,
Name: workflow.Name,
Namespace: workflow.Namespace,
ServiceAccount: workflow.Spec.ServiceAccountName,
capri-xiyue marked this conversation as resolved.
Show resolved Hide resolved
Conditions: workflow.Condition(),
Description: run.Description,
ResourceReferences: resourceReferences,
PipelineSpec: model.PipelineSpec{
PipelineId: run.GetPipelineSpec().GetPipelineId(),
PipelineName: pipelineName,
PipelineSpecManifest: manifest,
Parameters: params,
},
},
PipelineRuntime: model.PipelineRuntime{
// TODO Check whether this is the correct logic in code review
PipelineRuntimeManifest: workflow.ToStringForStore(),
Bobgy marked this conversation as resolved.
Show resolved Hide resolved
},
}, nil
} else {
return nil, fmt.Errorf("failed to generate RunDetai with templateType %s", templateType)
Bobgy marked this conversation as resolved.
Show resolved Hide resolved
}
}

func (r *ResourceManager) ToModelJob(job *api.Job, swf *util.ScheduledWorkflow, workflowSpecManifest string) (*model.Job, error) {
Expand Down
99 changes: 58 additions & 41 deletions backend/src/apiserver/resource/model_converter_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,10 @@
package resource

import (
"github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1"
"strings"
"testing"

"github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1"
"github.com/golang/protobuf/ptypes/timestamp"
"github.com/google/go-cmp/cmp"
api "github.com/kubeflow/pipelines/backend/api/go_client"
Expand Down Expand Up @@ -150,51 +150,68 @@ func TestToModelRunDetail(t *testing.T) {
store, manager, experiment := initWithExperiment(t)
defer store.Close()

apiRun := &api.Run{
Id: "run1",
Name: "name1",
Description: "this is a run",
PipelineSpec: &api.PipelineSpec{
Parameters: []*api.Parameter{{Name: "param2", Value: "world"}},
},
ResourceReferences: []*api.ResourceReference{
{Key: &api.ResourceKey{Type: api.ResourceType_EXPERIMENT, Id: experiment.UUID}, Relationship: api.Relationship_OWNER},
},
}
workflow := util.NewWorkflow(&v1alpha1.Workflow{
ObjectMeta: v1.ObjectMeta{Name: "workflow-name", UID: "123"},
Status: v1alpha1.WorkflowStatus{Phase: "running"},
})
modelRunDetail, err := manager.ToModelRunDetail(apiRun, "123", workflow, "workflow spec")
assert.Nil(t, err)

expectedModelRunDetail := &model.RunDetail{
Run: model.Run{
UUID: "123",
ExperimentUUID: experiment.UUID,
DisplayName: "name1",
Name: "workflow-name",
Conditions: "running",
Description: "this is a run",
PipelineSpec: model.PipelineSpec{
WorkflowSpecManifest: "workflow spec",
Parameters: `[{"name":"param2","value":"world"}]`,
// TODO: Add UT for V2 test case
tests := []struct {
apiRun *api.Run
workflow *util.Workflow
templateType util.TemplateType
expectedModelRunDetail *model.RunDetail
}{
{
apiRun: &api.Run{
Id: "run1",
Name: "name1",
Description: "this is a run",
PipelineSpec: &api.PipelineSpec{
Parameters: []*api.Parameter{{Name: "param2", Value: "world"}},
},
ResourceReferences: []*model.ResourceReference{
ResourceReferences: []*api.ResourceReference{
{
ResourceUUID: "123",
ResourceType: common.Run,
ReferenceUUID: experiment.UUID,
ReferenceName: experiment.Name,
ReferenceType: common.Experiment,
Relationship: common.Owner},
Key: &api.ResourceKey{Type: api.ResourceType_EXPERIMENT, Id: experiment.UUID},
Relationship: api.Relationship_OWNER},},
},
workflow: util.NewWorkflow(&v1alpha1.Workflow{
ObjectMeta: v1.ObjectMeta{Name: "workflow-name", UID: "123"},
Status:v1alpha1.WorkflowStatus{Phase: "running"},
}),
templateType: util.V1,
expectedModelRunDetail: &model.RunDetail{
Run: model.Run{
UUID: "123",
ExperimentUUID: experiment.UUID,
DisplayName: "name1",
Name: "workflow-name",
Conditions: "running",
Description: "this is a run",
PipelineSpec: model.PipelineSpec{
WorkflowSpecManifest: "workflow spec",
Parameters: `[{"name":"param2","value":"world"}]`,
},
ResourceReferences: []*model.ResourceReference{
{
ResourceUUID: "123",
ResourceType: common.Run,
ReferenceUUID: experiment.UUID,
ReferenceName: experiment.Name,
ReferenceType: common.Experiment,
Relationship: common.Owner},
},
},
PipelineRuntime: model.PipelineRuntime{
WorkflowRuntimeManifest: util.NewWorkflow(&v1alpha1.Workflow{
ObjectMeta: v1.ObjectMeta{Name: "workflow-name", UID: "123"},
Status:v1alpha1.WorkflowStatus{Phase: "running"},
}).ToStringForStore(),
},
},
},
PipelineRuntime: model.PipelineRuntime{
WorkflowRuntimeManifest: workflow.ToStringForStore(),
},
}
assert.Equal(t, expectedModelRunDetail, modelRunDetail)
for _, tt := range tests {
modelRunDetail, err := manager.ToModelRunDetail(tt.apiRun, "123", tt.workflow, "workflow spec", tt.templateType)
assert.Nil(t, err)
assert.Equal(t, tt.expectedModelRunDetail, modelRunDetail)
}

}

func TestToModelJob(t *testing.T) {
Expand Down
Loading