Skip to content

Commit

Permalink
Merge pull request #3711 from JesseStutler/master
Browse files Browse the repository at this point in the history 
fix: not remove podgroup uid will cause topology annotation to be useless
  • Loading branch information
@volcano-sh-bot
volcano-sh-bot authored Sep 14, 2024
2 parents 5d3106d + 07bafa6 commit a056b4d
Show file tree
Hide file tree
Showing 4 changed files with 68 additions and 41 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/e2e_spark.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ jobs:
kubectl describe node
- name: Upload Spark on K8S integration tests log files
if: failure()
uses: actions/upload-artifact@v2
uses: actions/upload-artifact@v4
with:
name: spark-on-kubernetes-it-log
path: |
Expand Down
10 changes: 2 additions & 8 deletions pkg/scheduler/plugins/task-topology/topology.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -247,15 +247,9 @@ func affinityCheck(job *api.JobInfo, affinity [][]string) error {

var taskNumber = len(job.Tasks)
var taskRef = make(map[string]bool, taskNumber)
var jobNamePrefix = job.Name + "-"
for _, task := range job.Tasks {
// the full task name looks like "${job name}-${task name}-${index}",
// so we can trim the jobNamePrefix and the indexSuffix to get the short task name.
tmpTaskName := task.Name[:strings.LastIndex(task.Name, "-")]
tmpTaskName = strings.TrimPrefix(tmpTaskName, jobNamePrefix)

if _, exist := taskRef[tmpTaskName]; !exist {
taskRef[tmpTaskName] = true
if _, exist := taskRef[task.TaskRole]; !exist {
taskRef[task.TaskRole] = true
}
}

Expand Down
96 changes: 64 additions & 32 deletions pkg/scheduler/plugins/task-topology/topology_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,22 +41,28 @@ func Test_readTopologyFromPgAnnotations(t *testing.T) {
Namespace: "default",
Tasks: map[api.TaskID]*api.TaskInfo{
"0": {
Name: "job1-ps-0",
Name: "job1-ps-0",
TaskRole: "ps",
},
"1": {
Name: "job1-ps-1",
Name: "job1-ps-1",
TaskRole: "ps",
},
"2": {
Name: "job1-worker-0",
Name: "job1-worker-0",
TaskRole: "worker",
},
"3": {
Name: "job1-worker-1",
Name: "job1-worker-1",
TaskRole: "worker",
},
"4": {
Name: "job1-chief-0",
Name: "job1-chief-0",
TaskRole: "chief",
},
"5": {
Name: "job1-evaluator-0",
Name: "job1-evaluator-0",
TaskRole: "evaluator",
},
},
PodGroup: &api.PodGroup{
Expand Down Expand Up @@ -107,22 +113,28 @@ func Test_readTopologyFromPgAnnotations(t *testing.T) {
Namespace: "default",
Tasks: map[api.TaskID]*api.TaskInfo{
"0": {
Name: "job1-ps-some-0",
Name: "job1-ps-some-0",
TaskRole: "ps-some",
},
"1": {
Name: "job1-ps-some-1",
Name: "job1-ps-some-1",
TaskRole: "ps-some",
},
"2": {
Name: "job1-worker-another-some-0",
Name: "job1-worker-another-some-0",
TaskRole: "worker-another-some",
},
"3": {
Name: "job1-worker-another-some-1",
Name: "job1-worker-another-some-1",
TaskRole: "worker-another-some",
},
"4": {
Name: "job1-chief-kk-0",
Name: "job1-chief-kk-0",
TaskRole: "chief-kk",
},
"5": {
Name: "job1-evaluator-tt-0",
Name: "job1-evaluator-tt-0",
TaskRole: "evaluator-tt",
},
},
PodGroup: &api.PodGroup{
Expand Down Expand Up @@ -187,22 +199,28 @@ func Test_readTopologyFromPgAnnotations(t *testing.T) {
Namespace: "default",
Tasks: map[api.TaskID]*api.TaskInfo{
"0": {
Name: "job1-ps-0",
Name: "job1-ps-0",
TaskRole: "ps",
},
"1": {
Name: "job1-ps-1",
Name: "job1-ps-1",
TaskRole: "ps",
},
"2": {
Name: "job1-worker-0",
Name: "job1-worker-0",
TaskRole: "worker",
},
"3": {
Name: "job1-worker-1",
Name: "job1-worker-1",
TaskRole: "worker",
},
"4": {
Name: "job1-chief-0",
Name: "job1-chief-0",
TaskRole: "chief",
},
"5": {
Name: "job1-evaluator-0",
Name: "job1-evaluator-0",
TaskRole: "evaluator",
},
},
PodGroup: &api.PodGroup{
Expand All @@ -227,16 +245,20 @@ func Test_readTopologyFromPgAnnotations(t *testing.T) {
Namespace: "default",
Tasks: map[api.TaskID]*api.TaskInfo{
"0": {
Name: "job1-ps-0",
Name: "job1-ps-0",
TaskRole: "ps",
},
"1": {
Name: "job1-ps-1",
Name: "job1-ps-1",
TaskRole: "ps",
},
"2": {
Name: "job1-worker-0",
Name: "job1-worker-0",
TaskRole: "worker",
},
"3": {
Name: "job1-worker-1",
Name: "job1-worker-1",
TaskRole: "worker",
},
},
PodGroup: &api.PodGroup{
Expand All @@ -261,16 +283,20 @@ func Test_readTopologyFromPgAnnotations(t *testing.T) {
Namespace: "default",
Tasks: map[api.TaskID]*api.TaskInfo{
"0": {
Name: "job1-ps-0",
Name: "job1-ps-0",
TaskRole: "ps",
},
"1": {
Name: "job1-ps-1",
Name: "job1-ps-1",
TaskRole: "ps",
},
"2": {
Name: "job1-worker-0",
Name: "job1-worker-0",
TaskRole: "worker",
},
"3": {
Name: "job1-worker-1",
Name: "job1-worker-1",
TaskRole: "worker",
},
},
PodGroup: &api.PodGroup{
Expand All @@ -295,22 +321,28 @@ func Test_readTopologyFromPgAnnotations(t *testing.T) {
Namespace: "default",
Tasks: map[api.TaskID]*api.TaskInfo{
"0": {
Name: "job1-ps-0",
Name: "job1-ps-0",
TaskRole: "ps",
},
"1": {
Name: "job1-ps-1",
Name: "job1-ps-1",
TaskRole: "ps",
},
"2": {
Name: "job1-worker-0",
Name: "job1-worker-0",
TaskRole: "worker",
},
"3": {
Name: "job1-worker-1",
Name: "job1-worker-1",
TaskRole: "worker",
},
"4": {
Name: "job1-chief-0",
Name: "job1-chief-0",
TaskRole: "chief",
},
"5": {
Name: "job1-evaluator-0",
Name: "job1-evaluator-0",
TaskRole: "evaluator",
},
},
PodGroup: &api.PodGroup{
Expand Down
1 change: 1 addition & 0 deletions test/e2e/schedulingaction/allocate.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,7 @@ var _ = ginkgo.Describe("Job E2E Test", func() {
})

ginkgo.It("allocate don't work when podgroup is pending", func() {
ginkgo.Skip("TODO: need to fix this test. It's useless to directly associate podgroup with job. The job will still create its own podgroup, so the resources are sufficient and j2 can be running.")
ctx := e2eutil.InitTestContext(e2eutil.Options{
NodesNumLimit: 2,
NodesResourceLimit: corev1.ResourceList{
Expand Down

0 comments on commit a056b4d

Please sign in to comment.