Skip to content

Commit

Permalink
feat: not update sub task progress if progress less than 1 pct
Browse files Browse the repository at this point in the history
[Refactor][core]Data inflation when using postgres apache#8142
  • Loading branch information
narrowizard committed Oct 22, 2024
1 parent e856b60 commit a0688e6
Showing 1 changed file with 16 additions and 8 deletions.
24 changes: 16 additions & 8 deletions backend/core/runner/run_task.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,10 +20,11 @@ package runner
import (
gocontext "context"
"fmt"
"github.com/apache/incubator-devlake/core/models/common"
"strings"
"time"

"github.com/apache/incubator-devlake/core/models/common"

"github.com/apache/incubator-devlake/core/context"
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
Expand Down Expand Up @@ -354,6 +355,7 @@ func UpdateProgressDetail(basicRes context.BasicRes, taskId uint64, progressDeta
Model: common.Model{ID: taskId},
}
subtask := &models.Subtask{}
originalFinishedRecords := progressDetail.FinishedRecords
switch p.Type {
case plugin.TaskSetProgress:
progressDetail.TotalSubTasks = p.Total
Expand All @@ -374,14 +376,20 @@ func UpdateProgressDetail(basicRes context.BasicRes, taskId uint64, progressDeta
progressDetail.SubTaskName = p.SubTaskName
progressDetail.SubTaskNumber = p.SubTaskNumber
}
// update subtask progress
where := dal.Where("task_id = ? and name = ?", taskId, progressDetail.SubTaskName)
err := basicRes.GetDal().UpdateColumns(subtask, []dal.DalSet{
{ColumnName: "finished_records", Value: progressDetail.FinishedRecords},
}, where)
if err != nil {
basicRes.GetLogger().Error(err, "failed to update _devlake_subtasks progress")
currentFinishedRecords := progressDetail.FinishedRecords
currentTotalRecords := progressDetail.TotalRecords
// do not update progress if progress less than 1%
if float64(currentFinishedRecords-originalFinishedRecords)*0.1/float64(currentTotalRecords) > 0.01 {
// update subtask progress
where := dal.Where("task_id = ? and name = ?", taskId, progressDetail.SubTaskName)
err := basicRes.GetDal().UpdateColumns(subtask, []dal.DalSet{
{ColumnName: "finished_records", Value: progressDetail.FinishedRecords},
}, where)
if err != nil {
basicRes.GetLogger().Error(err, "failed to update _devlake_subtasks progress")
}
}

}

func runSubtask(
Expand Down

0 comments on commit a0688e6

Please sign in to comment.