Skip to content

Commit

Permalink
Merge branch 'main' into feat/circleci-plugin-incremental
Browse files Browse the repository at this point in the history
  • Loading branch information
Nickcw6 authored Sep 18, 2024
2 parents 2b59a47 + 65e6256 commit bca1883
Show file tree
Hide file tree
Showing 185 changed files with 3,137 additions and 2,008 deletions.
1 change: 1 addition & 0 deletions backend/core/models/domainlayer/devops/cicd_pipeline.go
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ type CICDPipeline struct {
Environment string `gorm:"type:varchar(255)"`
TaskDatesInfo
CicdScopeId string `gorm:"index;type:varchar(255)"`
IsChild bool
}

func (CICDPipeline) TableName() string {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
/*
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package migrationscripts

import (
"github.com/apache/incubator-devlake/core/context"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
)

var _ plugin.MigrationScript = (*addIsChildToCicdPipeline)(nil)

type cicdPipeline20240906 struct {
IsChild bool
}

func (cicdPipeline20240906) TableName() string {
return "cicd_pipelines"
}

type addIsChildToCicdPipeline struct{}

func (*addIsChildToCicdPipeline) Up(basicRes context.BasicRes) errors.Error {
db := basicRes.GetDal()
if err := db.AutoMigrate(&cicdPipeline20240906{}); err != nil {
return err
}
return nil
}

func (*addIsChildToCicdPipeline) Version() uint64 {
return 20240906120000
}

func (*addIsChildToCicdPipeline) Name() string {
return "add is_child to cicd_pipelines"
}
1 change: 1 addition & 0 deletions backend/core/models/migrationscripts/register.go
Original file line number Diff line number Diff line change
Expand Up @@ -132,5 +132,6 @@ func All() []plugin.MigrationScript {
new(addDateFieldsToProjectPrMetric),
new(addAssigneeToIncident),
new(addIsSubtaskToIssue),
new(addIsChildToCicdPipeline),
}
}
1 change: 1 addition & 0 deletions backend/core/plugin/plugin_blueprint.go
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,7 @@ type DataSourcePluginBlueprintV200 interface {
MakeDataSourcePipelinePlanV200(
connectionId uint64,
scopes []*models.BlueprintScope,
skipCollectors bool,
) (models.PipelinePlan, []Scope, errors.Error)
}

Expand Down
23 changes: 11 additions & 12 deletions backend/core/runner/run_task.go
Original file line number Diff line number Diff line change
Expand Up @@ -244,11 +244,11 @@ func RunPluginSubTasks(
defer closeablePlugin.Close(taskCtx)
}
options := task.Options
taskCtx.SetSyncPolicy(syncPolicy)
taskData, err := pluginTask.PrepareTaskData(taskCtx, options)
if err != nil {
return errors.Default.Wrap(err, fmt.Sprintf("error preparing task data for %s", task.Plugin))
}
taskCtx.SetSyncPolicy(syncPolicy)
taskCtx.SetData(taskData)

// record subtasks sequence to DB
Expand Down Expand Up @@ -303,6 +303,7 @@ func RunPluginSubTasks(
// subtask was disabled
continue
}
subtaskNumber++
// run subtask
if progress != nil {
progress <- plugin.RunningProgress{
Expand Down Expand Up @@ -369,19 +370,17 @@ func UpdateProgressDetail(basicRes context.BasicRes, taskId uint64, progressDeta
progressDetail.TotalRecords = p.Total
case plugin.SubTaskIncProgress:
progressDetail.FinishedRecords = p.Current
// update subtask progress
where := dal.Where("task_id = ? and name = ?", taskId, progressDetail.SubTaskName)
err := basicRes.GetDal().UpdateColumns(subtask, []dal.DalSet{
{ColumnName: "finished_records", Value: progressDetail.FinishedRecords},
}, where)
if err != nil {
basicRes.GetLogger().Error(err, "failed to update _devlake_subtasks progress")
}
case plugin.SetCurrentSubTask:
progressDetail.SubTaskName = p.SubTaskName
progressDetail.SubTaskNumber = p.SubTaskNumber
default:
return
}
// update subtask progress
where := dal.Where("task_id = ? and name = ?", taskId, progressDetail.SubTaskName)
err := basicRes.GetDal().UpdateColumns(subtask, []dal.DalSet{
{ColumnName: "finished_records", Value: progressDetail.FinishedRecords},
}, where)
if err != nil {
basicRes.GetLogger().Error(err, "failed to update _devlake_subtasks progress")
}
}

Expand Down Expand Up @@ -417,7 +416,7 @@ func recordSubtask(basicRes context.BasicRes, subtask *models.Subtask) {
{ColumnName: "began_at", Value: subtask.BeganAt},
{ColumnName: "finished_at", Value: subtask.FinishedAt},
{ColumnName: "spent_seconds", Value: subtask.SpentSeconds},
{ColumnName: "finished_records", Value: subtask.FinishedRecords},
//{ColumnName: "finished_records", Value: subtask.FinishedRecords}, // FinishedRecords is zero always.
{ColumnName: "number", Value: subtask.Number},
}, where); err != nil {
basicRes.GetLogger().Error(err, "error writing subtask %d status to DB: %v", subtask.ID)
Expand Down
4 changes: 2 additions & 2 deletions backend/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ require (
github.com/go-openapi/swag v0.22.3 // indirect
github.com/go-playground/locales v0.14.1 // indirect
github.com/go-playground/universal-translator v0.18.1 // indirect
github.com/goccy/go-json v0.10.2 // indirect
github.com/goccy/go-json v0.10.3 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/protobuf v1.5.2 // indirect
Expand Down Expand Up @@ -108,7 +108,7 @@ require (
github.com/xanzy/ssh-agent v0.3.3 // indirect
golang.org/x/arch v0.7.0 // indirect
golang.org/x/net v0.28.0 // indirect
golang.org/x/sys v0.23.0 // indirect
golang.org/x/sys v0.24.0 // indirect
golang.org/x/text v0.17.0 // indirect
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect
google.golang.org/appengine v1.6.7 // indirect
Expand Down
19 changes: 15 additions & 4 deletions backend/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -49,11 +49,13 @@ github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5
github.com/ProtonMail/go-crypto v1.0.0 h1:LRuvITjQWX+WIfr930YHG2HNfjR1uOfyf5vE0kC2U78=
github.com/ProtonMail/go-crypto v1.0.0/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0=
github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8=
github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4=
github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM=
github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0=
Expand Down Expand Up @@ -116,6 +118,7 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.m
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY=
github.com/frankban/quicktest v1.14.4/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI=
github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU=
Expand All @@ -127,18 +130,21 @@ github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeME
github.com/gin-contrib/cors v1.6.0 h1:0Z7D/bVhE6ja07lI8CTjTonp6SB07o8bNuFyRbsBUQg=
github.com/gin-contrib/cors v1.6.0/go.mod h1:cI+h6iOAyxKRtUtC6iF/Si1KSFvGm/gK+kshxlCi8ro=
github.com/gin-contrib/gzip v0.0.6 h1:NjcunTcGAj5CO1gn4N8jHOSIeRFHIbn51z6K+xaN4d4=
github.com/gin-contrib/gzip v0.0.6/go.mod h1:QOJlmV2xmayAjkNS2Y8NQsMneuRShOU/kjovCXNuzzk=
github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE=
github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg=
github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU=
github.com/gliderlabs/ssh v0.3.7 h1:iV3Bqi942d9huXnzEF2Mt+CY9gLu8DNM4Obd+8bODRE=
github.com/gliderlabs/ssh v0.3.7/go.mod h1:zpHEXBstFnQYtGnB8k8kQLol82umzn/2/snG7alWVD8=
github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA=
github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og=
github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI=
github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic=
github.com/go-git/go-billy/v5 v5.5.0 h1:yEY4yhzCDuMGSv83oGxiBotRzhwhNr8VZyphhiu+mTU=
github.com/go-git/go-billy/v5 v5.5.0/go.mod h1:hmexnoNsr2SJU1Ju67OaNz5ASJY3+sHgFRpCtpDCKow=
github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 h1:eMje31YglSBqCdIqdhKBW8lokaMrL3uTkpGYlE2OOT4=
github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399/go.mod h1:1OCfN199q1Jm3HZlxleg+Dw/mwps2Wbk9frAWm+4FII=
github.com/go-git/go-git/v5 v5.12.0 h1:7Md+ndsjrzZxbddRDZjF14qK+NN56sy6wkqaVrjZtys=
github.com/go-git/go-git/v5 v5.12.0/go.mod h1:FTM9VKtnI2m65hNI/TenDDDnUf2Q9FHnXYjuz9i5OEY=
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
Expand All @@ -158,6 +164,7 @@ github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/
github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g=
github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14=
github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s=
github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA=
github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY=
github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY=
Expand All @@ -171,8 +178,8 @@ github.com/go-sql-driver/mysql v1.7.1/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/gocarina/gocsv v0.0.0-20220707092902-b9da1f06c77e h1:GMIV+S6grz+vlIaUsP+fedQ6L+FovyMPMY26WO8dwQE=
github.com/gocarina/gocsv v0.0.0-20220707092902-b9da1f06c77e/go.mod h1:5YoVOkjYAQumqlV356Hj3xeYh4BdZuLE0/nRkf2NKkI=
github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA=
github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
Expand Down Expand Up @@ -227,6 +234,7 @@ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
Expand Down Expand Up @@ -414,6 +422,7 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI=
github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M=
github.com/panjf2000/ants/v2 v2.4.6 h1:drmj9mcygn2gawZ155dRbo+NfXEfAssjZNU1qoIb4gQ=
github.com/panjf2000/ants/v2 v2.4.6/go.mod h1:f6F0NZVFsGCp5A7QW/Zj/m92atWwOkY0OIhFxRNFr4A=
github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
Expand All @@ -422,6 +431,7 @@ github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCko
github.com/pelletier/go-toml/v2 v2.1.1 h1:LWAJwfNvjQZCFIDKWYQaM62NcYeYViCmWIwmOStowAI=
github.com/pelletier/go-toml/v2 v2.1.1/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc=
github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4=
github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
github.com/pjbgf/sha1cd v0.3.0 h1:4D5XXmUUBUl/xQ6IjCkEAbqXskkq/4O7LmGn0AqMDs4=
github.com/pjbgf/sha1cd v0.3.0/go.mod h1:nZ1rrWOcGJ5uZgEEVL1VUM9iRQiZvWdbZjkKyFzPPsI=
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
Expand Down Expand Up @@ -741,15 +751,16 @@ golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.23.0 h1:YfKFowiIMvtgl1UERQoTPPToxltDeZfbj4H7dVUCwmM=
golang.org/x/sys v0.23.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg=
golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc=
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=
golang.org/x/term v0.23.0 h1:F6D4vR+EHoL9/sWAWgAR1H2DcHr4PareCbAaCo1RpuU=
golang.org/x/term v0.23.0/go.mod h1:DgV24QBUrK6jhZXl+20l6UWznPlwAHm1Q1mGHtydmSk=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
Expand Down
15 changes: 11 additions & 4 deletions backend/plugins/ae/impl/impl.go
Original file line number Diff line number Diff line change
Expand Up @@ -107,9 +107,14 @@ func (p AE) PrepareTaskData(taskCtx plugin.TaskContext, options map[string]inter
if err != nil {
return nil, errors.Default.Wrap(err, "error getting connection for AE plugin")
}
apiClient, err := tasks.CreateApiClient(taskCtx, connection)
if err != nil {
return nil, err
var apiClient *helper.ApiAsyncClient
syncPolicy := taskCtx.SyncPolicy()
if !syncPolicy.SkipCollectors {
newApiClient, err := tasks.CreateApiClient(taskCtx, connection)
if err != nil {
return nil, err
}
apiClient = newApiClient
}
return &tasks.AeTaskData{
Options: &op,
Expand Down Expand Up @@ -150,6 +155,8 @@ func (p AE) Close(taskCtx plugin.TaskContext) errors.Error {
if !ok {
return errors.Default.New(fmt.Sprintf("GetData failed when try to close %+v", taskCtx))
}
data.ApiClient.Release()
if data != nil && data.ApiClient != nil {
data.ApiClient.Release()
}
return nil
}
1 change: 1 addition & 0 deletions backend/plugins/azuredevops_go/api/blueprint_v200.go
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ func MakePipelinePlanV200(
subtaskMetas []plugin.SubTaskMeta,
connectionId uint64,
bpScopes []*coreModels.BlueprintScope,
skipCollectors bool,
) (coreModels.PipelinePlan, []plugin.Scope, errors.Error) {
// load connection, scope and scopeConfig from the db
connection, err := dsHelper.ConnSrv.FindByPk(connectionId)
Expand Down
18 changes: 13 additions & 5 deletions backend/plugins/azuredevops_go/impl/impl.go
Original file line number Diff line number Diff line change
Expand Up @@ -127,9 +127,14 @@ func (p Azuredevops) PrepareTaskData(taskCtx plugin.TaskContext, options map[str
return nil, errors.Default.Wrap(err, "failed to retrieve an Azure DevOps connection from the database using the provided connection ID")
}

apiClient, err := tasks.CreateApiClient(taskCtx, connection)
if err != nil {
return nil, errors.Default.Wrap(err, "failed to retrieve an Azure DevOps connection from the database using the provided connection ID")
var apiClient *helper.ApiAsyncClient
syncPolicy := taskCtx.SyncPolicy()
if !syncPolicy.SkipCollectors {
newApiClient, err := tasks.CreateApiClient(taskCtx, connection)
if err != nil {
return nil, errors.Default.Wrap(err, "failed to retrieve an Azure DevOps connection from the database using the provided connection ID")
}
apiClient = newApiClient
}

if op.RepositoryId != "" {
Expand Down Expand Up @@ -253,15 +258,18 @@ func (p Azuredevops) ApiResources() map[string]map[string]plugin.ApiResourceHand
func (p Azuredevops) MakeDataSourcePipelinePlanV200(
connectionId uint64,
scopes []*coreModels.BlueprintScope,
skipCollectors bool,
) (pp coreModels.PipelinePlan, sc []plugin.Scope, err errors.Error) {
return api.MakePipelinePlanV200(p.SubTaskMetas(), connectionId, scopes)
return api.MakePipelinePlanV200(p.SubTaskMetas(), connectionId, scopes, skipCollectors)
}

func (p Azuredevops) Close(taskCtx plugin.TaskContext) errors.Error {
data, ok := taskCtx.GetData().(*tasks.AzuredevopsTaskData)
if !ok {
return errors.Default.New(fmt.Sprintf("GetData failed when try to close %+v", taskCtx))
}
data.ApiClient.Release()
if data != nil && data.ApiClient != nil {
data.ApiClient.Release()
}
return nil
}
1 change: 1 addition & 0 deletions backend/plugins/bamboo/api/blueprint_v200.go
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ func MakePipelinePlanV200(
subtaskMetas []plugin.SubTaskMeta,
connectionId uint64,
bpScopes []*coreModels.BlueprintScope,
skipCollectors bool,
) (coreModels.PipelinePlan, []plugin.Scope, errors.Error) {
// load connection, scope and scopeConfig from the db
connection, err := dsHelper.ConnSrv.FindByPk(connectionId)
Expand Down
1 change: 1 addition & 0 deletions backend/plugins/bamboo/e2e/plan_build_commits_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ func TestBambooPlanBuildCommitsDataFlow(t *testing.T) {
},
},
ApiClient: getFakeAPIClient(),
EndPoint: getFakeAPIClient().GetEndpoint(),
}

dataflowTester.ImportCsvIntoTabler("./snapshot_tables/_tool_bamboo_plan_build_commits.csv", &models.BambooPlanBuildVcsRevision{})
Expand Down
5 changes: 3 additions & 2 deletions backend/plugins/bamboo/e2e/plan_build_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,8 @@ func TestBambooPlanBuildDataFlow(t *testing.T) {
dataflowTester.Subtask(tasks.ConvertPlanBuildsMeta, taskData)

dataflowTester.VerifyTableWithOptions(&devops.CICDPipeline{}, e2ehelper.TableOptions{
CSVRelPath: "./snapshot_tables/cicd_pipelines.csv",
IgnoreTypes: []interface{}{common.NoPKModel{}},
CSVRelPath: "./snapshot_tables/cicd_pipelines.csv",
IgnoreTypes: []interface{}{common.NoPKModel{}},
IgnoreFields: []string{"is_child"},
})
}
Loading

0 comments on commit bca1883

Please sign in to comment.