Skip to content

Commit

Permalink
Merge branch 'main' into feat/circleci-plugin-incremental
Browse files Browse the repository at this point in the history
  • Loading branch information
Nickcw6 authored Oct 9, 2024
2 parents 93f1170 + 0b8c8ea commit a9aa7dd
Show file tree
Hide file tree
Showing 281 changed files with 5,354 additions and 2,866 deletions.
4 changes: 2 additions & 2 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,8 @@
# https://stackoverflow.com/questions/920413/make-error-missing-separator
# https://tutorialedge.net/golang/makefiles-for-go-developers/

SHA := $(shell if [ -d .git ]; then git show -s --format=%h; else echo "default_SHA"; fi)
TAG := $(shell if [ -d .git ]; then git tag --points-at HEAD; else echo "default_TAG"; fi)
SHA ?= $(shell git show -s --format=%h 2>/dev/null || echo "default_SHA")
TAG ?= $(shell git tag --points-at HEAD 2>/dev/null || echo "default_TAG")
IMAGE_REPO ?= "apache"
VERSION = $(TAG)@$(SHA)

Expand Down
5 changes: 2 additions & 3 deletions backend/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -17,13 +17,12 @@
# https://stackoverflow.com/questions/920413/make-error-missing-separator
# https://tutorialedge.net/golang/makefiles-for-go-developers/

SHA := $(shell if [ -d .git ]; then git show -s --format=%h; else echo "default_SHA"; fi)
TAG := $(shell if [ -d .git ]; then git tag --points-at HEAD; else echo "default_TAG"; fi)
SHA ?= $(shell git show -s --format=%h 2>/dev/null || echo "default_SHA")
TAG ?= $(shell git tag --points-at HEAD 2>/dev/null || echo "default_TAG")
IMAGE_REPO ?= "apache"
VERSION = $(TAG)@$(SHA)
PYTHON_DIR ?= "./python"


all: build

go-dep:
Expand Down
9 changes: 9 additions & 0 deletions backend/core/models/blueprint.go
Original file line number Diff line number Diff line change
Expand Up @@ -92,3 +92,12 @@ type SyncPolicy struct {
TimeAfter *time.Time `json:"timeAfter"`
TriggerSyncPolicy
}

type ConnectionTokenCheckResult struct {
PluginName string `json:"pluginName" mapstructure:"pluginName"`
ConnectionID uint64 `json:"connectionId" mapstructure:"connectionId"`
Success bool `json:"success" mapstructure:"success"`
Message string `json:"message" mapstructure:"message"`
}

type ApiBlueprintConnectionTokenCheck []ConnectionTokenCheckResult
2 changes: 1 addition & 1 deletion backend/core/models/domainlayer/codequality/cq_issues.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ type CqIssue struct {
domainlayer.DomainEntity
Rule string `gorm:"type:varchar(255)"`
Severity string `gorm:"type:varchar(100)"`
Component string `gorm:"type:varchar(255)"`
Component string
ProjectKey string `gorm:"index;type:varchar(100)"` //domain project key
Line int
Status string `gorm:"type:varchar(20)"`
Expand Down
10 changes: 10 additions & 0 deletions backend/core/models/domainlayer/devops/cicd_task.go
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,16 @@ const (

const ENV_NAME_PATTERN = "ENV_NAME_PATTERN"

type TransformDeployment struct {
Name string `json:"name"`
URL string `json:"url"`
}

type TransformDeploymentResponse struct {
Total int `json:"total"`
Data []TransformDeployment `json:"data"`
}

type CICDTask struct {
domainlayer.DomainEntity
Name string `gorm:"type:varchar(255)"`
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
/*
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package migrationscripts

import (
"github.com/apache/incubator-devlake/core/context"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
)

var _ plugin.MigrationScript = (*increaseCqIssueComponentLength)(nil)

type increaseCqIssueComponentLength struct{}

func (script *increaseCqIssueComponentLength) Up(basicRes context.BasicRes) errors.Error {
return basicRes.GetDal().ModifyColumnType("cq_issues", "component", "text")
}

func (*increaseCqIssueComponentLength) Version() uint64 {
return 20240919160242
}

func (*increaseCqIssueComponentLength) Name() string {
return "increase cq_issues.component length to text"
}
1 change: 1 addition & 0 deletions backend/core/models/migrationscripts/register.go
Original file line number Diff line number Diff line change
Expand Up @@ -133,5 +133,6 @@ func All() []plugin.MigrationScript {
new(addAssigneeToIncident),
new(addIsSubtaskToIssue),
new(addIsChildToCicdPipeline),
new(increaseCqIssueComponentLength),
}
}
5 changes: 4 additions & 1 deletion backend/core/models/task.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,10 @@ const (
TASK_PARTIAL = "TASK_PARTIAL"
)

var PendingTaskStatus = []string{TASK_CREATED, TASK_RERUN, TASK_RUNNING}
var (
PendingTaskStatus = []string{TASK_CREATED, TASK_RERUN, TASK_RUNNING}
FinishedTaskStatus = []string{TASK_PARTIAL, TASK_CANCELLED, TASK_FAILED, TASK_COMPLETED}
)

type TaskProgressDetail struct {
TotalSubTasks int `json:"totalSubTasks"`
Expand Down
4 changes: 4 additions & 0 deletions backend/core/plugin/plugin_api.go
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,10 @@ type PluginApi interface {
ApiResources() map[string]map[string]ApiResourceHandler
}

type PluginTestConnectionAPI interface {
TestConnection(id uint64) errors.Error
}

const wrapResponseError = "WRAP_RESPONSE_ERROR"

func WrapTestConnectionErrResp(basicRes context.BasicRes, err errors.Error) errors.Error {
Expand Down
10 changes: 10 additions & 0 deletions backend/core/plugin/plugin_blueprint.go
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,16 @@ type ProjectMapper interface {
MapProject(projectName string, scopes []Scope) (models.PipelinePlan, errors.Error)
}

type ProjectTokenCheckerConnection struct {
PluginName string
ConnectionId uint64
}

// ProjectTokenChecker is implemented by the plugin org, which generate a task tp check all connection's tokens
type ProjectTokenChecker interface {
MakePipeline(skipCollectors bool, projectName string, scopes []ProjectTokenCheckerConnection) (models.PipelinePlan, errors.Error)
}

// CompositeDataSourcePluginBlueprintV200 is for unit test
type CompositeDataSourcePluginBlueprintV200 interface {
PluginMeta
Expand Down
48 changes: 36 additions & 12 deletions backend/helpers/pluginhelper/api/api_extractor_stateful.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ limitations under the License.
package api

import (
"encoding/json"
"reflect"

"github.com/apache/incubator-devlake/core/dal"
Expand All @@ -27,9 +28,10 @@ import (
)

// StatefulApiExtractorArgs is a struct that contains the arguments for a stateful api extractor
type StatefulApiExtractorArgs struct {
type StatefulApiExtractorArgs[InputType any] struct {
*SubtaskCommonArgs
Extract func(row *RawData) ([]any, errors.Error)
BeforeExtract func(issue *InputType, stateManager *SubtaskStateManager) errors.Error
Extract func(body *InputType, row *RawData) ([]any, errors.Error)
}

// StatefulApiExtractor is a struct that manages the stateful API extraction process.
Expand All @@ -41,7 +43,7 @@ type StatefulApiExtractorArgs struct {
//
// Example:
//
// extractor, err := api.NewStatefulApiExtractor(&api.StatefulApiExtractorArgs{
// extractor, err := api.NewStatefulApiExtractor(&api.StatefulApiExtractorArgs[apiv2models.Issue]{
// SubtaskCommonArgs: &api.SubtaskCommonArgs{
// SubTaskContext: subtaskCtx,
// Table: RAW_ISSUE_TABLE,
Expand All @@ -54,8 +56,17 @@ type StatefulApiExtractorArgs struct {
// // Ensure that the configuration is serializable and contains only public fields.
// // It is also recommended that the configuration includes only the necessary fields used by the extractor.
// ..},
// Extract: func(row *api.RawData) ([]interface{}, errors.Error) {
// return extractIssues(data, config, row, userFieldMap)
// BeforeExtract: func(body *IssuesResponse, stateManager *api.SubtaskStateManager) errors.Error {
// if stateManager.IsIncremental() {
// // It is important to delete all existing child-records under DiffSync Mode
// err := db.Delete(
// &models.JiraIssueLabel{},
// dal.Where("connection_id = ? AND issue_id = ?", data.Options.ConnectionId, body.Id),
// )
// }
// return nil
// },
// Extract: func(apiIssue *apiv2models.Issue, row *api.RawData) ([]interface{}, errors.Error) {
// },
// })
//
Expand All @@ -64,25 +75,25 @@ type StatefulApiExtractorArgs struct {
// }
//
// return extractor.Execute()
type StatefulApiExtractor struct {
*StatefulApiExtractorArgs
type StatefulApiExtractor[InputType any] struct {
*StatefulApiExtractorArgs[InputType]
*SubtaskStateManager
}

// NewStatefulApiExtractor creates a new StatefulApiExtractor
func NewStatefulApiExtractor(args *StatefulApiExtractorArgs) (*StatefulApiExtractor, errors.Error) {
func NewStatefulApiExtractor[InputType any](args *StatefulApiExtractorArgs[InputType]) (*StatefulApiExtractor[InputType], errors.Error) {
stateManager, err := NewSubtaskStateManager(args.SubtaskCommonArgs)
if err != nil {
return nil, err
}
return &StatefulApiExtractor{
return &StatefulApiExtractor[InputType]{
StatefulApiExtractorArgs: args,
SubtaskStateManager: stateManager,
}, nil
}

// Execute sub-task
func (extractor *StatefulApiExtractor) Execute() errors.Error {
func (extractor *StatefulApiExtractor[InputType]) Execute() errors.Error {
// load data from database
db := extractor.GetDal()
logger := extractor.GetLogger()
Expand Down Expand Up @@ -135,7 +146,20 @@ func (extractor *StatefulApiExtractor) Execute() errors.Error {
return errors.Default.Wrap(err, "error fetching row")
}

results, err := extractor.Extract(row)
body := new(InputType)
err = errors.Convert(json.Unmarshal(row.Data, body))
if err != nil {
return err
}

if extractor.BeforeExtract != nil {
err = extractor.BeforeExtract(body, extractor.SubtaskStateManager)
if err != nil {
return err
}
}

results, err := extractor.Extract(body, row)
if err != nil {
return errors.Default.Wrap(err, "error calling plugin Extract implementation")
}
Expand Down Expand Up @@ -169,4 +193,4 @@ func (extractor *StatefulApiExtractor) Execute() errors.Error {
return extractor.SubtaskStateManager.Close()
}

var _ plugin.SubTask = (*StatefulApiExtractor)(nil)
var _ plugin.SubTask = (*StatefulApiExtractor[any])(nil)
2 changes: 1 addition & 1 deletion backend/helpers/pluginhelper/api/api_rawdata.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ type RawData struct {
Data []byte
Url string
Input json.RawMessage `gorm:"type:json"`
CreatedAt time.Time
CreatedAt time.Time `gorm:"index"`
}

type TaskOptions interface {
Expand Down
26 changes: 21 additions & 5 deletions backend/helpers/pluginhelper/api/data_convertor_stateful.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,10 @@ import (

type StatefulDataConverterArgs[InputType any] struct {
*SubtaskCommonArgs
Input func(*SubtaskStateManager) (dal.Rows, errors.Error)
Convert func(row *InputType) ([]any, errors.Error)
BatchSize int
Input func(*SubtaskStateManager) (dal.Rows, errors.Error)
BeforeConvert func(issue *InputType, stateManager *SubtaskStateManager) errors.Error
Convert func(row *InputType) ([]any, errors.Error)
BatchSize int
}

// StatefulDataConverter is a struct that manages the stateful data conversion process.
Expand All @@ -47,7 +48,7 @@ type StatefulDataConverterArgs[InputType any] struct {
//
// Example:
//
// converter, err := api.NewStatefulDataConverter[models.JiraIssue](&api.StatefulDataConverterArgs[models.JiraIssue]{
// converter, err := api.NewStatefulDataConverter(&api.StatefulDataConverterArgs[models.JiraIssue]{
// SubtaskCommonArgs: &api.SubtaskCommonArgs{
// SubTaskContext: subtaskCtx,
// Table: RAW_ISSUE_TABLE,
Expand Down Expand Up @@ -78,6 +79,15 @@ type StatefulDataConverterArgs[InputType any] struct {
// }
// return db.Cursor(clauses...)
// },
// BeforeConvert: func(jiraIssue *models.GitlabMergeRequest, stateManager *api.SubtaskStateManager) errors.Error {
// // It is important to delete all existing child-records under DiffSync Mode
// issueId := issueIdGen.Generate(data.Options.ConnectionId, jiraIssue.IssueId)
// if err := db.Delete(&ticket.IssueAssignee{}, dal.Where("issue_id = ?", issueId)); err != nil {
// return err
// }
// ...
// return nil
// },
// Convert: func(jiraIssue *models.JiraIssue) ([]interface{}, errors.Error) {
// },
// })
Expand All @@ -93,7 +103,6 @@ type StatefulDataConverter[InputType any] struct {
}

func NewStatefulDataConverter[
OptType any,
InputType any,
](
args *StatefulDataConverterArgs[InputType],
Expand Down Expand Up @@ -146,6 +155,13 @@ func (converter *StatefulDataConverter[InputType]) Execute() errors.Error {
return errors.Default.Wrap(err, "error fetching rows")
}

if converter.BeforeConvert != nil {
err = converter.BeforeConvert(inputRow, converter.SubtaskStateManager)
if err != nil {
return err
}
}

results, err := converter.Convert(inputRow)
if err != nil {
return errors.Default.Wrap(err, "error calling Converter plugin implementation")
Expand Down
Loading

0 comments on commit a9aa7dd

Please sign in to comment.