feat(sonarqube): add bp v100 support

backend/plugins/sonarqube/api/blueprint_v200.go

@@ -27,7 +27,11 @@ import (
 	"github.com/apache/incubator-devlake/core/plugin"
 	"github.com/apache/incubator-devlake/core/utils"
 	helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+	aha "github.com/apache/incubator-devlake/helpers/pluginhelper/api/apihelperabstract"
 	"github.com/apache/incubator-devlake/plugins/sonarqube/models"
+	"github.com/apache/incubator-devlake/plugins/sonarqube/tasks"
+	"net/http"
+	"net/url"
 	"time"
 )
 
@@ -104,3 +108,29 @@ func makeScopesV200(bpScopes []*plugin.BlueprintScopeV200, connectionId uint64)
 	}
 	return scopes, nil
 }
+
+func GetApiProject(
+	projectKey string,
+	apiClient aha.ApiClientAbstract,
+) (*tasks.SonarqubeApiProject, errors.Error) {
+	var resData struct {
+		Data []tasks.SonarqubeApiProject `json:"components"`
+	}
+	query := url.Values{}
+	query.Set("q", projectKey)
+	res, err := apiClient.Get("projects/search", query, nil)
+	if err != nil {
+		return nil, err
+	}
+	if res.StatusCode != http.StatusOK {
+		return nil, errors.HttpStatus(res.StatusCode).New(fmt.Sprintf("unexpected status code when requesting project detail from %s", res.Request.URL.String()))
+	}
+	err = helper.UnmarshalResponse(res, &resData)
+	if err != nil {
+		return nil, err
+	}
+	if len(resData.Data) > 0 {
+		return &resData.Data[0], nil
+	}
+	return nil, errors.BadInput.New(fmt.Sprintf("Cannot find project: %s", projectKey))
+}

backend/plugins/sonarqube/e2e/project_test.go

@@ -33,36 +33,19 @@ func TestSonarqubeProjectDataFlow(t *testing.T) {
 	var sonarqube impl.Sonarqube
 	dataflowTester := e2ehelper.NewDataFlowTester(t, "sonarqube", sonarqube)
 
-	taskData := &tasks.SonarqubeTaskData{
-		Options: &tasks.SonarqubeOptions{
-			ConnectionId: 1,
-			ProjectKey:   "fa2cf9cd-c448-4fc3-99a5-1c893f15d84c",
-		},
-	}
-
 	// import raw data table
-	dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_sonarqube_projects.csv",
-		"_raw_sonarqube_projects")
+	dataflowTester.ImportCsvIntoTabler("./raw_tables/_tool_sonarqube_projects.csv",
+		&models.SonarqubeProject{})
 
-	// verify extraction
-	dataflowTester.FlushTabler(&models.SonarqubeProject{})
-	dataflowTester.Subtask(tasks.ExtractProjectsMeta, taskData)
-
-	taskData2 := &tasks.SonarqubeTaskData{
+	taskData := &tasks.SonarqubeTaskData{
 		Options: &tasks.SonarqubeOptions{
 			ConnectionId: 2,
 			ProjectKey:   "e2c6d5e9-a321-4e8c-b322-03d9599ef962",
 		},
 	}
 
-	dataflowTester.Subtask(tasks.ExtractProjectsMeta, taskData2)
-	dataflowTester.VerifyTableWithOptions(&models.SonarqubeProject{}, e2ehelper.TableOptions{
-		CSVRelPath:  "./snapshot_tables/_tool_sonarqube_projects.csv",
-		IgnoreTypes: []interface{}{common.NoPKModel{}},
-	})
-
 	dataflowTester.FlushTabler(&codequality.CqProject{})
-	dataflowTester.Subtask(tasks.ConvertProjectsMeta, taskData2)
+	dataflowTester.Subtask(tasks.ConvertProjectsMeta, taskData)
 	dataflowTester.VerifyTableWithOptions(&codequality.CqProject{}, e2ehelper.TableOptions{
 		CSVRelPath:  "./snapshot_tables/projects.csv",
 		IgnoreTypes: []interface{}{common.NoPKModel{}},

backend/plugins/sonarqube/impl/impl.go

@@ -82,8 +82,6 @@ func (p Sonarqube) GetTablesInfo() []dal.Tabler {
 
 func (p Sonarqube) SubTaskMetas() []plugin.SubTaskMeta {
 	return []plugin.SubTaskMeta{
-		tasks.CollectProjectsMeta,
-		tasks.ExtractProjectsMeta,
 		tasks.CollectIssuesMeta,
 		tasks.ExtractIssuesMeta,
 		tasks.CollectHotspotsMeta,
@@ -125,6 +123,30 @@ func (p Sonarqube) PrepareTaskData(taskCtx plugin.TaskContext, options map[strin
 		Options:   op,
 		ApiClient: apiClient,
 	}
+	if op.ProjectKey != "" {
+		var scope *models.SonarqubeProject
+		var apiProject *tasks.SonarqubeApiProject
+		// support v100 & advance mode
+		// If we still cannot find the record in db, we have to request from remote server and save it to db
+		db := taskCtx.GetDal()
+		err = db.First(&scope, dal.Where("connection_id = ? AND project_key = ?", op.ConnectionId, op.ProjectKey))
+		if err != nil && db.IsErrorNotFound(err) {
+			apiProject, err = api.GetApiProject(op.ProjectKey, apiClient)
+			if err != nil {
+				return nil, err
+			}
+			logger.Debug(fmt.Sprintf("Current project: %s", apiProject.ProjectKey))
+			scope = tasks.ConvertProject(apiProject)
+			scope.ConnectionId = op.ConnectionId
+			err = taskCtx.GetDal().CreateIfNotExist(&scope)
+			if err != nil {
+				return nil, err
+			}
+		}
+		if err != nil {
+			return nil, errors.Default.Wrap(err, fmt.Sprintf("fail to find project: %s", op.ProjectKey))
+		}
+	}
 	var createdDateAfter time.Time
 	if op.CreatedDateAfter != "" {
 		createdDateAfter, err = errors.Convert01(time.Parse(time.RFC3339, op.CreatedDateAfter))

backend/plugins/sonarqube/tasks/filemetrics_convertor.go

@@ -29,6 +29,8 @@ import (
 	"reflect"
 )
 
+const RAW_PROJECTS_TABLE = "sonarqube_projects"
+
 var ConvertFileMetricsMeta = plugin.SubTaskMeta{
 	Name:             "convertFileMetrics",
 	EntryPoint:       ConvertFileMetrics,

backend/plugins/sonarqube/tasks/shared.go

@@ -21,6 +21,7 @@ import (
 	"github.com/apache/incubator-devlake/core/errors"
 	"github.com/apache/incubator-devlake/core/plugin"
 	"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+	"github.com/apache/incubator-devlake/plugins/sonarqube/models"
 	"net/http"
 )
 
@@ -62,3 +63,25 @@ type Paging struct {
 	PageSize  int `json:"pageSize"`
 	Total     int `json:"total"`
 }
+
+type SonarqubeApiProject struct {
+	ProjectKey       string           `json:"key"`
+	Name             string           `json:"name"`
+	Qualifier        string           `json:"qualifier"`
+	Visibility       string           `json:"visibility"`
+	LastAnalysisDate *api.Iso8601Time `json:"lastAnalysisDate"`
+	Revision         string           `json:"revision"`
+}
+
+// Convert the API response to our DB model instance
+func ConvertProject(sonarqubeApiProject *SonarqubeApiProject) *models.SonarqubeProject {
+	sonarqubeProject := &models.SonarqubeProject{
+		ProjectKey:       sonarqubeApiProject.ProjectKey,
+		Name:             sonarqubeApiProject.Name,
+		Qualifier:        sonarqubeApiProject.Qualifier,
+		Visibility:       sonarqubeApiProject.Visibility,
+		LastAnalysisDate: sonarqubeApiProject.LastAnalysisDate,
+		Revision:         sonarqubeApiProject.Revision,
+	}
+	return sonarqubeProject
+}