Skip to content

Commit

Permalink
Bamboo scopes and remote api (#4461)
Browse files Browse the repository at this point in the history
* feat: bamboo blueprint

Add bamboo blueprint.

Nddtfjiang <[email protected]>

* fix: fix for merge

fix for merge

Nddtfjiang <[email protected]>

* feat: add scopes and remote

Add scopes and remote api for bamboo

Nddtfjiang <[email protected]>

* feat: scope and remote api for bamboo

Add scope and remote api for bamboo

Nddtfjiang <[email protected]>
  • Loading branch information
mappjzc authored Feb 21, 2023
1 parent 86db4ca commit dca2682
Show file tree
Hide file tree
Showing 13 changed files with 629 additions and 53 deletions.
11 changes: 8 additions & 3 deletions backend/plugins/bamboo/api/blueprint_V200_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ import (
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/helpers/unithelper"
"github.com/apache/incubator-devlake/plugins/bamboo/models"
"github.com/apache/incubator-devlake/plugins/bamboo/tasks"
"github.com/go-playground/validator/v10"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
Expand Down Expand Up @@ -94,13 +95,17 @@ func TestMakeDataSourcePipelinePlanV200(t *testing.T) {

var expectRepoId = "bamboo:BambooProject:1:TEST"

var testSubTaskMeta = []plugin.SubTaskMeta{}
var testSubTaskMeta = []plugin.SubTaskMeta{
tasks.ConvertProjectsMeta,
}

var expectPlans = plugin.PipelinePlan{
{
{
Plugin: "bamboo",
Subtasks: []string{},
Plugin: "bamboo",
Subtasks: []string{
tasks.ConvertProjectsMeta.Name,
},
Options: map[string]interface{}{
"connectionId": uint64(1),
"projectKey": testKey,
Expand Down
8 changes: 4 additions & 4 deletions backend/plugins/bamboo/api/blueprint_v200.go
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ func makeScopeV200(connectionId uint64, scopes []*plugin.BlueprintScopeV200) ([]
id := didgen.NewDomainIdGenerator(&models.BambooProject{}).Generate(connectionId, scope.Id)

// get project from db
BambooProject, err := GetprojectByConnectionIdAndscopeId(connectionId, scope.Id)
BambooProject, err := GetProjectByConnectionIdAndscopeId(connectionId, scope.Id)
if err != nil {
return nil, err
}
Expand All @@ -93,7 +93,7 @@ func makePipelinePlanV200(
var stage plugin.PipelineStage
var err errors.Error
// get project
project, err := GetprojectByConnectionIdAndscopeId(connection.ID, scope.Id)
project, err := GetProjectByConnectionIdAndscopeId(connection.ID, scope.Id)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -127,8 +127,8 @@ func makePipelinePlanV200(
return plans, nil
}

// GetprojectByConnectionIdAndscopeId get tbe project by the connectionId and the scopeId
func GetprojectByConnectionIdAndscopeId(connectionId uint64, scopeId string) (*models.BambooProject, errors.Error) {
// GetProjectByConnectionIdAndscopeId get tbe project by the connectionId and the scopeId
func GetProjectByConnectionIdAndscopeId(connectionId uint64, scopeId string) (*models.BambooProject, errors.Error) {
key := scopeId
project := &models.BambooProject{}
db := basicRes.GetDal()
Expand Down
302 changes: 302 additions & 0 deletions backend/plugins/bamboo/api/remote.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,302 @@
/*
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package api

import (
"context"
"encoding/base64"
"encoding/json"
"fmt"
"net/http"
"net/url"
"strconv"

"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/bamboo/models"
)

type RemoteScopesChild struct {
Type string `json:"type"`
ParentId *string `json:"parentId"`
Id string `json:"id"`
Name string `json:"name"`
Data interface{} `json:"data"`
}

type RemoteScopesOutput struct {
Children []RemoteScopesChild `json:"children"`
NextPageToken string `json:"nextPageToken"`
}

type SearchRemoteScopesOutput struct {
Children []RemoteScopesChild `json:"children"`
Page int `json:"page"`
PageSize int `json:"pageSize"`
}

type PageData struct {
Page int `json:"page"`
PageSize int `json:"per_page"`
}

const BambooRemoteScopesPerPage int = 100
const TypeProject string = "scope"
const TypeGroup string = "group"

// RemoteScopes list all available scope for users
// @Summary list all available scope for users
// @Description list all available scope for users
// @Tags plugins/bamboo
// @Accept application/json
// @Param connectionId path int false "connection ID"
// @Param groupId query string false "group ID"
// @Param pageToken query string false "page Token"
// @Success 200 {object} RemoteScopesOutput
// @Failure 400 {object} shared.ApiBody "Bad Request"
// @Failure 500 {object} shared.ApiBody "Internal Error"
// @Router /plugins/bamboo/connections/{connectionId}/remote-scopes [GET]
func RemoteScopes(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
connectionId, _ := extractParam(input.Params)
if connectionId == 0 {
return nil, errors.BadInput.New("invalid connectionId")
}

connection := &models.BambooConnection{}
err := connectionHelper.First(connection, input.Params)
if err != nil {
return nil, err
}

pageToken, ok := input.Query["pageToken"]
if !ok || len(pageToken) == 0 {
pageToken = []string{""}
}

// get pageData
pageData, err := GetPageDataFromPageToken(pageToken[0])
if err != nil {
return nil, errors.BadInput.New("failed to get paget token")
}

// create api client
apiClient, err := api.NewApiClientFromConnection(context.TODO(), basicRes, connection)
if err != nil {
return nil, err
}

var res *http.Response
outputBody := &RemoteScopesOutput{}

query := GetQueryFromPageData(pageData)

res, err = apiClient.Get("/project.json", query, nil)

if err != nil {
return nil, err
}

resBody := models.ApiBambooProjectResponse{}
err = api.UnmarshalResponse(res, &resBody)
if err != nil {
return nil, err
}

// append project to output
for _, apiProject := range resBody.Projects.Projects {
project := &models.BambooProject{}
project.Convert(&apiProject)
child := RemoteScopesChild{
Type: TypeProject,
ParentId: nil,
Id: project.ProjectKey,
Name: project.Name,
Data: project,
}

outputBody.Children = append(outputBody.Children, child)
}

// check project count
if len(resBody.Projects.Projects) < pageData.PageSize {
pageData = nil
}

// get the next page token
outputBody.NextPageToken = ""
if pageData != nil {
pageData.Page += 1
pageData.PageSize = BambooRemoteScopesPerPage

outputBody.NextPageToken, err = GetPageTokenFromPageData(pageData)
if err != nil {
return nil, err
}
}

return &plugin.ApiResourceOutput{Body: outputBody, Status: http.StatusOK}, nil
}

// SearchRemoteScopes use the Search API and only return project
// @Summary use the Search API and only return project
// @Description use the Search API and only return project
// @Tags plugins/bamboo
// @Accept application/json
// @Param connectionId path int false "connection ID"
// @Param search query string false "search"
// @Param page query int false "page number"
// @Param pageSize query int false "page size per page"
// @Success 200 {object} SearchRemoteScopesOutput
// @Failure 400 {object} shared.ApiBody "Bad Request"
// @Failure 500 {object} shared.ApiBody "Internal Error"
// @Router /plugins/bamboo/connections/{connectionId}/search-remote-scopes [GET]
func SearchRemoteScopes(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
connectionId, _ := extractParam(input.Params)
if connectionId == 0 {
return nil, errors.BadInput.New("invalid connectionId")
}

connection := &models.BambooConnection{}
err := connectionHelper.First(connection, input.Params)
if err != nil {
return nil, err
}

search, ok := input.Query["search"]
if !ok || len(search) == 0 {
search = []string{""}
}

var p int
var err1 error
page, ok := input.Query["page"]
if !ok || len(page) == 0 {
p = 1
} else {
p, err1 = strconv.Atoi(page[0])
if err != nil {
return nil, errors.BadInput.Wrap(err1, fmt.Sprintf("failed to Atoi page:%s", page[0]))
}
}
var ps int
pageSize, ok := input.Query["pageSize"]
if !ok || len(pageSize) == 0 {
ps = BambooRemoteScopesPerPage
} else {
ps, err1 = strconv.Atoi(pageSize[0])
if err1 != nil {
return nil, errors.BadInput.Wrap(err1, fmt.Sprintf("failed to Atoi pageSize:%s", pageSize[0]))
}
}
// create api client
apiClient, err := api.NewApiClientFromConnection(context.TODO(), basicRes, connection)
if err != nil {
return nil, err
}

// set query
query := GetQueryForSearchProject(search[0], p, ps)

// request search
res, err := apiClient.Get("search/projects.json", query, nil)
if err != nil {
return nil, err
}
resBody := models.ApiBambooSearchProjectResponse{}
err = api.UnmarshalResponse(res, &resBody)
if err != nil {
return nil, err
}

outputBody := &SearchRemoteScopesOutput{}

// append project to output
for _, apiResult := range resBody.SearchResults {
var project models.BambooProject
apiProject, err := GetApiProject(apiResult.SearchEntity.Key, apiClient)
if err != nil {
return nil, err
}

project.Convert(apiProject)
child := RemoteScopesChild{
Type: TypeProject,
Id: project.ProjectKey,
ParentId: nil,
Name: project.Name,
Data: project,
}

outputBody.Children = append(outputBody.Children, child)
}

outputBody.Page = p
outputBody.PageSize = ps

return &plugin.ApiResourceOutput{Body: outputBody, Status: http.StatusOK}, nil
}

func GetPageTokenFromPageData(pageData *PageData) (string, errors.Error) {
// Marshal json
pageTokenDecode, err := json.Marshal(pageData)
if err != nil {
return "", errors.Default.Wrap(err, fmt.Sprintf("Marshal pageToken failed %+v", pageData))
}

// Encode pageToken Base64
return base64.StdEncoding.EncodeToString(pageTokenDecode), nil
}

func GetPageDataFromPageToken(pageToken string) (*PageData, errors.Error) {
if pageToken == "" {
return &PageData{
Page: 1,
PageSize: BambooRemoteScopesPerPage,
}, nil
}

// Decode pageToken Base64
pageTokenDecode, err := base64.StdEncoding.DecodeString(pageToken)
if err != nil {
return nil, errors.Default.Wrap(err, fmt.Sprintf("decode pageToken failed %s", pageToken))
}
// Unmarshal json
pt := &PageData{}
err = json.Unmarshal(pageTokenDecode, pt)
if err != nil {
return nil, errors.Default.Wrap(err, fmt.Sprintf("json Unmarshal pageTokenDecode failed %s", pageTokenDecode))
}

return pt, nil
}

func GetQueryFromPageData(pageData *PageData) url.Values {
query := url.Values{}
query.Set("showEmpty", fmt.Sprintf("%v", true))
query.Set("max-result", fmt.Sprintf("%v", pageData.PageSize))
query.Set("start-index", fmt.Sprintf("%v", (pageData.Page-1)*pageData.PageSize))
return query
}

func GetQueryForSearchProject(search string, page int, perPage int) url.Values {
query := GetQueryFromPageData(&PageData{Page: page, PageSize: perPage})
query.Set("searchTerm", search)

return query
}
Loading

0 comments on commit dca2682

Please sign in to comment.