From 79701715d14a6bced9fbf9aa2f56cba4c9fb8923 Mon Sep 17 00:00:00 2001 From: Kaixiang-AWS Date: Thu, 20 Feb 2020 19:59:59 -0800 Subject: [PATCH] feat(codebuild): Add API to list projects (#635) Add a new API to get all projects for specific account and region. This API iterates over all pages and aggregates the results. The number of projects per region per account is at most 5000. Each page contains 100 results, so at most 50 calls will be made to fetch all projects. I was thinking about returning the nextToken to caller and let caller do pagination. However, I found out that none of list* APIs igor provides supports pagination. Therefore I decide to follow the pattern to return all results in one batch. It should be fine since the number of projects is less than 100 in most cases. Co-authored-by: Clare Liguori --- .../igor/codebuild/AwsCodeBuildAccount.java | 22 +++++++++++++++ .../codebuild/AwsCodeBuildController.java | 5 ++++ .../codebuild/AwsCodeBuildAccountSpec.groovy | 27 +++++++++++++++++++ 3 files changed, 54 insertions(+) diff --git a/igor-web/src/main/java/com/netflix/spinnaker/igor/codebuild/AwsCodeBuildAccount.java b/igor-web/src/main/java/com/netflix/spinnaker/igor/codebuild/AwsCodeBuildAccount.java index 7d752e0a7..bacb31d1f 100644 --- a/igor-web/src/main/java/com/netflix/spinnaker/igor/codebuild/AwsCodeBuildAccount.java +++ b/igor-web/src/main/java/com/netflix/spinnaker/igor/codebuild/AwsCodeBuildAccount.java @@ -22,6 +22,9 @@ import com.amazonaws.services.codebuild.model.BatchGetBuildsRequest; import com.amazonaws.services.codebuild.model.Build; import com.amazonaws.services.codebuild.model.BuildArtifacts; +import com.amazonaws.services.codebuild.model.ListProjectsRequest; +import com.amazonaws.services.codebuild.model.ListProjectsResult; +import com.amazonaws.services.codebuild.model.ProjectSortByType; import com.amazonaws.services.codebuild.model.StartBuildRequest; import com.amazonaws.services.codebuild.model.StopBuildRequest; import com.amazonaws.services.securitytoken.AWSSecurityTokenServiceClient; @@ -56,6 +59,25 @@ public AwsCodeBuildAccount(String accountId, String assumeRole, String region) { .build(); } + // The number of projects has an upper limit of 5000 per region per account + // P99 latency could be high if user has more than 1000 projects + public List getProjects() { + List projects = new ArrayList<>(); + String nextToken = null; + + do { + ListProjectsResult result = + client.listProjects( + new ListProjectsRequest() + .withSortBy(ProjectSortByType.NAME) + .withNextToken(nextToken)); + projects.addAll(result.getProjects()); + nextToken = result.getNextToken(); + } while (nextToken != null); + + return projects; + } + public Build startBuild(StartBuildRequest request) { return client.startBuild(request).getBuild(); } diff --git a/igor-web/src/main/java/com/netflix/spinnaker/igor/codebuild/AwsCodeBuildController.java b/igor-web/src/main/java/com/netflix/spinnaker/igor/codebuild/AwsCodeBuildController.java index 3194c1ce5..6d766de69 100644 --- a/igor-web/src/main/java/com/netflix/spinnaker/igor/codebuild/AwsCodeBuildController.java +++ b/igor-web/src/main/java/com/netflix/spinnaker/igor/codebuild/AwsCodeBuildController.java @@ -40,6 +40,11 @@ List getAccounts() { return awsCodeBuildAccountRepository.getAccountNames(); } + @RequestMapping(value = "/projects/{account}", method = RequestMethod.GET) + List getProjects(@PathVariable String account) { + return awsCodeBuildAccountRepository.getAccount(account).getProjects(); + } + @RequestMapping( value = "/builds/start/{account}", method = RequestMethod.POST, diff --git a/igor-web/src/test/groovy/com/netflix/spinnaker/igor/codebuild/AwsCodeBuildAccountSpec.groovy b/igor-web/src/test/groovy/com/netflix/spinnaker/igor/codebuild/AwsCodeBuildAccountSpec.groovy index 863a103dc..f4747197a 100644 --- a/igor-web/src/test/groovy/com/netflix/spinnaker/igor/codebuild/AwsCodeBuildAccountSpec.groovy +++ b/igor-web/src/test/groovy/com/netflix/spinnaker/igor/codebuild/AwsCodeBuildAccountSpec.groovy @@ -21,6 +21,9 @@ import com.amazonaws.services.codebuild.model.BatchGetBuildsRequest import com.amazonaws.services.codebuild.model.BatchGetBuildsResult import com.amazonaws.services.codebuild.model.Build import com.amazonaws.services.codebuild.model.BuildArtifacts +import com.amazonaws.services.codebuild.model.ListProjectsRequest +import com.amazonaws.services.codebuild.model.ListProjectsResult +import com.amazonaws.services.codebuild.model.ProjectSortByType import com.amazonaws.services.codebuild.model.StartBuildRequest import com.amazonaws.services.codebuild.model.StartBuildResult import spock.lang.Specification @@ -145,6 +148,30 @@ class AwsCodeBuildAccountSpec extends Specification { result.get(1).getName() == "s3://another-bucket/another/path/file.zip" } + def "getProjects should return all projects in the account"() { + given: + def firstPage = (1..100).collect{ it.toString() } + def secondPage = (101..150).collect{ it.toString() } + + when: + def result = awsCodeBuildAccount.getProjects() + + then: + 1 * client.listProjects(new ListProjectsRequest().withSortBy(ProjectSortByType.NAME)) >> new ListProjectsResult().withProjects(firstPage).withNextToken("nextToken") + 1 * client.listProjects(new ListProjectsRequest().withSortBy(ProjectSortByType.NAME).withNextToken("nextToken")) >> new ListProjectsResult().withProjects(secondPage) + result.size() == 150 + result == (1..150).collect{ it.toString() } + } + + def "getProjects should return empty when no project found"() { + when: + def result = awsCodeBuildAccount.getProjects() + + then: + 1 * client.listProjects(new ListProjectsRequest().withSortBy(ProjectSortByType.NAME)) >> new ListProjectsResult().withProjects([]) + result == [] + } + private static StartBuildRequest getStartBuildInput(String projectName) { return new StartBuildRequest() .withProjectName(projectName);