Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(appset): added topic filter for Gitlab SCM #14965

Merged
merged 3 commits into from
Aug 20, 2023
Merged

feat(appset): added topic filter for Gitlab SCM #14965

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
13b9370
added topic (tag) filter for Gitlab SCM
prune998 Aug 8, 2023
e5c6607
corrected few comments
prune998 Aug 9, 2023
7b1e7cf
removed latest tag references
prune998 Aug 17, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 26 additions & 0 deletions applicationset/examples/scm-provider-generator/scm-provider-example-fasttemplate-gitlab.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
apiVersion: argoproj.io/v1alpha1
kind: ApplicationSet
metadata:
name: guestbook
spec:
generators:
- scmProvider:
gitlab:
api: https://gitlab.com
group: test-argocd-proton
includeSubgroups: true
cloneProtocol: https
filters:
- repositoryMatch: test-app
template:
metadata:
name: '{{ repository }}-guestbook'
spec:
project: "default"
source:
repoURL: '{{ url }}'
targetRevision: '{{ branch }}'
path: guestbook
destination:
server: https://kubernetes.default.svc
namespace: guestbook
2 changes: 1 addition & 1 deletion applicationset/generators/scm_provider.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -118,7 +118,7 @@ func (g *SCMProviderGenerator) GenerateParams(appSetGenerator *argoprojiov1alpha
if err != nil {
return nil, fmt.Errorf("error fetching Gitlab token: %v", err)
}
provider, err = scm_provider.NewGitlabProvider(ctx, providerConfig.Gitlab.Group, token, providerConfig.Gitlab.API, providerConfig.Gitlab.AllBranches, providerConfig.Gitlab.IncludeSubgroups, providerConfig.Gitlab.WillIncludeSharedProjects(), providerConfig.Gitlab.Insecure, g.scmRootCAPath)
provider, err = scm_provider.NewGitlabProvider(ctx, providerConfig.Gitlab.Group, token, providerConfig.Gitlab.API, providerConfig.Gitlab.AllBranches, providerConfig.Gitlab.IncludeSubgroups, providerConfig.Gitlab.WillIncludeSharedProjects(), providerConfig.Gitlab.Insecure, g.scmRootCAPath, providerConfig.Gitlab.Topic)
if err != nil {
return nil, fmt.Errorf("error initializing Gitlab service: %v", err)
}
Expand Down
6 changes: 4 additions & 2 deletions applicationset/services/scm_provider/gitlab.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,12 @@ type GitlabProvider struct {
allBranches bool
includeSubgroups bool
includeSharedProjects bool
topic string
}

var _ SCMProviderService = &GitlabProvider{}

func NewGitlabProvider(ctx context.Context, organization string, token string, url string, allBranches, includeSubgroups, includeSharedProjects, insecure bool, scmRootCAPath string) (*GitlabProvider, error) {
func NewGitlabProvider(ctx context.Context, organization string, token string, url string, allBranches, includeSubgroups, includeSharedProjects, insecure bool, scmRootCAPath, topic string) (*GitlabProvider, error) {
// Undocumented environment variable to set a default token, to be used in testing to dodge anonymous rate limits.
if token == "" {
token = os.Getenv("GITLAB_TOKEN")
Expand All @@ -49,7 +50,7 @@ func NewGitlabProvider(ctx context.Context, organization string, token string, u
}
}

return &GitlabProvider{client: client, organization: organization, allBranches: allBranches, includeSubgroups: includeSubgroups, includeSharedProjects: includeSharedProjects}, nil
return &GitlabProvider{client: client, organization: organization, allBranches: allBranches, includeSubgroups: includeSubgroups, includeSharedProjects: includeSharedProjects, topic: topic}, nil
}

func (g *GitlabProvider) GetBranches(ctx context.Context, repo *Repository) ([]*Repository, error) {
Expand Down Expand Up @@ -78,6 +79,7 @@ func (g *GitlabProvider) ListRepos(ctx context.Context, cloneProtocol string) ([
ListOptions: gitlab.ListOptions{PerPage: 100},
IncludeSubGroups: &g.includeSubgroups,
WithShared: &g.includeSharedProjects,
Topic: &g.topic,
}

repos := []*Repository{}
Expand Down
207 changes: 188 additions & 19 deletions applicationset/services/scm_provider/gitlab_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ func gitlabMockHandler(t *testing.T) func(http.ResponseWriter, *http.Request) {
switch r.RequestURI {
case "/api/v4":
fmt.Println("here1")
case "/api/v4/groups/test-argocd-proton/projects?include_subgroups=false&per_page=100", "/api/v4/groups/test-argocd-proton/projects?include_subgroups=false&per_page=100&with_shared=false":
case "/api/v4/groups/test-argocd-proton/projects?include_subgroups=false&per_page=100", "/api/v4/groups/test-argocd-proton/projects?include_subgroups=false&per_page=100&topic=&with_shared=false":
fmt.Println("here")
_, err := io.WriteString(w, `[{
"id": 27084533,
Expand All @@ -30,8 +30,12 @@ func gitlabMockHandler(t *testing.T) func(http.ResponseWriter, *http.Request) {
"path_with_namespace": "test-argocd-proton/argocd",
"created_at": "2021-06-01T17:30:44.724Z",
"default_branch": "master",
"tag_list": [],
"topics": [],
"tag_list": [
"test-topic"
],
"topics": [
"test-topic"
],
"ssh_url_to_repo": "[email protected]:test-argocd-proton/argocd.git",
"http_url_to_repo": "https://gitlab.com/test-argocd-proton/argocd.git",
"web_url": "https://gitlab.com/test-argocd-proton/argocd",
Expand Down Expand Up @@ -143,7 +147,7 @@ func gitlabMockHandler(t *testing.T) func(http.ResponseWriter, *http.Request) {
if err != nil {
t.Fail()
}
case "/api/v4/groups/test-argocd-proton/projects?include_subgroups=true&per_page=100&with_shared=false":
case "/api/v4/groups/test-argocd-proton/projects?include_subgroups=true&per_page=100&topic=&with_shared=false":
fmt.Println("here")
_, err := io.WriteString(w, `[{
"id": 27084533,
Expand All @@ -154,8 +158,14 @@ func gitlabMockHandler(t *testing.T) func(http.ResponseWriter, *http.Request) {
"path_with_namespace": "test-argocd-proton/argocd",
"created_at": "2021-06-01T17:30:44.724Z",
"default_branch": "master",
"tag_list": [],
"topics": [],
"tag_list": [
"test-topic",
"specific-topic"
],
"topics": [
"test-topic",
"specific-topic"
],
"ssh_url_to_repo": "[email protected]:test-argocd-proton/argocd.git",
"http_url_to_repo": "https://gitlab.com/test-argocd-proton/argocd.git",
"web_url": "https://gitlab.com/test-argocd-proton/argocd",
Expand Down Expand Up @@ -273,8 +283,12 @@ func gitlabMockHandler(t *testing.T) func(http.ResponseWriter, *http.Request) {
"path_with_namespace": "test-argocd-proton/subgroup/argocd-subgroup",
"created_at": "2021-06-01T17:30:44.724Z",
"default_branch": "master",
"tag_list": [],
"topics": [],
"tag_list": [
"test-topic"
],
"topics": [
"test-topic"
],
"ssh_url_to_repo": "[email protected]:test-argocd-proton/subgroup/argocd-subgroup.git",
"http_url_to_repo": "https://gitlab.com/test-argocd-proton/subgroup/argocd-subgroup.git",
"web_url": "https://gitlab.com/test-argocd-proton/subgroup/argocd-subgroup",
Expand Down Expand Up @@ -387,7 +401,7 @@ func gitlabMockHandler(t *testing.T) func(http.ResponseWriter, *http.Request) {
if err != nil {
t.Fail()
}
case "/api/v4/groups/test-argocd-proton/projects?include_subgroups=true&per_page=100&with_shared=true":
case "/api/v4/groups/test-argocd-proton/projects?include_subgroups=false&per_page=100&topic=specific-topic&with_shared=false":
fmt.Println("here")
_, err := io.WriteString(w, `[{
"id": 27084533,
Expand All @@ -398,8 +412,143 @@ func gitlabMockHandler(t *testing.T) func(http.ResponseWriter, *http.Request) {
"path_with_namespace": "test-argocd-proton/argocd",
"created_at": "2021-06-01T17:30:44.724Z",
"default_branch": "master",
"tag_list": [],
"topics": [],
"tag_list": [
"test-topic",
"specific-topic"
],
"topics": [
"test-topic",
"specific-topic"
],
"ssh_url_to_repo": "[email protected]:test-argocd-proton/argocd.git",
"http_url_to_repo": "https://gitlab.com/test-argocd-proton/argocd.git",
"web_url": "https://gitlab.com/test-argocd-proton/argocd",
"readme_url": null,
"avatar_url": null,
"forks_count": 0,
"star_count": 0,
"last_activity_at": "2021-06-04T08:19:51.656Z",
"namespace": {
"id": 12258515,
"name": "test argocd proton",
"path": "test-argocd-proton",
"kind": "gro* Connection #0 to host gitlab.com left intact up ",
"full_path ": "test - argocd - proton ",
"parent_id ": null,
"avatar_url ": null,
"web_url ": "https: //gitlab.com/groups/test-argocd-proton"
},
"container_registry_image_prefix": "registry.gitlab.com/test-argocd-proton/argocd",
"_links": {
"self": "https://gitlab.com/api/v4/projects/27084533",
"issues": "https://gitlab.com/api/v4/projects/27084533/issues",
"merge_requests": "https://gitlab.com/api/v4/projects/27084533/merge_requests",
"repo_branches": "https://gitlab.com/api/v4/projects/27084533/repository/branches",
"labels": "https://gitlab.com/api/v4/projects/27084533/labels",
"events": "https://gitlab.com/api/v4/projects/27084533/events",
"members": "https://gitlab.com/api/v4/projects/27084533/members",
"cluster_agents": "https://gitlab.com/api/v4/projects/27084533/cluster_agents"
},
"packages_enabled": true,
"empty_repo": false,
"archived": false,
"visibility": "public",
"resolve_outdated_diff_discussions": false,
"container_expiration_policy": {
"cadence": "1d",
"enabled": false,
"keep_n": 10,
"older_than": "90d",
"name_regex": ".*",
"name_regex_keep": null,
"next_run_at": "2021-06-02T17:30:44.740Z"
},
"issues_enabled": true,
"merge_requests_enabled": true,
"wiki_enabled": true,
"jobs_enabled": true,
"snippets_enabled": true,
"container_registry_enabled": true,
"service_desk_enabled": true,
"can_create_merge_request_in": false,
"issues_access_level": "enabled",
"repository_access_level": "enabled",
"merge_requests_access_level": "enabled",
"forking_access_level": "enabled",
"wiki_access_level": "enabled",
"builds_access_level": "enabled",
"snippets_access_level": "enabled",
"pages_access_level": "enabled",
"operations_access_level": "enabled",
"analytics_access_level": "enabled",
"container_registry_access_level": "enabled",
"security_and_compliance_access_level": "private",
"emails_disabled": null,
"shared_runners_enabled": true,
"lfs_enabled": true,
"creator_id": 2378866,
"import_status": "none",
"open_issues_count": 0,
"ci_default_git_depth": 50,
"ci_forward_deployment_enabled": true,
"ci_job_token_scope_enabled": false,
"public_jobs": true,
"build_timeout": 3600,
"auto_cancel_pending_pipelines": "enabled",
"ci_config_path": "",
"shared_with_groups": [],
"only_allow_merge_if_pipeline_succeeds": false,
"allow_merge_on_skipped_pipeline": null,
"restrict_user_defined_variables": false,
"request_access_enabled": true,
"only_allow_merge_if_all_discussions_are_resolved": false,
"remove_source_branch_after_merge": true,
"printing_merge_request_link_enabled": true,
"merge_method": "merge",
"squash_option": "default_off",
"suggestion_commit_message": null,
"merge_commit_template": null,
"squash_commit_template": null,
"auto_devops_enabled": false,
"auto_devops_deploy_strategy": "continuous",
"autoclose_referenced_issues": true,
"keep_latest_artifact": true,
"runner_token_expiration_interval": null,
"approvals_before_merge": 0,
"mirror": false,
"external_authorization_classification_label": "",
"marked_for_deletion_at": null,
"marked_for_deletion_on": null,
"requirements_enabled": true,
"requirements_access_level": "enabled",
"security_and_compliance_enabled": false,
"compliance_frameworks": [],
"issues_template": null,
"merge_requests_template": null,
"merge_pipelines_enabled": false,
"merge_trains_enabled": false
}
]`)
if err != nil {
t.Fail()
}
case "/api/v4/groups/test-argocd-proton/projects?include_subgroups=true&per_page=100&topic=&with_shared=true":
fmt.Println("here")
_, err := io.WriteString(w, `[{
"id": 27084533,
"description": "",
"name": "argocd",
"name_with_namespace": "test argocd proton / argocd",
"path": "argocd",
"path_with_namespace": "test-argocd-proton/argocd",
"created_at": "2021-06-01T17:30:44.724Z",
"default_branch": "master",
"tag_list": [
"test-topic"
],
"topics": [
"test-topic"
],
"ssh_url_to_repo": "[email protected]:test-argocd-proton/argocd.git",
"http_url_to_repo": "https://gitlab.com/test-argocd-proton/argocd.git",
"web_url": "https://gitlab.com/test-argocd-proton/argocd",
Expand Down Expand Up @@ -517,8 +666,12 @@ func gitlabMockHandler(t *testing.T) func(http.ResponseWriter, *http.Request) {
"path_with_namespace": "test-shared-argocd-proton/shared-argocd",
"created_at": "2021-06-11T17:30:44.724Z",
"default_branch": "master",
"tag_list": [],
"topics": [],
"tag_list": [
"test-topic"
],
"topics": [
"test-topic"
],
"ssh_url_to_repo": "[email protected]:test-shared-argocd-proton/shared-argocd.git",
"http_url_to_repo": "https://gitlab.com/test-shared-argocd-proton/shared-argocd.git",
"web_url": "https://gitlab.com/test-shared-argocd-proton/shared-argocd",
Expand Down Expand Up @@ -845,8 +998,12 @@ func gitlabMockHandler(t *testing.T) func(http.ResponseWriter, *http.Request) {
"path_with_namespace": "test-argocd-proton/argocd",
"created_at": "2021-06-01T17:30:44.724Z",
"default_branch": "master",
"tag_list": [],
"topics": [],
"tag_list": [
"test-topic"
],
"topics": [
"test-topic"
],
"ssh_url_to_repo": "[email protected]:test-argocd-proton/argocd.git",
"http_url_to_repo": "https://gitlab.com/test-argocd-proton/argocd.git",
"web_url": "https://gitlab.com/test-argocd-proton/argocd",
Expand Down Expand Up @@ -891,7 +1048,7 @@ func gitlabMockHandler(t *testing.T) func(http.ResponseWriter, *http.Request) {
}
func TestGitlabListRepos(t *testing.T) {
cases := []struct {
name, proto, url string
name, proto, url, topic string
hasError, allBranches, includeSubgroups, includeSharedProjects, insecure bool
branches []string
filters []v1alpha1.SCMProviderGeneratorFilter
Expand Down Expand Up @@ -938,13 +1095,21 @@ func TestGitlabListRepos(t *testing.T) {
includeSharedProjects: true,
includeSubgroups: true,
},
{
name: "specific topic",
allBranches: true,
url: "[email protected]:test-argocd-proton/argocd.git",
branches: []string{"master"},
includeSubgroups: false,
topic: "specific-topic",
},
}
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
gitlabMockHandler(t)(w, r)
}))
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
provider, _ := NewGitlabProvider(context.Background(), "test-argocd-proton", "", ts.URL, c.allBranches, c.includeSubgroups, c.includeSharedProjects, c.insecure, "")
provider, _ := NewGitlabProvider(context.Background(), "test-argocd-proton", "", ts.URL, c.allBranches, c.includeSubgroups, c.includeSharedProjects, c.insecure, "", c.topic)
rawRepos, err := ListRepos(context.Background(), provider, c.filters, c.proto)
if c.hasError {
assert.NotNil(t, err)
Expand All @@ -970,6 +1135,10 @@ func TestGitlabListRepos(t *testing.T) {
if c.includeSubgroups || c.includeSharedProjects {
assert.Equal(t, 2, len(uniqueRepos))
}
// In case we filter on the topic, ensure we got only one repo returned
if c.topic != "" {
assert.Equal(t, 1, len(uniqueRepos))
}
}
})
}
Expand All @@ -979,7 +1148,7 @@ func TestGitlabHasPath(t *testing.T) {
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
gitlabMockHandler(t)(w, r)
}))
host, _ := NewGitlabProvider(context.Background(), "test-argocd-proton", "", ts.URL, false, true, true, false, "")
host, _ := NewGitlabProvider(context.Background(), "test-argocd-proton", "", ts.URL, false, true, true, false, "", "")
repo := &Repository{
Organization: "test-argocd-proton",
Repository: "argocd",
Expand Down Expand Up @@ -1025,7 +1194,7 @@ func TestGitlabGetBranches(t *testing.T) {
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
gitlabMockHandler(t)(w, r)
}))
host, _ := NewGitlabProvider(context.Background(), "test-argocd-proton", "", ts.URL, false, true, true, false, "")
host, _ := NewGitlabProvider(context.Background(), "test-argocd-proton", "", ts.URL, false, true, true, false, "", "")

repo := &Repository{
RepositoryId: 27084533,
Expand Down
4 changes: 4 additions & 0 deletions assets/swagger.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8693,6 +8693,10 @@
},
"tokenRef": {
"$ref": "#/definitions/v1alpha1SecretRef"
},
"topic": {
"description": "Filter repos list based on Gitlab Topic.",
"type": "string"
}
}
},
Expand Down
Loading