Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

cherry-pick #6876 fix: github run api page<35 limit #6878

Merged
merged 1 commit into from
Jan 26, 2024
Merged

cherry-pick #6876 fix: github run api page<35 limit #6878

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
270 changes: 262 additions & 8 deletions backend/plugins/github/tasks/cicd_run_collector.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,11 +41,6 @@ const RAW_RUN_TABLE = "github_api_runs"
// https://github.com/apache/incubator-devlake/issues/3199
const PAGE_SIZE = 30

type GithubRawRunsResult struct {
TotalCount int64 `json:"total_count"`
GithubWorkflowRuns []json.RawMessage `json:"workflow_runs"`
}

type SimpleGithubApiJob struct {
ID int64
CreatedAt common.Iso8601Time `json:"created_at"`
Expand Down Expand Up @@ -80,7 +75,9 @@ func CollectRuns(taskCtx plugin.SubTaskContext) errors.Error {
UrlTemplate: "repos/{{ .Params.Name }}/actions/runs",
Query: func(reqData *helper.RequestData, createdAfter *time.Time) (url.Values, errors.Error) {
query := url.Values{}
query.Set("status", "completed")
// GitHub API returns only the first 34 pages (with a size of 30) when specifying status=compleleted, try the following API request to verify the problem.
// https://api.github.com/repos/apache/incubator-devlake/actions/runs?per_page=30&page=35&status=completed
// query.Set("status", "completed")
query.Set("page", fmt.Sprintf("%v", reqData.Pager.Page))
query.Set("per_page", fmt.Sprintf("%v", reqData.Pager.Size))
return query, nil
Expand All @@ -91,10 +88,21 @@ func CollectRuns(taskCtx plugin.SubTaskContext) errors.Error {
if err != nil {
return nil, err
}
if len(body.GithubWorkflowRuns) == 0 {
if len(body.WorkflowRuns) == 0 {
return nil, nil
}
return body.GithubWorkflowRuns, nil
// filter out the runs that are not completed
filteredRuns := make([]json.RawMessage, 0)
for _, run := range body.WorkflowRuns {
if run.Status == "completed" {
runJSON, err := json.Marshal(run)
if err != nil {
return nil, errors.Convert(err)
}
filteredRuns = append(filteredRuns, json.RawMessage(runJSON))
}
}
return filteredRuns, nil
},
},
GetCreated: func(item json.RawMessage) (time.Time, errors.Error) {
Expand All @@ -115,3 +123,249 @@ func CollectRuns(taskCtx plugin.SubTaskContext) errors.Error {
return collector.Execute()

}

type GithubRawRunsResult struct {
TotalCount int `json:"total_count"`
WorkflowRuns []struct {
ID int64 `json:"id"`
Name string `json:"name"`
NodeID string `json:"node_id"`
HeadBranch string `json:"head_branch"`
HeadSha string `json:"head_sha"`
Path string `json:"path"`
DisplayTitle string `json:"display_title"`
RunNumber int `json:"run_number"`
Event string `json:"event"`
Status string `json:"status"`
Conclusion string `json:"conclusion"`
WorkflowID int `json:"workflow_id"`
CheckSuiteID int64 `json:"check_suite_id"`
CheckSuiteNodeID string `json:"check_suite_node_id"`
URL string `json:"url"`
HTMLURL string `json:"html_url"`
PullRequests []struct {
URL string `json:"url"`
ID int `json:"id"`
Number int `json:"number"`
Head struct {
Ref string `json:"ref"`
Sha string `json:"sha"`
Repo struct {
ID int `json:"id"`
URL string `json:"url"`
Name string `json:"name"`
} `json:"repo"`
} `json:"head"`
Base struct {
Ref string `json:"ref"`
Sha string `json:"sha"`
Repo struct {
ID int `json:"id"`
URL string `json:"url"`
Name string `json:"name"`
} `json:"repo"`
} `json:"base"`
} `json:"pull_requests"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
Actor struct {
Login string `json:"login"`
ID int `json:"id"`
NodeID string `json:"node_id"`
AvatarURL string `json:"avatar_url"`
GravatarID string `json:"gravatar_id"`
URL string `json:"url"`
HTMLURL string `json:"html_url"`
FollowersURL string `json:"followers_url"`
FollowingURL string `json:"following_url"`
GistsURL string `json:"gists_url"`
StarredURL string `json:"starred_url"`
SubscriptionsURL string `json:"subscriptions_url"`
OrganizationsURL string `json:"organizations_url"`
ReposURL string `json:"repos_url"`
EventsURL string `json:"events_url"`
ReceivedEventsURL string `json:"received_events_url"`
Type string `json:"type"`
SiteAdmin bool `json:"site_admin"`
} `json:"actor"`
RunAttempt int `json:"run_attempt"`
ReferencedWorkflows []any `json:"referenced_workflows"`
RunStartedAt time.Time `json:"run_started_at"`
TriggeringActor struct {
Login string `json:"login"`
ID int `json:"id"`
NodeID string `json:"node_id"`
AvatarURL string `json:"avatar_url"`
GravatarID string `json:"gravatar_id"`
URL string `json:"url"`
HTMLURL string `json:"html_url"`
FollowersURL string `json:"followers_url"`
FollowingURL string `json:"following_url"`
GistsURL string `json:"gists_url"`
StarredURL string `json:"starred_url"`
SubscriptionsURL string `json:"subscriptions_url"`
OrganizationsURL string `json:"organizations_url"`
ReposURL string `json:"repos_url"`
EventsURL string `json:"events_url"`
ReceivedEventsURL string `json:"received_events_url"`
Type string `json:"type"`
SiteAdmin bool `json:"site_admin"`
} `json:"triggering_actor"`
JobsURL string `json:"jobs_url"`
LogsURL string `json:"logs_url"`
CheckSuiteURL string `json:"check_suite_url"`
ArtifactsURL string `json:"artifacts_url"`
CancelURL string `json:"cancel_url"`
RerunURL string `json:"rerun_url"`
PreviousAttemptURL any `json:"previous_attempt_url"`
WorkflowURL string `json:"workflow_url"`
HeadCommit struct {
ID string `json:"id"`
TreeID string `json:"tree_id"`
Message string `json:"message"`
Timestamp time.Time `json:"timestamp"`
Author struct {
Name string `json:"name"`
Email string `json:"email"`
} `json:"author"`
Committer struct {
Name string `json:"name"`
Email string `json:"email"`
} `json:"committer"`
} `json:"head_commit"`
Repository struct {
ID int `json:"id"`
NodeID string `json:"node_id"`
Name string `json:"name"`
FullName string `json:"full_name"`
Private bool `json:"private"`
Owner struct {
Login string `json:"login"`
ID int `json:"id"`
NodeID string `json:"node_id"`
AvatarURL string `json:"avatar_url"`
GravatarID string `json:"gravatar_id"`
URL string `json:"url"`
HTMLURL string `json:"html_url"`
FollowersURL string `json:"followers_url"`
FollowingURL string `json:"following_url"`
GistsURL string `json:"gists_url"`
StarredURL string `json:"starred_url"`
SubscriptionsURL string `json:"subscriptions_url"`
OrganizationsURL string `json:"organizations_url"`
ReposURL string `json:"repos_url"`
EventsURL string `json:"events_url"`
ReceivedEventsURL string `json:"received_events_url"`
Type string `json:"type"`
SiteAdmin bool `json:"site_admin"`
} `json:"owner"`
HTMLURL string `json:"html_url"`
Description string `json:"description"`
Fork bool `json:"fork"`
URL string `json:"url"`
ForksURL string `json:"forks_url"`
KeysURL string `json:"keys_url"`
CollaboratorsURL string `json:"collaborators_url"`
TeamsURL string `json:"teams_url"`
HooksURL string `json:"hooks_url"`
IssueEventsURL string `json:"issue_events_url"`
EventsURL string `json:"events_url"`
AssigneesURL string `json:"assignees_url"`
BranchesURL string `json:"branches_url"`
TagsURL string `json:"tags_url"`
BlobsURL string `json:"blobs_url"`
GitTagsURL string `json:"git_tags_url"`
GitRefsURL string `json:"git_refs_url"`
TreesURL string `json:"trees_url"`
StatusesURL string `json:"statuses_url"`
LanguagesURL string `json:"languages_url"`
StargazersURL string `json:"stargazers_url"`
ContributorsURL string `json:"contributors_url"`
SubscribersURL string `json:"subscribers_url"`
SubscriptionURL string `json:"subscription_url"`
CommitsURL string `json:"commits_url"`
GitCommitsURL string `json:"git_commits_url"`
CommentsURL string `json:"comments_url"`
IssueCommentURL string `json:"issue_comment_url"`
ContentsURL string `json:"contents_url"`
CompareURL string `json:"compare_url"`
MergesURL string `json:"merges_url"`
ArchiveURL string `json:"archive_url"`
DownloadsURL string `json:"downloads_url"`
IssuesURL string `json:"issues_url"`
PullsURL string `json:"pulls_url"`
MilestonesURL string `json:"milestones_url"`
NotificationsURL string `json:"notifications_url"`
LabelsURL string `json:"labels_url"`
ReleasesURL string `json:"releases_url"`
DeploymentsURL string `json:"deployments_url"`
} `json:"repository"`
HeadRepository struct {
ID int `json:"id"`
NodeID string `json:"node_id"`
Name string `json:"name"`
FullName string `json:"full_name"`
Private bool `json:"private"`
Owner struct {
Login string `json:"login"`
ID int `json:"id"`
NodeID string `json:"node_id"`
AvatarURL string `json:"avatar_url"`
GravatarID string `json:"gravatar_id"`
URL string `json:"url"`
HTMLURL string `json:"html_url"`
FollowersURL string `json:"followers_url"`
FollowingURL string `json:"following_url"`
GistsURL string `json:"gists_url"`
StarredURL string `json:"starred_url"`
SubscriptionsURL string `json:"subscriptions_url"`
OrganizationsURL string `json:"organizations_url"`
ReposURL string `json:"repos_url"`
EventsURL string `json:"events_url"`
ReceivedEventsURL string `json:"received_events_url"`
Type string `json:"type"`
SiteAdmin bool `json:"site_admin"`
} `json:"owner"`
HTMLURL string `json:"html_url"`
Description string `json:"description"`
Fork bool `json:"fork"`
URL string `json:"url"`
ForksURL string `json:"forks_url"`
KeysURL string `json:"keys_url"`
CollaboratorsURL string `json:"collaborators_url"`
TeamsURL string `json:"teams_url"`
HooksURL string `json:"hooks_url"`
IssueEventsURL string `json:"issue_events_url"`
EventsURL string `json:"events_url"`
AssigneesURL string `json:"assignees_url"`
BranchesURL string `json:"branches_url"`
TagsURL string `json:"tags_url"`
BlobsURL string `json:"blobs_url"`
GitTagsURL string `json:"git_tags_url"`
GitRefsURL string `json:"git_refs_url"`
TreesURL string `json:"trees_url"`
StatusesURL string `json:"statuses_url"`
LanguagesURL string `json:"languages_url"`
StargazersURL string `json:"stargazers_url"`
ContributorsURL string `json:"contributors_url"`
SubscribersURL string `json:"subscribers_url"`
SubscriptionURL string `json:"subscription_url"`
CommitsURL string `json:"commits_url"`
GitCommitsURL string `json:"git_commits_url"`
CommentsURL string `json:"comments_url"`
IssueCommentURL string `json:"issue_comment_url"`
ContentsURL string `json:"contents_url"`
CompareURL string `json:"compare_url"`
MergesURL string `json:"merges_url"`
ArchiveURL string `json:"archive_url"`
DownloadsURL string `json:"downloads_url"`
IssuesURL string `json:"issues_url"`
PullsURL string `json:"pulls_url"`
MilestonesURL string `json:"milestones_url"`
NotificationsURL string `json:"notifications_url"`
LabelsURL string `json:"labels_url"`
ReleasesURL string `json:"releases_url"`
DeploymentsURL string `json:"deployments_url"`
} `json:"head_repository"`
} `json:"workflow_runs"`
}
Loading