From 11db952710ff9af4dbf2c83c1a93ddb9df348c5d Mon Sep 17 00:00:00 2001 From: Zeger-Jan van de Weg Date: Tue, 5 Nov 2019 13:27:06 +0100 Subject: [PATCH] Add Rich data for GitLab.com as code provider When packages were searched for and disployed, platforms like GitHub and Bitbucket had slightly better data presentation. For example the number of stars a project has. Additionally, the fallback Git provider that was being used for GitLab, leverages `git ls-remote` and `git fetch` to determine what blobs are in a tree, and what subdirectories. This combination of requests is much slower than the 3 API calls done now. --- gosrc/gitlab.go | 162 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 162 insertions(+) create mode 100644 gosrc/gitlab.go diff --git a/gosrc/gitlab.go b/gosrc/gitlab.go new file mode 100644 index 00000000..b2bcc929 --- /dev/null +++ b/gosrc/gitlab.go @@ -0,0 +1,162 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd. + +package gosrc + +import ( + "context" + "fmt" + "net/http" + "path" + "regexp" + "strconv" + "strings" + "time" +) + +func init() { + addService(&service{ + pattern: regexp.MustCompile(`^gitlab\.com/(?P[a-z0-9A-Z_.\-]+)/(?P[a-z0-9A-Z_.\-]+)(?P/[a-z0-9A-Z_.\-/]*)?$`), + prefix: "gitlab.com/", + get: getGitLabDir, + getProject: getGitLabProject, + }) +} + +type glProject struct { + ID int `json:"id"` + Name string `json:"name"` + Description string `json:"description"` + DefaultBranch string `json:"default_branch"` + Stars int `json:"star_count"` + LastActivity time.Time `json:"updated_on"` + WebURL string `json:"web_url"` +} + +func getGitLabDir(ctx context.Context, client *http.Client, match map[string]string, savedEtag string) (*Directory, error) { + c := &httpClient{client: client} + + project, err := gitlabProject(ctx, c, match) + if err != nil { + return nil, err + } + match["ref"] = project.DefaultBranch + match["project_id"] = strconv.Itoa(project.ID) + match["dir"] = strings.TrimPrefix(match["dir"], "/") + + type glCommit struct { + ID string `json:"id"` + CommittedDate time.Time `json:"committed_date"` + } + + var commits []*glCommit + url := expand("https://gitlab.com/api/v4/projects/{project_id}/repository/commits?path={0}&per_page=1", match, strings.TrimPrefix(match["dir"], "/")) + if _, err := c.getJSON(ctx, url, &commits); err != nil { + return nil, err + } + if len(commits) == 0 { + return nil, NotFoundError{Message: "package directory changed or removed"} + } + + status := Active + lastCommitted := commits[0].CommittedDate + if lastCommitted.Add(ExpiresAfter).Before(time.Now()) { + status = NoRecentCommits + } + + lastCommitOid := commits[0].ID + newEtag := "git-" + lastCommitOid + if newEtag == savedEtag { + return nil, NotModifiedError{ + Since: lastCommitted, + Status: status, + } + } + + var contents []*struct { + ID string `json:"id"` + Type string `json:"type"` + Path string `json:"path"` + } + + var files []*File + var dataURLs []string + var subDirs []string + + for i := 1; true; i++ { + resp, err := c.getJSON(ctx, + expand("https://gitlab.com/api/v4/projects/{project_id}/repository/tree?path={dir}&per_page=100&page=", match, strconv.Itoa(i)), + &contents) + if err != nil { + return nil, err + } + + for _, treeEntry := range contents { + switch treeEntry.Type { + case "blob": + _, name := path.Split(treeEntry.Path) + if isDocFile(name) { + files = append(files, &File{Name: name, BrowseURL: expand("https://gitlab.com/{namespace}/{project}/blob/{ref}/{0}", match, treeEntry.Path)}) + dataURLs = append(dataURLs, expand("https://gitlab.com/api/v4/projects/{project_id}/repository/blobs/{0}/raw", match, treeEntry.ID)) + } + case "tree": + subDirs = append(subDirs, treeEntry.Path) + } + } + + if len(resp.Header["X-Total-Pages"]) == 0 { + break + } + if pages, err := strconv.Atoi(resp.Header["X-Total-Pages"][0]); err != nil || pages <= i { + break + } + } + + if err := c.getFiles(ctx, dataURLs, files); err != nil { + return nil, err + } + + browseURL := project.WebURL + if match["dir"] != "" { + browseURL = expand("https://gitlab.com/{namespace}/{project}/tree/{ref}/{dir}", match) + } + + return &Directory{ + BrowseURL: browseURL, + Etag: newEtag, + Files: files, + Subdirectories: subDirs, + LineFmt: "%s#L%d", + ProjectName: project.Name, + ProjectRoot: expand("gitlab.com/{namespace}/{project}", match), + ProjectURL: project.WebURL, + VCS: "git", + Status: status, + Stars: project.Stars, + }, nil +} + +func getGitLabProject(ctx context.Context, c *http.Client, match map[string]string) (*Project, error) { + pr, err := getGitLabProject(ctx, c, match) + if err != nil { + return nil, err + } + + return &Project{Description: pr.Description}, nil +} + +func gitlabProject(ctx context.Context, c *httpClient, match map[string]string) (*glProject, error) { + var project glProject + + // GitLab API accepts a numerical ID, or a specially encoded string. The ID is unknown + // here, so we use the backup method + reqPath := match["namespace"] + "%2f" + match["project"] + if _, err := c.getJSON(ctx, fmt.Sprintf("https://gitlab.com/api/v4/projects/%v", reqPath), &project); err != nil { + return nil, err + } + + return &project, nil +}