diff --git a/cmd/root.go b/cmd/root.go index a15a593..0b00144 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -72,13 +72,13 @@ var rootCmd = &cobra.Command{ if err != nil { return err } - cmd.Printf("Use %s\n", a.GetName()) + cmd.Printf("Use %s\n", a.Name) m, err := setup.Bin(fsys, sOpt) if err != nil { return err } if len(m) == 0 { - return fmt.Errorf("setup failed: %s", a.GetName()) + return fmt.Errorf("setup failed: %s", a.Name) } cmd.Println("Setup binaries to executable path (PATH):") for b, bp := range m { diff --git a/gh/client.go b/gh/client.go new file mode 100644 index 0000000..570de78 --- /dev/null +++ b/gh/client.go @@ -0,0 +1,280 @@ +package gh + +import ( + "bufio" + "bytes" + "context" + "errors" + "fmt" + "io" + "log" + "net/http" + "strings" + "time" + + "github.com/cli/go-gh" + "github.com/cli/go-gh/pkg/api" + "github.com/google/go-github/v50/github" + "github.com/k1LoW/go-github-client/v50/factory" +) + +type releaseAsset struct { + ID int64 + Name string + ContentType string + DownloadURL string +} + +type client struct { + gc *github.Client + hc *http.Client + owner string + repo string + token string + v3ep string +} + +const ( + defaultV3Endpoint = "https://api.github.com" + acceptHeader = "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9" +) + +func newClient(ctx context.Context, owner, repo string) (*client, error) { + token, v3ep, _, _ := factory.GetTokenAndEndpoints() + if token == "" { + log.Println("No credentials found, access without credentials") + return newNoAuthClient(ctx, owner, repo, v3ep) + } + log.Println("Access with credentials") + gc, err := factory.NewGithubClient(factory.SkipAuth(true)) + if err != nil { + return nil, err + } + if _, _, err := gc.Repositories.Get(ctx, owner, repo); err != nil { + log.Println("Authentication failed, access without credentials") + return newNoAuthClient(ctx, owner, repo, v3ep) + } + hc, err := gh.HTTPClient(&api.ClientOptions{}) + if err != nil { + return nil, err + } + c := &client{ + owner: owner, + repo: repo, + token: token, + v3ep: v3ep, + gc: gc, + hc: hc, + } + return c, nil +} + +func newNoAuthClient(ctx context.Context, owner, repo, v3ep string) (*client, error) { + gc, err := factory.NewGithubClient(factory.SkipAuth(true)) + if err != nil { + return nil, err + } + hc := &http.Client{ + Timeout: 30 * time.Second, + Transport: http.DefaultTransport.(*http.Transport).Clone(), + } + c := &client{ + owner: owner, + repo: repo, + v3ep: v3ep, + gc: gc, + hc: hc, + } + return c, nil +} + +func (c *client) getReleaseAssets(ctx context.Context, opt *AssetOption) ([]*releaseAsset, error) { + if c.token != "" { + assets, err := c.getReleaseAssetsWithAPI(ctx, opt) + if err == nil { + return assets, nil + } + } + return c.getReleaseAssetsWithoutAPI(ctx, opt) +} + +func (c *client) getReleaseAssetsWithoutAPI(ctx context.Context, opt *AssetOption) ([]*releaseAsset, error) { + if c.v3ep != defaultV3Endpoint { + return nil, fmt.Errorf("not support for non API access: %s", c.v3ep) + } + page := 1 + for { + urls, err := c.getReleaseAssetsURL(ctx, page) + if err != nil { + return nil, err + } + if len(urls) == 0 { + break + } + if opt == nil || (opt.Version == "" || opt.Version == versionLatest) { + return c.getReleaseAssetsViaURL(ctx, urls[0]) + } else { + for _, url := range urls { + if strings.HasSuffix(url, opt.Version) { + return c.getReleaseAssetsViaURL(ctx, url) + } + } + } + page++ + } + return nil, errors.New("no assets found") +} + +func (c *client) getReleaseAssetsURL(ctx context.Context, page int) ([]string, error) { + u := fmt.Sprintf("https://github.com/%s/%s/releases?page=%d", c.owner, c.repo, page) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil) + if err != nil { + return nil, err + } + req.Header.Add("Accept", acceptHeader) + resp, err := c.hc.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + b, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + scanner := bufio.NewScanner(bytes.NewReader(b)) + scanner.Split(bufio.ScanLines) + urls := []string{} + for scanner.Scan() { + line := scanner.Text() + if strings.Contains(line, fmt.Sprintf("https://github.com/%s/%s/releases/expanded_assets/", c.owner, c.repo)) { + splitted := strings.Split(line, `src="`) + if len(splitted) == 2 { + splitted2 := strings.Split(splitted[1], `"`) + urls = append(urls, splitted2[0]) + } + } + } + if err := scanner.Err(); err != nil { + return nil, err + } + return urls, nil +} + +func (c *client) getReleaseAssetsViaURL(ctx context.Context, url string) ([]*releaseAsset, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, err + } + req.Header.Add("Accept", acceptHeader) + resp, err := c.hc.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + b, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + scanner := bufio.NewScanner(bytes.NewReader(b)) + scanner.Split(bufio.ScanLines) + assets := []*releaseAsset{} + for scanner.Scan() { + line := scanner.Text() + if strings.Contains(line, "/download/") { + splitted := strings.Split(line, `href="`) + if len(splitted) == 2 { + splitted2 := strings.Split(splitted[1], `"`) + u := fmt.Sprintf("https://github.com%s", splitted2[0]) + splitted3 := strings.Split(splitted2[0], "/") + name := splitted3[len(splitted3)-1] + assets = append(assets, &releaseAsset{ + Name: name, + DownloadURL: u, + }) + } + } + } + if err := scanner.Err(); err != nil { + return nil, err + } + return assets, nil +} + +func (c *client) getReleaseAssetsWithAPI(ctx context.Context, opt *AssetOption) ([]*releaseAsset, error) { + var ( + r *github.RepositoryRelease + err error + ) + if opt == nil || (opt.Version == "" || opt.Version == versionLatest) { + r, _, err = c.gc.Repositories.GetLatestRelease(ctx, c.owner, c.repo) + if err != nil { + return nil, err + } + } else { + r, _, err = c.gc.Repositories.GetReleaseByTag(ctx, c.owner, c.repo, opt.Version) + if err != nil { + return nil, err + } + } + assets := []*releaseAsset{} + for _, a := range r.Assets { + assets = append(assets, &releaseAsset{ + ID: a.GetID(), + Name: a.GetName(), + ContentType: a.GetContentType(), + DownloadURL: a.GetBrowserDownloadURL(), + }) + } + return assets, nil +} + +func (c *client) downloadAsset(ctx context.Context, a *releaseAsset) ([]byte, error) { + if c.token != "" { + b, err := c.downloadAssetWithAPI(ctx, a) + if err == nil { + return b, nil + } + } + return c.downloadAssetWithoutAPI(ctx, a) +} + +func (c *client) downloadAssetWithoutAPI(ctx context.Context, a *releaseAsset) ([]byte, error) { + if a.DownloadURL == "" { + return nil, errors.New("empty download URL") + } + u := a.DownloadURL + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil) + if err != nil { + return nil, err + } + req.Header.Add("Accept", "application/octet-stream") + resp, err := c.hc.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + b, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + return b, nil +} + +func (c *client) downloadAssetWithAPI(ctx context.Context, a *releaseAsset) ([]byte, error) { + u := fmt.Sprintf("%s/repos/%s/%s/releases/assets/%d", c.v3ep, c.owner, c.repo, a.ID) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil) + if err != nil { + return nil, err + } + req.Header.Add("Accept", "application/octet-stream") + resp, err := c.hc.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + b, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + return b, nil +} diff --git a/gh/client_test.go b/gh/client_test.go new file mode 100644 index 0000000..94be457 --- /dev/null +++ b/gh/client_test.go @@ -0,0 +1,32 @@ +package gh + +import ( + "context" + "testing" +) + +func TestGetReleaseAssetsWithoutAPI(t *testing.T) { + tests := []struct { + owner string + repo string + opt *AssetOption + }{ + {"k1LoW", "tbls", nil}, + } + ctx := context.Background() + for _, tt := range tests { + c, err := newClient(ctx, tt.owner, tt.repo) + if err != nil { + t.Error(err) + continue + } + assets, err := c.getReleaseAssetsWithoutAPI(ctx, tt.opt) + if err != nil { + t.Error(err) + continue + } + if len(assets) == 0 { + t.Error("want assets") + } + } +} diff --git a/gh/gh.go b/gh/gh.go index 591e8d3..17842fe 100644 --- a/gh/gh.go +++ b/gh/gh.go @@ -19,10 +19,7 @@ import ( "time" "github.com/cli/go-gh" - "github.com/cli/go-gh/pkg/api" "github.com/cli/go-gh/pkg/repository" - "github.com/google/go-github/v50/github" - "github.com/k1LoW/go-github-client/v50/factory" "github.com/nlepage/go-tarfs" ) @@ -47,6 +44,8 @@ var supportContentType = []string{ "application/octet-stream", } +const versionLatest = "latest" + type AssetOption struct { Match string Version string @@ -55,29 +54,24 @@ type AssetOption struct { Strict bool } -func GetReleaseAsset(ctx context.Context, owner, repo string, opt *AssetOption) (*github.ReleaseAsset, fs.FS, error) { - const versionLatest = "latest" - c, err := client(ctx, owner, repo) +func GetReleaseAsset(ctx context.Context, owner, repo string, opt *AssetOption) (*releaseAsset, fs.FS, error) { + c, err := newClient(ctx, owner, repo) if err != nil { return nil, nil, err } - var r *github.RepositoryRelease - if opt != nil && (opt.Version == "" || opt.Version == versionLatest) { - r, _, err = c.Repositories.GetLatestRelease(ctx, owner, repo) - if err != nil { - return nil, nil, err - } - } else { - r, _, err = c.Repositories.GetReleaseByTag(ctx, owner, repo, opt.Version) - if err != nil { - return nil, nil, err - } + assets, err := c.getReleaseAssets(ctx, opt) + if err != nil { + return nil, nil, err + } + a, err := detectAsset(assets, opt) + if err != nil { + return nil, nil, err } - a, err := detectAsset(r.Assets, opt) + b, err := c.downloadAsset(ctx, a) if err != nil { return nil, nil, err } - fsys, err := makeFS(owner, repo, a) + fsys, err := makeFS(ctx, b, repo, a.Name, []string{a.ContentType, http.DetectContentType(b)}) if err != nil { return nil, nil, err } @@ -106,7 +100,7 @@ func DetectHostOwnerRepo(ownerrepo string) (string, string, string, error) { return host, owner, repo, nil } -func detectAsset(assets []*github.ReleaseAsset, opt *AssetOption) (*github.ReleaseAsset, error) { +func detectAsset(assets []*releaseAsset, opt *AssetOption) (*releaseAsset, error) { var ( od, ad, om *regexp.Regexp err error @@ -129,15 +123,15 @@ func detectAsset(assets []*github.ReleaseAsset, opt *AssetOption) (*github.Relea } type assetScore struct { - asset *github.ReleaseAsset + asset *releaseAsset score int } assetScores := []*assetScore{} for _, a := range assets { - if om != nil && om.MatchString(a.GetName()) { + if om != nil && om.MatchString(a.Name) { return a, nil } - if !contains(supportContentType, a.GetContentType()) { + if a.ContentType != "" && !contains(supportContentType, a.ContentType) { continue } as := &assetScore{ @@ -146,19 +140,19 @@ func detectAsset(assets []*github.ReleaseAsset, opt *AssetOption) (*github.Relea } assetScores = append(assetScores, as) // os - if od.MatchString(a.GetName()) { + if od.MatchString(a.Name) { as.score += 7 } // arch - if ad.MatchString(a.GetName()) { + if ad.MatchString(a.Name) { as.score += 3 } // content type - if a.GetContentType() == "application/octet-stream" { + if a.ContentType == "application/octet-stream" { as.score += 1 } } - if opt.Strict && om != nil { + if opt != nil && opt.Strict && om != nil { return nil, fmt.Errorf("no matching assets found: %s", opt.Match) } if len(assetScores) == 0 { @@ -169,7 +163,7 @@ func detectAsset(assets []*github.ReleaseAsset, opt *AssetOption) (*github.Relea return assetScores[i].score > assetScores[j].score }) - if opt.Strict && assetScores[0].score < 10 { + if opt != nil && opt.Strict && assetScores[0].score < 10 { return nil, fmt.Errorf("no matching assets found for OS/Arch: %s/%s", opt.OS, opt.Arch) } @@ -185,23 +179,27 @@ func getDictRegexp(key string, dict map[string][]string) *regexp.Regexp { return regexp.MustCompile(fmt.Sprintf("(?i)(%s)", strings.ToLower(key))) } -func makeFS(owner, repo string, a *github.ReleaseAsset) (fs.FS, error) { - b, err := downloadAsset(owner, repo, a) - if err != nil { - return nil, err +func contains(s []string, e string) bool { + for _, v := range s { + if e == v { + return true + } } - cts := []string{a.GetContentType(), http.DetectContentType(b)} - log.Println("asset content type:", cts) + return false +} + +func makeFS(ctx context.Context, b []byte, repo, name string, contentTypes []string) (fs.FS, error) { + log.Println("asset content type:", contentTypes) switch { - case matchContentTypes([]string{"application/zip", "application/x-zip-compressed"}, cts): + case matchContentTypes([]string{"application/zip", "application/x-zip-compressed"}, contentTypes): return zip.NewReader(bytes.NewReader(b), int64(len(b))) - case matchContentTypes([]string{"application/gzip", "application/x-gzip"}, cts): + case matchContentTypes([]string{"application/gzip", "application/x-gzip"}, contentTypes): gr, err := gzip.NewReader(bytes.NewReader(b)) if err != nil { return nil, err } defer gr.Close() - if strings.HasSuffix(a.GetName(), ".tar.gz") { + if strings.HasSuffix(name, ".tar.gz") { fsys, err := tarfs.New(gr) if err != nil { return nil, err @@ -220,7 +218,7 @@ func makeFS(owner, repo string, a *github.ReleaseAsset) (fs.FS, error) { } return fsys, nil } - case matchContentTypes([]string{"application/octet-stream"}, cts): + case matchContentTypes([]string{"application/octet-stream"}, contentTypes): fsys := fstest.MapFS{} fsys[repo] = &fstest.MapFile{ Data: b, @@ -229,93 +227,8 @@ func makeFS(owner, repo string, a *github.ReleaseAsset) (fs.FS, error) { } return fsys, nil default: - return nil, fmt.Errorf("unsupport content type: %s", a.GetContentType()) - } -} - -func downloadAsset(owner, repo string, a *github.ReleaseAsset) ([]byte, error) { - client, err := httpClient(owner, repo) - if err != nil { - return nil, err - } - _, v3ep, _, _ := factory.GetTokenAndEndpoints() - u := fmt.Sprintf("%s/repos/%s/%s/releases/assets/%d", v3ep, owner, repo, a.GetID()) - req, err := http.NewRequest(http.MethodGet, u, nil) - if err != nil { - return nil, err - } - req.Header.Add("Accept", "application/octet-stream") - resp, err := client.Do(req) - if err != nil { - return nil, err - } - defer resp.Body.Close() - b, err := io.ReadAll(resp.Body) - if err != nil { - return nil, err - } - return b, nil -} - -func contains(s []string, e string) bool { - for _, v := range s { - if e == v { - return true - } - } - return false -} - -func client(ctx context.Context, owner, repo string) (*github.Client, error) { - token, _, _, _ := factory.GetTokenAndEndpoints() - if token == "" { - log.Println("No credentials found, access without credentials") - return factory.NewGithubClient(factory.SkipAuth(true)) - } - log.Println("Access with credentials") - c, err := factory.NewGithubClient() - if err != nil { - return nil, err - } - if _, _, err := c.Repositories.Get(ctx, owner, repo); err != nil { - log.Println("Authentication failed, access without credentials") - return factory.NewGithubClient(factory.SkipAuth(true)) - } - return c, nil -} - -func httpClient(owner, repo string) (*http.Client, error) { - token, v3ep, _, _ := factory.GetTokenAndEndpoints() - if token == "" { - log.Println("No credentials found, access without credentials") - return &http.Client{ - Timeout: 30 * time.Second, - Transport: http.DefaultTransport.(*http.Transport).Clone(), - }, nil - } - log.Println("Access with credentials") - client, err := gh.HTTPClient(&api.ClientOptions{}) - if err != nil { - return nil, err - } - u := fmt.Sprintf("%s/repos/%s/%s", v3ep, owner, repo) - req, err := http.NewRequest(http.MethodGet, u, nil) - if err != nil { - return nil, err - } - resp, err := client.Do(req) - if err != nil { - return nil, err - } - resp.Body.Close() - if resp.StatusCode != http.StatusOK { - log.Println("Authentication failed, access without credentials") - client = &http.Client{ - Timeout: 30 * time.Second, - Transport: http.DefaultTransport.(*http.Transport).Clone(), - } + return nil, fmt.Errorf("unsupport content types: %s", contentTypes) } - return client, nil } func matchContentTypes(m, ct []string) bool { diff --git a/gh/gh_test.go b/gh/gh_test.go new file mode 100644 index 0000000..addd789 --- /dev/null +++ b/gh/gh_test.go @@ -0,0 +1,48 @@ +package gh + +import ( + "context" + "io/fs" + "os" + "testing" + + "github.com/k1LoW/go-github-client/v50/factory" +) + +func TestMain(m *testing.M) { + os.Setenv("GH_CONFIG_DIR", "/tmp") // Disable reading credentials from config + m.Run() +} + +func TestGetReleaseAsset(t *testing.T) { + tests := []struct { + owner string + repo string + opt *AssetOption + wantFile string + useToken bool + }{ + {"k1LoW", "tbls", nil, "tbls", true}, + {"k1LoW", "tbls", nil, "tbls", false}, + {"k1LoW", "tbls", &AssetOption{Version: "v1.60.0"}, "tbls", true}, + {"k1LoW", "tbls", &AssetOption{Version: "v1.60.0"}, "tbls", false}, + } + ctx := context.Background() + token, _, _, _ := factory.GetTokenAndEndpoints() + for _, tt := range tests { + if tt.useToken { + t.Setenv("GITHUB_TOKEN", token) + } else { + t.Setenv("GITHUB_TOKEN", "") + t.Setenv("GH_TOKEN", "") + } + _, fsys, err := GetReleaseAsset(ctx, tt.owner, tt.repo, tt.opt) + if err != nil { + t.Error(err) + continue + } + if _, err := fs.ReadFile(fsys, tt.wantFile); err != nil { + t.Error(err) + } + } +}