From 81f890be2502d84664aacfd7c2ffd2ab3e645433 Mon Sep 17 00:00:00 2001 From: k1LoW Date: Sat, 18 Feb 2023 12:42:14 +0900 Subject: [PATCH 1/6] Fix GitHub client --- gh/gh.go | 166 +++++++++++++++++++++++++++---------------------------- 1 file changed, 83 insertions(+), 83 deletions(-) diff --git a/gh/gh.go b/gh/gh.go index 591e8d3..c73a046 100644 --- a/gh/gh.go +++ b/gh/gh.go @@ -57,18 +57,18 @@ type AssetOption struct { func GetReleaseAsset(ctx context.Context, owner, repo string, opt *AssetOption) (*github.ReleaseAsset, fs.FS, error) { const versionLatest = "latest" - c, err := client(ctx, owner, repo) + c, err := newClient(ctx, owner, repo) if err != nil { return nil, nil, err } var r *github.RepositoryRelease if opt != nil && (opt.Version == "" || opt.Version == versionLatest) { - r, _, err = c.Repositories.GetLatestRelease(ctx, owner, repo) + r, _, err = c.gc.Repositories.GetLatestRelease(ctx, owner, repo) if err != nil { return nil, nil, err } } else { - r, _, err = c.Repositories.GetReleaseByTag(ctx, owner, repo, opt.Version) + r, _, err = c.gc.Repositories.GetReleaseByTag(ctx, owner, repo, opt.Version) if err != nil { return nil, nil, err } @@ -77,7 +77,11 @@ func GetReleaseAsset(ctx context.Context, owner, repo string, opt *AssetOption) if err != nil { return nil, nil, err } - fsys, err := makeFS(owner, repo, a) + b, err := c.downloadAsset(ctx, a) + if err != nil { + return nil, nil, err + } + fsys, err := makeFS(ctx, b, repo, a.GetName(), []string{a.GetContentType(), http.DetectContentType(b)}) if err != nil { return nil, nil, err } @@ -185,23 +189,85 @@ func getDictRegexp(key string, dict map[string][]string) *regexp.Regexp { return regexp.MustCompile(fmt.Sprintf("(?i)(%s)", strings.ToLower(key))) } -func makeFS(owner, repo string, a *github.ReleaseAsset) (fs.FS, error) { - b, err := downloadAsset(owner, repo, a) +func contains(s []string, e string) bool { + for _, v := range s { + if e == v { + return true + } + } + return false +} + +type client struct { + gc *github.Client + hc *http.Client + owner string + repo string + token string + v3ep string +} + +func newClient(ctx context.Context, owner, repo string) (*client, error) { + token, v3ep, _, _ := factory.GetTokenAndEndpoints() + if token == "" { + log.Println("No credentials found, access without credentials") + return newNoAuthClient(ctx, owner, repo, v3ep) + } + log.Println("Access with credentials") + gc, err := factory.NewGithubClient(factory.SkipAuth(true)) + if err != nil { + return nil, err + } + if _, _, err := gc.Repositories.Get(ctx, owner, repo); err != nil { + log.Println("Authentication failed, access without credentials") + return newNoAuthClient(ctx, owner, repo, v3ep) + } + hc, err := gh.HTTPClient(&api.ClientOptions{}) + if err != nil { + return nil, err + } + c := &client{ + owner: owner, + repo: repo, + token: token, + v3ep: v3ep, + gc: gc, + hc: hc, + } + return c, nil +} + +func newNoAuthClient(ctx context.Context, owner, repo, v3ep string) (*client, error) { + gc, err := factory.NewGithubClient(factory.SkipAuth(true)) if err != nil { return nil, err } - cts := []string{a.GetContentType(), http.DetectContentType(b)} - log.Println("asset content type:", cts) + hc := &http.Client{ + Timeout: 30 * time.Second, + Transport: http.DefaultTransport.(*http.Transport).Clone(), + } + c := &client{ + owner: owner, + repo: repo, + v3ep: v3ep, + gc: gc, + hc: hc, + } + return c, nil +} + +func makeFS(ctx context.Context, b []byte, repo, name string, contentTypes []string) (fs.FS, error) { + log.Println("asset content type:", contentTypes) switch { - case matchContentTypes([]string{"application/zip", "application/x-zip-compressed"}, cts): + case matchContentTypes([]string{"application/zip", "application/x-zip-compressed"}, contentTypes): return zip.NewReader(bytes.NewReader(b), int64(len(b))) - case matchContentTypes([]string{"application/gzip", "application/x-gzip"}, cts): + case matchContentTypes([]string{"application/gzip", "application/x-gzip"}, contentTypes): gr, err := gzip.NewReader(bytes.NewReader(b)) if err != nil { return nil, err } defer gr.Close() - if strings.HasSuffix(a.GetName(), ".tar.gz") { + if strings.HasSuffix(name, ".tar.gz") { fsys, err := tarfs.New(gr) if err != nil { return nil, err @@ -220,7 +286,7 @@ func makeFS(owner, repo string, a *github.ReleaseAsset) (fs.FS, error) { } return fsys, nil } - case matchContentTypes([]string{"application/octet-stream"}, cts): + case matchContentTypes([]string{"application/octet-stream"}, contentTypes): fsys := fstest.MapFS{} fsys[repo] = &fstest.MapFile{ Data: b, @@ -229,23 +295,18 @@ func makeFS(owner, repo string, a *github.ReleaseAsset) (fs.FS, error) { } return fsys, nil default: - return nil, fmt.Errorf("unsupport content type: %s", a.GetContentType()) + return nil, fmt.Errorf("unsupport content types: %s", contentTypes) } } -func downloadAsset(owner, repo string, a *github.ReleaseAsset) ([]byte, error) { - client, err := httpClient(owner, repo) - if err != nil { - return nil, err - } - _, v3ep, _, _ := factory.GetTokenAndEndpoints() - u := fmt.Sprintf("%s/repos/%s/%s/releases/assets/%d", v3ep, owner, repo, a.GetID()) - req, err := http.NewRequest(http.MethodGet, u, nil) +func (c *client) downloadAsset(ctx context.Context, a *github.ReleaseAsset) ([]byte, error) { + u := fmt.Sprintf("%s/repos/%s/%s/releases/assets/%d", c.v3ep, c.owner, c.repo, a.GetID()) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil) if err != nil { return nil, err } req.Header.Add("Accept", "application/octet-stream") - resp, err := client.Do(req) + resp, err := c.hc.Do(req) if err != nil { return nil, err } @@ -257,67 +318,6 @@ func downloadAsset(owner, repo string, a *github.ReleaseAsset) ([]byte, error) { return b, nil } -func contains(s []string, e string) bool { - for _, v := range s { - if e == v { - return true - } - } - return false -} - -func client(ctx context.Context, owner, repo string) (*github.Client, error) { - token, _, _, _ := factory.GetTokenAndEndpoints() - if token == "" { - log.Println("No credentials found, access without credentials") - return factory.NewGithubClient(factory.SkipAuth(true)) - } - log.Println("Access with credentials") - c, err := factory.NewGithubClient() - if err != nil { - return nil, err - } - if _, _, err := c.Repositories.Get(ctx, owner, repo); err != nil { - log.Println("Authentication failed, access without credentials") - return factory.NewGithubClient(factory.SkipAuth(true)) - } - return c, nil -} - -func httpClient(owner, repo string) (*http.Client, error) { - token, v3ep, _, _ := factory.GetTokenAndEndpoints() - if token == "" { - log.Println("No credentials found, access without credentials") - return &http.Client{ - Timeout: 30 * time.Second, - Transport: http.DefaultTransport.(*http.Transport).Clone(), - }, nil - } - log.Println("Access with credentials") - client, err := gh.HTTPClient(&api.ClientOptions{}) - if err != nil { - return nil, err - } - u := fmt.Sprintf("%s/repos/%s/%s", v3ep, owner, repo) - req, err := http.NewRequest(http.MethodGet, u, nil) - if err != nil { - return nil, err - } - resp, err := client.Do(req) - if err != nil { - return nil, err - } - resp.Body.Close() - if resp.StatusCode != http.StatusOK { - log.Println("Authentication failed, access without credentials") - client = &http.Client{ - Timeout: 30 * time.Second, - Transport: http.DefaultTransport.(*http.Transport).Clone(), - } - } - return client, nil -} - func matchContentTypes(m, ct []string) bool { for _, v := range m { for _, vv := range ct { From cd26f89c766a05ef8ed4235a0be10f377e3bc183 Mon Sep 17 00:00:00 2001 From: k1LoW Date: Sat, 18 Feb 2023 14:30:51 +0900 Subject: [PATCH 2/6] Fix opt handling --- gh/gh.go | 6 +++--- gh/gh_test.go | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 3 deletions(-) create mode 100644 gh/gh_test.go diff --git a/gh/gh.go b/gh/gh.go index c73a046..30c1908 100644 --- a/gh/gh.go +++ b/gh/gh.go @@ -62,7 +62,7 @@ func GetReleaseAsset(ctx context.Context, owner, repo string, opt *AssetOption) return nil, nil, err } var r *github.RepositoryRelease - if opt != nil && (opt.Version == "" || opt.Version == versionLatest) { + if opt == nil || (opt.Version == "" || opt.Version == versionLatest) { r, _, err = c.gc.Repositories.GetLatestRelease(ctx, owner, repo) if err != nil { return nil, nil, err @@ -162,7 +162,7 @@ func detectAsset(assets []*github.ReleaseAsset, opt *AssetOption) (*github.Relea as.score += 1 } } - if opt.Strict && om != nil { + if opt != nil && opt.Strict && om != nil { return nil, fmt.Errorf("no matching assets found: %s", opt.Match) } if len(assetScores) == 0 { @@ -173,7 +173,7 @@ func detectAsset(assets []*github.ReleaseAsset, opt *AssetOption) (*github.Relea return assetScores[i].score > assetScores[j].score }) - if opt.Strict && assetScores[0].score < 10 { + if opt != nil && opt.Strict && assetScores[0].score < 10 { return nil, fmt.Errorf("no matching assets found for OS/Arch: %s/%s", opt.OS, opt.Arch) } diff --git a/gh/gh_test.go b/gh/gh_test.go new file mode 100644 index 0000000..e4782ab --- /dev/null +++ b/gh/gh_test.go @@ -0,0 +1,36 @@ +package gh + +import ( + "context" + "io/fs" + "testing" +) + +func TestGetReleaseAsset(t *testing.T) { + tests := []struct { + owner string + repo string + opt *AssetOption + wantFile string + useToken bool + }{ + {"k1LoW", "tbls", nil, "tbls", true}, + {"k1LoW", "tbls", nil, "tbls", false}, + } + ctx := context.Background() + for _, tt := range tests { + if !tt.useToken { + t.Setenv("GITHUB_TOKEN", "") + t.Setenv("GH_TOKEN", "") + } + _, fsys, err := GetReleaseAsset(ctx, tt.owner, tt.repo, tt.opt) + if err != nil { + t.Error(err) + return + } + if _, err := fs.ReadFile(fsys, tt.wantFile); err != nil { + t.Error(err) + return + } + } +} From 1a5151146f999abdd1b4b7b56a3cdf1b5f844bc1 Mon Sep 17 00:00:00 2001 From: k1LoW Date: Sat, 18 Feb 2023 14:56:46 +0900 Subject: [PATCH 3/6] Fix test --- gh/gh_test.go | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/gh/gh_test.go b/gh/gh_test.go index e4782ab..ff541d9 100644 --- a/gh/gh_test.go +++ b/gh/gh_test.go @@ -4,6 +4,8 @@ import ( "context" "io/fs" "testing" + + "github.com/k1LoW/go-github-client/v50/factory" ) func TestGetReleaseAsset(t *testing.T) { @@ -16,21 +18,26 @@ func TestGetReleaseAsset(t *testing.T) { }{ {"k1LoW", "tbls", nil, "tbls", true}, {"k1LoW", "tbls", nil, "tbls", false}, + {"k1LoW", "tbls", &AssetOption{Version: "v1.60.0"}, "tbls", true}, + {"k1LoW", "tbls", &AssetOption{Version: "v1.60.0"}, "tbls", false}, } ctx := context.Background() + t.Setenv("GH_CONFIG_DIR", "/tmp") + token, _, _, _ := factory.GetTokenAndEndpoints() for _, tt := range tests { - if !tt.useToken { + if tt.useToken { + t.Setenv("GITHUB_TOKEN", token) + } else { t.Setenv("GITHUB_TOKEN", "") t.Setenv("GH_TOKEN", "") } _, fsys, err := GetReleaseAsset(ctx, tt.owner, tt.repo, tt.opt) if err != nil { t.Error(err) - return + continue } if _, err := fs.ReadFile(fsys, tt.wantFile); err != nil { t.Error(err) - return } } } From 983e44fbfa8127c897e54ba3662dc2616f55d743 Mon Sep 17 00:00:00 2001 From: k1LoW Date: Sat, 18 Feb 2023 15:40:19 +0900 Subject: [PATCH 4/6] Refactor --- cmd/root.go | 4 +- gh/client.go | 125 +++++++++++++++++++++++++++++++++++++++++++++++++++ gh/gh.go | 117 +++++++---------------------------------------- 3 files changed, 142 insertions(+), 104 deletions(-) create mode 100644 gh/client.go diff --git a/cmd/root.go b/cmd/root.go index a15a593..0b00144 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -72,13 +72,13 @@ var rootCmd = &cobra.Command{ if err != nil { return err } - cmd.Printf("Use %s\n", a.GetName()) + cmd.Printf("Use %s\n", a.Name) m, err := setup.Bin(fsys, sOpt) if err != nil { return err } if len(m) == 0 { - return fmt.Errorf("setup failed: %s", a.GetName()) + return fmt.Errorf("setup failed: %s", a.Name) } cmd.Println("Setup binaries to executable path (PATH):") for b, bp := range m { diff --git a/gh/client.go b/gh/client.go new file mode 100644 index 0000000..f3dd73f --- /dev/null +++ b/gh/client.go @@ -0,0 +1,125 @@ +package gh + +import ( + "context" + "fmt" + "io" + "log" + "net/http" + "time" + + "github.com/cli/go-gh" + "github.com/cli/go-gh/pkg/api" + "github.com/google/go-github/v50/github" + "github.com/k1LoW/go-github-client/v50/factory" +) + +type releaseAsset struct { + ID int64 + Name string + ContentType string +} + +type client struct { + gc *github.Client + hc *http.Client + owner string + repo string + token string + v3ep string +} + +func newClient(ctx context.Context, owner, repo string) (*client, error) { + token, v3ep, _, _ := factory.GetTokenAndEndpoints() + if token == "" { + log.Println("No credentials found, access without credentials") + return newNoAuthClient(ctx, owner, repo, v3ep) + } + log.Println("Access with credentials") + gc, err := factory.NewGithubClient(factory.SkipAuth(true)) + if err != nil { + return nil, err + } + if _, _, err := gc.Repositories.Get(ctx, owner, repo); err != nil { + log.Println("Authentication failed, access without credentials") + return newNoAuthClient(ctx, owner, repo, v3ep) + } + hc, err := gh.HTTPClient(&api.ClientOptions{}) + if err != nil { + return nil, err + } + c := &client{ + owner: owner, + repo: repo, + token: token, + v3ep: v3ep, + gc: gc, + hc: hc, + } + return c, nil +} + +func newNoAuthClient(ctx context.Context, owner, repo, v3ep string) (*client, error) { + gc, err := factory.NewGithubClient(factory.SkipAuth(true)) + if err != nil { + return nil, err + } + hc := &http.Client{ + Timeout: 30 * time.Second, + Transport: http.DefaultTransport.(*http.Transport).Clone(), + } + c := &client{ + owner: owner, + repo: repo, + v3ep: v3ep, + gc: gc, + hc: hc, + } + return c, nil +} + +func (c *client) getReleaseAssets(ctx context.Context, opt *AssetOption) ([]*releaseAsset, error) { + var ( + r *github.RepositoryRelease + err error + ) + if opt == nil || (opt.Version == "" || opt.Version == versionLatest) { + r, _, err = c.gc.Repositories.GetLatestRelease(ctx, c.owner, c.repo) + if err != nil { + return nil, err + } + } else { + r, _, err = c.gc.Repositories.GetReleaseByTag(ctx, c.owner, c.repo, opt.Version) + if err != nil { + return nil, err + } + } + assets := []*releaseAsset{} + for _, a := range r.Assets { + assets = append(assets, &releaseAsset{ + ID: a.GetID(), + Name: a.GetName(), + ContentType: a.GetContentType(), + }) + } + return assets, nil +} + +func (c *client) downloadAsset(ctx context.Context, a *releaseAsset) ([]byte, error) { + u := fmt.Sprintf("%s/repos/%s/%s/releases/assets/%d", c.v3ep, c.owner, c.repo, a.ID) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil) + if err != nil { + return nil, err + } + req.Header.Add("Accept", "application/octet-stream") + resp, err := c.hc.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + b, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + return b, nil +} diff --git a/gh/gh.go b/gh/gh.go index 30c1908..e233026 100644 --- a/gh/gh.go +++ b/gh/gh.go @@ -19,10 +19,7 @@ import ( "time" "github.com/cli/go-gh" - "github.com/cli/go-gh/pkg/api" "github.com/cli/go-gh/pkg/repository" - "github.com/google/go-github/v50/github" - "github.com/k1LoW/go-github-client/v50/factory" "github.com/nlepage/go-tarfs" ) @@ -47,6 +44,8 @@ var supportContentType = []string{ "application/octet-stream", } +const versionLatest = "latest" + type AssetOption struct { Match string Version string @@ -55,25 +54,16 @@ type AssetOption struct { Strict bool } -func GetReleaseAsset(ctx context.Context, owner, repo string, opt *AssetOption) (*github.ReleaseAsset, fs.FS, error) { - const versionLatest = "latest" +func GetReleaseAsset(ctx context.Context, owner, repo string, opt *AssetOption) (*releaseAsset, fs.FS, error) { c, err := newClient(ctx, owner, repo) if err != nil { return nil, nil, err } - var r *github.RepositoryRelease - if opt == nil || (opt.Version == "" || opt.Version == versionLatest) { - r, _, err = c.gc.Repositories.GetLatestRelease(ctx, owner, repo) - if err != nil { - return nil, nil, err - } - } else { - r, _, err = c.gc.Repositories.GetReleaseByTag(ctx, owner, repo, opt.Version) - if err != nil { - return nil, nil, err - } + assets, err := c.getReleaseAssets(ctx, opt) + if err != nil { + return nil, nil, err } - a, err := detectAsset(r.Assets, opt) + a, err := detectAsset(assets, opt) if err != nil { return nil, nil, err } @@ -81,7 +71,7 @@ func GetReleaseAsset(ctx context.Context, owner, repo string, opt *AssetOption) if err != nil { return nil, nil, err } - fsys, err := makeFS(ctx, b, repo, a.GetName(), []string{a.GetContentType(), http.DetectContentType(b)}) + fsys, err := makeFS(ctx, b, repo, a.Name, []string{a.ContentType, http.DetectContentType(b)}) if err != nil { return nil, nil, err } @@ -110,7 +100,7 @@ func DetectHostOwnerRepo(ownerrepo string) (string, string, string, error) { return host, owner, repo, nil } -func detectAsset(assets []*github.ReleaseAsset, opt *AssetOption) (*github.ReleaseAsset, error) { +func detectAsset(assets []*releaseAsset, opt *AssetOption) (*releaseAsset, error) { var ( od, ad, om *regexp.Regexp err error @@ -133,15 +123,15 @@ func detectAsset(assets []*github.ReleaseAsset, opt *AssetOption) (*github.Relea } type assetScore struct { - asset *github.ReleaseAsset + asset *releaseAsset score int } assetScores := []*assetScore{} for _, a := range assets { - if om != nil && om.MatchString(a.GetName()) { + if om != nil && om.MatchString(a.Name) { return a, nil } - if !contains(supportContentType, a.GetContentType()) { + if !contains(supportContentType, a.ContentType) { continue } as := &assetScore{ @@ -150,15 +140,15 @@ func detectAsset(assets []*github.ReleaseAsset, opt *AssetOption) (*github.Relea } assetScores = append(assetScores, as) // os - if od.MatchString(a.GetName()) { + if od.MatchString(a.Name) { as.score += 7 } // arch - if ad.MatchString(a.GetName()) { + if ad.MatchString(a.Name) { as.score += 3 } // content type - if a.GetContentType() == "application/octet-stream" { + if a.ContentType == "application/octet-stream" { as.score += 1 } } @@ -198,64 +188,6 @@ func contains(s []string, e string) bool { return false } -type client struct { - gc *github.Client - hc *http.Client - owner string - repo string - token string - v3ep string -} - -func newClient(ctx context.Context, owner, repo string) (*client, error) { - token, v3ep, _, _ := factory.GetTokenAndEndpoints() - if token == "" { - log.Println("No credentials found, access without credentials") - return newNoAuthClient(ctx, owner, repo, v3ep) - } - log.Println("Access with credentials") - gc, err := factory.NewGithubClient(factory.SkipAuth(true)) - if err != nil { - return nil, err - } - if _, _, err := gc.Repositories.Get(ctx, owner, repo); err != nil { - log.Println("Authentication failed, access without credentials") - return newNoAuthClient(ctx, owner, repo, v3ep) - } - hc, err := gh.HTTPClient(&api.ClientOptions{}) - if err != nil { - return nil, err - } - c := &client{ - owner: owner, - repo: repo, - token: token, - v3ep: v3ep, - gc: gc, - hc: hc, - } - return c, nil -} - -func newNoAuthClient(ctx context.Context, owner, repo, v3ep string) (*client, error) { - gc, err := factory.NewGithubClient(factory.SkipAuth(true)) - if err != nil { - return nil, err - } - hc := &http.Client{ - Timeout: 30 * time.Second, - Transport: http.DefaultTransport.(*http.Transport).Clone(), - } - c := &client{ - owner: owner, - repo: repo, - v3ep: v3ep, - gc: gc, - hc: hc, - } - return c, nil -} - func makeFS(ctx context.Context, b []byte, repo, name string, contentTypes []string) (fs.FS, error) { log.Println("asset content type:", contentTypes) switch { @@ -299,25 +231,6 @@ func makeFS(ctx context.Context, b []byte, repo, name string, contentTypes []str } } -func (c *client) downloadAsset(ctx context.Context, a *github.ReleaseAsset) ([]byte, error) { - u := fmt.Sprintf("%s/repos/%s/%s/releases/assets/%d", c.v3ep, c.owner, c.repo, a.GetID()) - req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil) - if err != nil { - return nil, err - } - req.Header.Add("Accept", "application/octet-stream") - resp, err := c.hc.Do(req) - if err != nil { - return nil, err - } - defer resp.Body.Close() - b, err := io.ReadAll(resp.Body) - if err != nil { - return nil, err - } - return b, nil -} - func matchContentTypes(m, ct []string) bool { for _, v := range m { for _, vv := range ct { From b59bd1c4b129dc03c642fac43e60300f949f2395 Mon Sep 17 00:00:00 2001 From: k1LoW Date: Sat, 18 Feb 2023 16:01:00 +0900 Subject: [PATCH 5/6] Fix to use API as little as possible when downloading asset without token --- gh/client.go | 155 ++++++++++++++++++++++++++++++++++++++++++++++ gh/client_test.go | 32 ++++++++++ gh/gh.go | 2 +- gh/gh_test.go | 7 ++- 4 files changed, 194 insertions(+), 2 deletions(-) create mode 100644 gh/client_test.go diff --git a/gh/client.go b/gh/client.go index f3dd73f..deac479 100644 --- a/gh/client.go +++ b/gh/client.go @@ -1,11 +1,15 @@ package gh import ( + "bufio" + "bytes" "context" + "errors" "fmt" "io" "log" "net/http" + "strings" "time" "github.com/cli/go-gh" @@ -18,6 +22,7 @@ type releaseAsset struct { ID int64 Name string ContentType string + DownloadURL string } type client struct { @@ -29,6 +34,11 @@ type client struct { v3ep string } +const ( + defaultV3Endpoint = "https://api.github.com" + acceptHeader = "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9" +) + func newClient(ctx context.Context, owner, repo string) (*client, error) { token, v3ep, _, _ := factory.GetTokenAndEndpoints() if token == "" { @@ -79,6 +89,118 @@ func newNoAuthClient(ctx context.Context, owner, repo, v3ep string) (*client, er } func (c *client) getReleaseAssets(ctx context.Context, opt *AssetOption) ([]*releaseAsset, error) { + if c.token != "" { + assets, err := c.getReleaseAssetsWithAPI(ctx, opt) + if err == nil { + return assets, nil + } + } + return c.getReleaseAssetsWithoutAPI(ctx, opt) +} + +func (c *client) getReleaseAssetsWithoutAPI(ctx context.Context, opt *AssetOption) ([]*releaseAsset, error) { + if c.v3ep != defaultV3Endpoint { + return nil, fmt.Errorf("not support for non API access: %s", c.v3ep) + } + page := 1 + for { + urls, err := c.getReleaseAssetsURL(ctx, page) + if err != nil { + return nil, err + } + if len(urls) == 0 { + break + } + if opt == nil || (opt.Version == "" || opt.Version == versionLatest) { + return c.getReleaseAssetsViaURL(ctx, urls[0]) + } else { + for _, url := range urls { + if strings.HasSuffix(url, opt.Version) { + return c.getReleaseAssetsViaURL(ctx, url) + } + } + } + page++ + } + return nil, errors.New("no assets found") +} + +func (c *client) getReleaseAssetsURL(ctx context.Context, page int) ([]string, error) { + u := fmt.Sprintf("https://github.com/%s/%s/releases?page=%d", c.owner, c.repo, page) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil) + if err != nil { + return nil, err + } + req.Header.Add("Accept", acceptHeader) + resp, err := c.hc.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + b, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + scanner := bufio.NewScanner(bytes.NewReader(b)) + scanner.Split(bufio.ScanLines) + urls := []string{} + for scanner.Scan() { + line := scanner.Text() + if strings.Contains(line, fmt.Sprintf("https://github.com/%s/%s/releases/expanded_assets/", c.owner, c.repo)) { + splitted := strings.Split(line, `src="`) + if len(splitted) == 2 { + splitted2 := strings.Split(splitted[1], `"`) + urls = append(urls, splitted2[0]) + } + } + } + if err := scanner.Err(); err != nil { + return nil, err + } + return urls, nil +} + +func (c *client) getReleaseAssetsViaURL(ctx context.Context, url string) ([]*releaseAsset, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, err + } + req.Header.Add("Accept", acceptHeader) + resp, err := c.hc.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + b, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + scanner := bufio.NewScanner(bytes.NewReader(b)) + scanner.Split(bufio.ScanLines) + assets := []*releaseAsset{} + for scanner.Scan() { + line := scanner.Text() + if strings.Contains(line, "/download/") { + splitted := strings.Split(line, `href="`) + if len(splitted) == 2 { + splitted2 := strings.Split(splitted[1], `"`) + u := fmt.Sprintf(fmt.Sprintf("https://github.com%s", splitted2[0])) + splitted3 := strings.Split(splitted2[0], "/") + name := splitted3[len(splitted3)-1] + assets = append(assets, &releaseAsset{ + Name: name, + DownloadURL: u, + }) + } + } + } + if err := scanner.Err(); err != nil { + return nil, err + } + return assets, nil +} + +func (c *client) getReleaseAssetsWithAPI(ctx context.Context, opt *AssetOption) ([]*releaseAsset, error) { var ( r *github.RepositoryRelease err error @@ -100,12 +222,45 @@ func (c *client) getReleaseAssets(ctx context.Context, opt *AssetOption) ([]*rel ID: a.GetID(), Name: a.GetName(), ContentType: a.GetContentType(), + DownloadURL: a.GetBrowserDownloadURL(), }) } return assets, nil } func (c *client) downloadAsset(ctx context.Context, a *releaseAsset) ([]byte, error) { + if c.token != "" { + b, err := c.downloadAssetWithAPI(ctx, a) + if err == nil { + return b, nil + } + } + return c.downloadAssetWithoutAPI(ctx, a) +} + +func (c *client) downloadAssetWithoutAPI(ctx context.Context, a *releaseAsset) ([]byte, error) { + if a.DownloadURL == "" { + return nil, errors.New("empty download URL") + } + u := a.DownloadURL + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil) + if err != nil { + return nil, err + } + req.Header.Add("Accept", "application/octet-stream") + resp, err := c.hc.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + b, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + return b, nil +} + +func (c *client) downloadAssetWithAPI(ctx context.Context, a *releaseAsset) ([]byte, error) { u := fmt.Sprintf("%s/repos/%s/%s/releases/assets/%d", c.v3ep, c.owner, c.repo, a.ID) req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil) if err != nil { diff --git a/gh/client_test.go b/gh/client_test.go new file mode 100644 index 0000000..94be457 --- /dev/null +++ b/gh/client_test.go @@ -0,0 +1,32 @@ +package gh + +import ( + "context" + "testing" +) + +func TestGetReleaseAssetsWithoutAPI(t *testing.T) { + tests := []struct { + owner string + repo string + opt *AssetOption + }{ + {"k1LoW", "tbls", nil}, + } + ctx := context.Background() + for _, tt := range tests { + c, err := newClient(ctx, tt.owner, tt.repo) + if err != nil { + t.Error(err) + continue + } + assets, err := c.getReleaseAssetsWithoutAPI(ctx, tt.opt) + if err != nil { + t.Error(err) + continue + } + if len(assets) == 0 { + t.Error("want assets") + } + } +} diff --git a/gh/gh.go b/gh/gh.go index e233026..17842fe 100644 --- a/gh/gh.go +++ b/gh/gh.go @@ -131,7 +131,7 @@ func detectAsset(assets []*releaseAsset, opt *AssetOption) (*releaseAsset, error if om != nil && om.MatchString(a.Name) { return a, nil } - if !contains(supportContentType, a.ContentType) { + if a.ContentType != "" && !contains(supportContentType, a.ContentType) { continue } as := &assetScore{ diff --git a/gh/gh_test.go b/gh/gh_test.go index ff541d9..addd789 100644 --- a/gh/gh_test.go +++ b/gh/gh_test.go @@ -3,11 +3,17 @@ package gh import ( "context" "io/fs" + "os" "testing" "github.com/k1LoW/go-github-client/v50/factory" ) +func TestMain(m *testing.M) { + os.Setenv("GH_CONFIG_DIR", "/tmp") // Disable reading credentials from config + m.Run() +} + func TestGetReleaseAsset(t *testing.T) { tests := []struct { owner string @@ -22,7 +28,6 @@ func TestGetReleaseAsset(t *testing.T) { {"k1LoW", "tbls", &AssetOption{Version: "v1.60.0"}, "tbls", false}, } ctx := context.Background() - t.Setenv("GH_CONFIG_DIR", "/tmp") token, _, _, _ := factory.GetTokenAndEndpoints() for _, tt := range tests { if tt.useToken { From 5fe49be15ccddd318aadca8ecb48d880c76a561e Mon Sep 17 00:00:00 2001 From: k1LoW Date: Sat, 18 Feb 2023 20:38:43 +0900 Subject: [PATCH 6/6] Fix lint warn --- gh/client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gh/client.go b/gh/client.go index deac479..570de78 100644 --- a/gh/client.go +++ b/gh/client.go @@ -184,7 +184,7 @@ func (c *client) getReleaseAssetsViaURL(ctx context.Context, url string) ([]*rel splitted := strings.Split(line, `href="`) if len(splitted) == 2 { splitted2 := strings.Split(splitted[1], `"`) - u := fmt.Sprintf(fmt.Sprintf("https://github.com%s", splitted2[0])) + u := fmt.Sprintf("https://github.com%s", splitted2[0]) splitted3 := strings.Split(splitted2[0], "/") name := splitted3[len(splitted3)-1] assets = append(assets, &releaseAsset{