From ac3ccc82a9ef5fef3c57458362f1393c58611cd3 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Thu, 22 Sep 2022 14:30:21 +0000 Subject: [PATCH 1/3] Make NuGet service index public. --- routers/api/packages/api.go | 52 +++++++------ tests/integration/api_packages_nuget_test.go | 80 ++++++++++++-------- 2 files changed, 77 insertions(+), 55 deletions(-) diff --git a/routers/api/packages/api.go b/routers/api/packages/api.go index 93f9afca7b283..bf710ffd0459e 100644 --- a/routers/api/packages/api.go +++ b/routers/api/packages/api.go @@ -69,7 +69,7 @@ func Routes(ctx gocontext.Context) *web.Route { r.Get("/p2/{vendorname}/{projectname}.json", composer.PackageMetadata) r.Get("/files/{package}/{version}/{filename}", composer.DownloadPackageFile) r.Put("", reqPackageAccess(perm.AccessModeWrite), composer.UploadPackage) - }) + }, reqPackageAccess(perm.AccessModeRead)) r.Group("/conan", func() { r.Group("/v1", func() { r.Get("/ping", conan.Ping) @@ -157,7 +157,7 @@ func Routes(ctx gocontext.Context) *web.Route { }, conan.ExtractPathParameters) }) }) - }) + }, reqPackageAccess(perm.AccessModeRead)) r.Group("/generic", func() { r.Group("/{packagename}/{packageversion}", func() { r.Delete("", reqPackageAccess(perm.AccessModeWrite), generic.DeletePackage) @@ -169,33 +169,35 @@ func Routes(ctx gocontext.Context) *web.Route { }, reqPackageAccess(perm.AccessModeWrite)) }) }) - }) + }, reqPackageAccess(perm.AccessModeRead)) r.Group("/helm", func() { r.Get("/index.yaml", helm.Index) r.Get("/{filename}", helm.DownloadPackageFile) r.Post("/api/charts", reqPackageAccess(perm.AccessModeWrite), helm.UploadPackage) - }) + }, reqPackageAccess(perm.AccessModeRead)) r.Group("/maven", func() { r.Put("/*", reqPackageAccess(perm.AccessModeWrite), maven.UploadPackageFile) r.Get("/*", maven.DownloadPackageFile) - }) + }, reqPackageAccess(perm.AccessModeRead)) r.Group("/nuget", func() { r.Get("/index.json", nuget.ServiceIndex) - r.Get("/query", nuget.SearchService) - r.Group("/registration/{id}", func() { - r.Get("/index.json", nuget.RegistrationIndex) - r.Get("/{version}", nuget.RegistrationLeaf) - }) - r.Group("/package/{id}", func() { - r.Get("/index.json", nuget.EnumeratePackageVersions) - r.Get("/{version}/{filename}", nuget.DownloadPackageFile) - }) r.Group("", func() { - r.Put("/", nuget.UploadPackage) - r.Put("/symbolpackage", nuget.UploadSymbolPackage) - r.Delete("/{id}/{version}", nuget.DeletePackage) - }, reqPackageAccess(perm.AccessModeWrite)) - r.Get("/symbols/{filename}/{guid:[0-9a-f]{32}}FFFFFFFF/{filename2}", nuget.DownloadSymbolFile) + r.Get("/query", nuget.SearchService) + r.Group("/registration/{id}", func() { + r.Get("/index.json", nuget.RegistrationIndex) + r.Get("/{version}", nuget.RegistrationLeaf) + }) + r.Group("/package/{id}", func() { + r.Get("/index.json", nuget.EnumeratePackageVersions) + r.Get("/{version}/{filename}", nuget.DownloadPackageFile) + }) + r.Group("", func() { + r.Put("/", nuget.UploadPackage) + r.Put("/symbolpackage", nuget.UploadSymbolPackage) + r.Delete("/{id}/{version}", nuget.DeletePackage) + }, reqPackageAccess(perm.AccessModeWrite)) + r.Get("/symbols/{filename}/{guid:[0-9a-f]{32}}FFFFFFFF/{filename2}", nuget.DownloadSymbolFile) + }, reqPackageAccess(perm.AccessModeRead)) }) r.Group("/npm", func() { r.Group("/@{scope}/{id}", func() { @@ -236,7 +238,7 @@ func Routes(ctx gocontext.Context) *web.Route { r.Delete("", npm.DeletePackageTag) }, reqPackageAccess(perm.AccessModeWrite)) }) - }) + }, reqPackageAccess(perm.AccessModeRead)) r.Group("/pub", func() { r.Group("/api/packages", func() { r.Group("/versions/new", func() { @@ -250,12 +252,12 @@ func Routes(ctx gocontext.Context) *web.Route { r.Get("/{version}", pub.PackageVersionMetadata) }) }) - }) + }, reqPackageAccess(perm.AccessModeRead)) r.Group("/pypi", func() { r.Post("/", reqPackageAccess(perm.AccessModeWrite), pypi.UploadPackageFile) r.Get("/files/{id}/{version}/{filename}", pypi.DownloadPackageFile) r.Get("/simple/{id}", pypi.PackageMetadata) - }) + }, reqPackageAccess(perm.AccessModeRead)) r.Group("/rubygems", func() { r.Get("/specs.4.8.gz", rubygems.EnumeratePackages) r.Get("/latest_specs.4.8.gz", rubygems.EnumeratePackagesLatest) @@ -266,7 +268,7 @@ func Routes(ctx gocontext.Context) *web.Route { r.Post("/", rubygems.UploadPackageFile) r.Delete("/yank", rubygems.DeletePackage) }, reqPackageAccess(perm.AccessModeWrite)) - }) + }, reqPackageAccess(perm.AccessModeRead)) r.Group("/vagrant", func() { r.Group("/authenticate", func() { r.Get("", vagrant.CheckAuthenticate) @@ -279,8 +281,8 @@ func Routes(ctx gocontext.Context) *web.Route { r.Put("", reqPackageAccess(perm.AccessModeWrite), vagrant.UploadPackageFile) }) }) - }) - }, context_service.UserAssignmentWeb(), context.PackageAssignment(), reqPackageAccess(perm.AccessModeRead)) + }, reqPackageAccess(perm.AccessModeRead)) + }, context_service.UserAssignmentWeb(), context.PackageAssignment()) return r } diff --git a/tests/integration/api_packages_nuget_test.go b/tests/integration/api_packages_nuget_test.go index 87275feb3ef85..9d53311d3564a 100644 --- a/tests/integration/api_packages_nuget_test.go +++ b/tests/integration/api_packages_nuget_test.go @@ -19,6 +19,7 @@ import ( user_model "code.gitea.io/gitea/models/user" nuget_module "code.gitea.io/gitea/modules/packages/nuget" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/routers/api/packages/nuget" "code.gitea.io/gitea/tests" @@ -66,39 +67,58 @@ func TestPackageNuGet(t *testing.T) { t.Run("ServiceIndex", func(t *testing.T) { defer tests.PrintCurrentTest(t)() - req := NewRequest(t, "GET", fmt.Sprintf("%s/index.json", url)) - req = AddBasicAuthHeader(req, user.Name) - MakeRequest(t, req, http.StatusOK) + privateUser := unittest.AssertExistsAndLoadBean(t, &user_model.User{Visibility: structs.VisibleTypePrivate}) - req = NewRequest(t, "GET", fmt.Sprintf("%s/index.json", url)) - req = addNuGetAPIKeyHeader(req, token) - resp := MakeRequest(t, req, http.StatusOK) + cases := []struct { + Owner string + UseBasicAuth bool + UseTokenAuth bool + }{ + {privateUser.Name, false, false}, + {privateUser.Name, true, false}, + {privateUser.Name, false, true}, + {user.Name, false, false}, + {user.Name, true, false}, + {user.Name, false, true}, + } - var result nuget.ServiceIndexResponse - DecodeJSON(t, resp, &result) + for _, c := range cases { + url := fmt.Sprintf("/api/packages/%s/nuget", c.Owner) - assert.Equal(t, "3.0.0", result.Version) - assert.NotEmpty(t, result.Resources) - - root := setting.AppURL + url[1:] - for _, r := range result.Resources { - switch r.Type { - case "SearchQueryService": - fallthrough - case "SearchQueryService/3.0.0-beta": - fallthrough - case "SearchQueryService/3.0.0-rc": - assert.Equal(t, root+"/query", r.ID) - case "RegistrationsBaseUrl": - fallthrough - case "RegistrationsBaseUrl/3.0.0-beta": - fallthrough - case "RegistrationsBaseUrl/3.0.0-rc": - assert.Equal(t, root+"/registration", r.ID) - case "PackageBaseAddress/3.0.0": - assert.Equal(t, root+"/package", r.ID) - case "PackagePublish/2.0.0": - assert.Equal(t, root, r.ID) + req := NewRequest(t, "GET", fmt.Sprintf("%s/index.json", url)) + if c.UseBasicAuth { + req = AddBasicAuthHeader(req, user.Name) + } else if c.UseTokenAuth { + req = addNuGetAPIKeyHeader(req, token) + } + resp := MakeRequest(t, req, http.StatusOK) + + var result nuget.ServiceIndexResponse + DecodeJSON(t, resp, &result) + + assert.Equal(t, "3.0.0", result.Version) + assert.NotEmpty(t, result.Resources) + + root := setting.AppURL + url[1:] + for _, r := range result.Resources { + switch r.Type { + case "SearchQueryService": + fallthrough + case "SearchQueryService/3.0.0-beta": + fallthrough + case "SearchQueryService/3.0.0-rc": + assert.Equal(t, root+"/query", r.ID) + case "RegistrationsBaseUrl": + fallthrough + case "RegistrationsBaseUrl/3.0.0-beta": + fallthrough + case "RegistrationsBaseUrl/3.0.0-rc": + assert.Equal(t, root+"/registration", r.ID) + case "PackageBaseAddress/3.0.0": + assert.Equal(t, root+"/package", r.ID) + case "PackagePublish/2.0.0": + assert.Equal(t, root, r.ID) + } } } }) From f5693d0d982880738a89d1c2e63540555685c1d4 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Sat, 24 Sep 2022 11:19:08 +0000 Subject: [PATCH 2/3] Add comment. --- routers/api/packages/api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routers/api/packages/api.go b/routers/api/packages/api.go index bf710ffd0459e..c6e91b4dc4d5b 100644 --- a/routers/api/packages/api.go +++ b/routers/api/packages/api.go @@ -180,7 +180,7 @@ func Routes(ctx gocontext.Context) *web.Route { r.Get("/*", maven.DownloadPackageFile) }, reqPackageAccess(perm.AccessModeRead)) r.Group("/nuget", func() { - r.Get("/index.json", nuget.ServiceIndex) + r.Get("/index.json", nuget.ServiceIndex) // Needs to be unauthenticated for the NuGet client. r.Group("", func() { r.Get("/query", nuget.SearchService) r.Group("/registration/{id}", func() { From dfd45fb37840b8a6cf9642ed86c7925fe4c96e62 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Sat, 24 Sep 2022 11:29:15 +0000 Subject: [PATCH 3/3] Merge branch 'main' of https://github.com/go-gitea/gitea into fix-20717-2 --- docs/content/doc/packages/npm.en-us.md | 5 ++++ modules/packages/npm/creator.go | 28 +++++++++++++++++ routers/api/packages/api.go | 3 ++ routers/api/packages/npm/api.go | 35 ++++++++++++++++++++++ routers/api/packages/npm/npm.go | 32 ++++++++++++++++++++ tests/integration/api_packages_npm_test.go | 31 +++++++++++++++++++ 6 files changed, 134 insertions(+) diff --git a/docs/content/doc/packages/npm.en-us.md b/docs/content/doc/packages/npm.en-us.md index 122f306ee5e3a..decb3f743c419 100644 --- a/docs/content/doc/packages/npm.en-us.md +++ b/docs/content/doc/packages/npm.en-us.md @@ -127,6 +127,10 @@ npm dist-tag add test_package@1.0.2 release The tag name must not be a valid version. All tag names which are parsable as a version are rejected. +## Search packages + +The registry supports [searching](https://docs.npmjs.com/cli/v7/commands/npm-search/) but does not support special search qualifiers like `author:gitea`. + ## Supported commands ``` @@ -136,4 +140,5 @@ npm publish npm unpublish npm dist-tag npm view +npm search ``` diff --git a/modules/packages/npm/creator.go b/modules/packages/npm/creator.go index 88ce55ecdbef8..2ed4d262481d1 100644 --- a/modules/packages/npm/creator.go +++ b/modules/packages/npm/creator.go @@ -96,6 +96,34 @@ type PackageDistribution struct { NpmSignature string `json:"npm-signature,omitempty"` } +type PackageSearch struct { + Objects []*PackageSearchObject `json:"objects"` + Total int64 `json:"total"` +} + +type PackageSearchObject struct { + Package *PackageSearchPackage `json:"package"` +} + +type PackageSearchPackage struct { + Scope string `json:"scope"` + Name string `json:"name"` + Version string `json:"version"` + Date time.Time `json:"date"` + Description string `json:"description"` + Author User `json:"author"` + Publisher User `json:"publisher"` + Maintainers []User `json:"maintainers"` + Keywords []string `json:"keywords,omitempty"` + Links *PackageSearchPackageLinks `json:"links"` +} + +type PackageSearchPackageLinks struct { + Registry string `json:"npm"` + Homepage string `json:"homepage,omitempty"` + Repository string `json:"repository,omitempty"` +} + // User https://github.com/npm/registry/blob/master/docs/REGISTRY-API.md#package type User struct { Username string `json:"username,omitempty"` diff --git a/routers/api/packages/api.go b/routers/api/packages/api.go index c6e91b4dc4d5b..3354fe12d43ca 100644 --- a/routers/api/packages/api.go +++ b/routers/api/packages/api.go @@ -238,6 +238,9 @@ func Routes(ctx gocontext.Context) *web.Route { r.Delete("", npm.DeletePackageTag) }, reqPackageAccess(perm.AccessModeWrite)) }) + r.Group("/-/v1/search", func() { + r.Get("", npm.PackageSearch) + }) }, reqPackageAccess(perm.AccessModeRead)) r.Group("/pub", func() { r.Group("/api/packages", func() { diff --git a/routers/api/packages/npm/api.go b/routers/api/packages/npm/api.go index 763c595152ec1..d1027763a836a 100644 --- a/routers/api/packages/npm/api.go +++ b/routers/api/packages/npm/api.go @@ -74,3 +74,38 @@ func createPackageMetadataVersion(registryURL string, pd *packages_model.Package }, } } + +func createPackageSearchResponse(pds []*packages_model.PackageDescriptor, total int64) *npm_module.PackageSearch { + objects := make([]*npm_module.PackageSearchObject, 0, len(pds)) + for _, pd := range pds { + metadata := pd.Metadata.(*npm_module.Metadata) + + scope := metadata.Scope + if scope == "" { + scope = "unscoped" + } + + objects = append(objects, &npm_module.PackageSearchObject{ + Package: &npm_module.PackageSearchPackage{ + Scope: scope, + Name: metadata.Name, + Version: pd.Version.Version, + Date: pd.Version.CreatedUnix.AsLocalTime(), + Description: metadata.Description, + Author: npm_module.User{Name: metadata.Author}, + Publisher: npm_module.User{Name: pd.Owner.Name}, + Maintainers: []npm_module.User{}, // npm cli needs this field + Keywords: metadata.Keywords, + Links: &npm_module.PackageSearchPackageLinks{ + Registry: pd.FullWebLink(), + Homepage: metadata.ProjectURL, + }, + }, + }) + } + + return &npm_module.PackageSearch{ + Objects: objects, + Total: total, + } +} diff --git a/routers/api/packages/npm/npm.go b/routers/api/packages/npm/npm.go index 66b999d47ee50..2989ce6e7f6bd 100644 --- a/routers/api/packages/npm/npm.go +++ b/routers/api/packages/npm/npm.go @@ -350,3 +350,35 @@ func setPackageTag(tag string, pv *packages_model.PackageVersion, deleteOnly boo return committer.Commit() } + +func PackageSearch(ctx *context.Context) { + pvs, total, err := packages_model.SearchLatestVersions(ctx, &packages_model.PackageSearchOptions{ + OwnerID: ctx.Package.Owner.ID, + Type: packages_model.TypeNpm, + Name: packages_model.SearchValue{ + ExactMatch: false, + Value: ctx.FormTrim("text"), + }, + Paginator: db.NewAbsoluteListOptions( + ctx.FormInt("from"), + ctx.FormInt("size"), + ), + }) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + pds, err := packages_model.GetPackageDescriptors(ctx, pvs) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + resp := createPackageSearchResponse( + pds, + total, + ) + + ctx.JSON(http.StatusOK, resp) +} diff --git a/tests/integration/api_packages_npm_test.go b/tests/integration/api_packages_npm_test.go index fe6cea1cb68da..14ed681be3a7a 100644 --- a/tests/integration/api_packages_npm_test.go +++ b/tests/integration/api_packages_npm_test.go @@ -224,6 +224,37 @@ func TestPackageNpm(t *testing.T) { test(t, http.StatusOK, packageTag2) }) + t.Run("Search", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + url := fmt.Sprintf("/api/packages/%s/npm/-/v1/search", user.Name) + + cases := []struct { + Query string + Skip int + Take int + ExpectedTotal int64 + ExpectedResults int + }{ + {"", 0, 0, 1, 1}, + {"", 0, 10, 1, 1}, + {"gitea", 0, 10, 0, 0}, + {"test", 0, 10, 1, 1}, + {"test", 1, 10, 1, 0}, + } + + for i, c := range cases { + req := NewRequest(t, "GET", fmt.Sprintf("%s?text=%s&from=%d&size=%d", url, c.Query, c.Skip, c.Take)) + resp := MakeRequest(t, req, http.StatusOK) + + var result npm.PackageSearch + DecodeJSON(t, resp, &result) + + assert.Equal(t, c.ExpectedTotal, result.Total, "case %d: unexpected total hits", i) + assert.Len(t, result.Objects, c.ExpectedResults, "case %d: unexpected result count", i) + } + }) + t.Run("Delete", func(t *testing.T) { defer tests.PrintCurrentTest(t)()