From cf2ed8f799b0339d6aec10d9e8ebf9492d48889c Mon Sep 17 00:00:00 2001 From: Rhys Arkins Date: Sat, 19 Oct 2024 08:30:23 +0200 Subject: [PATCH 1/3] feat(datasource/pypi): always fallback to try simple endpoints Closes #26483 Closes #26383 --- .../pypi/__snapshots__/index.spec.ts.snap | 42 ---------------- lib/modules/datasource/pypi/index.spec.ts | 50 ++++++++++++------- lib/modules/datasource/pypi/index.ts | 4 -- 3 files changed, 31 insertions(+), 65 deletions(-) diff --git a/lib/modules/datasource/pypi/__snapshots__/index.spec.ts.snap b/lib/modules/datasource/pypi/__snapshots__/index.spec.ts.snap index 1af50f1d6d055b..1704b163844bff 100644 --- a/lib/modules/datasource/pypi/__snapshots__/index.spec.ts.snap +++ b/lib/modules/datasource/pypi/__snapshots__/index.spec.ts.snap @@ -1,47 +1,5 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP -exports[`modules/datasource/pypi/index getReleases fall back from json and process data from simple endpoint 1`] = ` -{ - "registryUrl": "https://custom.pypi.net/foo", - "releases": [ - { - "version": "0.1.2", - }, - { - "version": "0.1.3", - }, - { - "version": "0.1.4", - }, - { - "version": "0.2.0", - }, - { - "version": "0.2.1", - }, - { - "version": "0.2.2", - }, - { - "version": "0.3.0", - }, - { - "version": "0.4.0", - }, - { - "version": "0.4.1", - }, - { - "version": "0.4.2", - }, - { - "isDeprecated": true, - "version": "0.5.0", - }, - ], -} -`; - exports[`modules/datasource/pypi/index getReleases parses data-requires-python and respects constraints from simple endpoint 1`] = ` { "registryUrl": "https://some.registry.org/simple", diff --git a/lib/modules/datasource/pypi/index.spec.ts b/lib/modules/datasource/pypi/index.spec.ts index 678101cfa03c30..295a78c241b4ae 100644 --- a/lib/modules/datasource/pypi/index.spec.ts +++ b/lib/modules/datasource/pypi/index.spec.ts @@ -160,6 +160,10 @@ describe('modules/datasource/pypi/index', () => { .scope('https://custom.pypi.net/foo') .get('/azure-cli-monitor/json') .replyWithError('error'); + httpMock + .scope('https://custom.pypi.net/foo') + .get('/azure-cli-monitor/') + .replyWithError('error'); httpMock .scope('https://second-index/foo') .get('/azure-cli-monitor/json') @@ -308,6 +312,11 @@ describe('modules/datasource/pypi/index', () => { .get('/not-normalized-package/json') .reply(200, htmlResponse); + httpMock + .scope(baseUrl) + .get('/not-normalized-package/') + .reply(200, htmlResponse); + await getPkgReleases({ datasource, registryUrls: [baseUrl], @@ -716,25 +725,28 @@ describe('modules/datasource/pypi/index', () => { ).toBeNull(); }); - it('fall back from json and process data from simple endpoint', async () => { - httpMock - .scope('https://custom.pypi.net/foo') - .get('/dj-database-url/json') - .reply(404); - httpMock - .scope('https://custom.pypi.net/foo') - .get('/dj-database-url/') - .reply(200, htmlResponse); - const config = { - registryUrls: ['https://custom.pypi.net/foo'], - }; - const result = await getPkgReleases({ - datasource, - ...config, - packageName: 'dj-database-url', - }); - expect(result).toMatchSnapshot(); - }); + it.each([404, 403])( + 'fall back from json and process data from simple endpoint', + async (code: number) => { + httpMock + .scope('https://custom.pypi.net/foo') + .get('/dj-database-url/json') + .reply(code); + httpMock + .scope('https://custom.pypi.net/foo') + .get('/dj-database-url/') + .reply(200, htmlResponse); + const config = { + registryUrls: ['https://custom.pypi.net/foo'], + }; + const result = await getPkgReleases({ + datasource, + ...config, + packageName: 'dj-database-url', + }); + expect(result).not.toBeNull(); + }, + ); it('parses data-requires-python and respects constraints from simple endpoint', async () => { httpMock diff --git a/lib/modules/datasource/pypi/index.ts b/lib/modules/datasource/pypi/index.ts index dc8a5437dc259e..7bc7f46983a37c 100644 --- a/lib/modules/datasource/pypi/index.ts +++ b/lib/modules/datasource/pypi/index.ts @@ -66,10 +66,6 @@ export class PypiDatasource extends Datasource { // we need to resolve early here so we can catch any 404s and fallback to a simple lookup dependency = await this.getDependency(normalizedLookupName, hostUrl); } catch (err) { - if (err.statusCode !== 404) { - throw err; - } - // error contacting json-style api -- attempt to fallback to a simple-style api logger.trace( { packageName, hostUrl }, From 9649254b2b18790cab9d67e03f955b981f18740c Mon Sep 17 00:00:00 2001 From: Rhys Arkins Date: Sat, 19 Oct 2024 08:35:34 +0200 Subject: [PATCH 2/3] docs --- lib/modules/datasource/pypi/readme.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 lib/modules/datasource/pypi/readme.md diff --git a/lib/modules/datasource/pypi/readme.md b/lib/modules/datasource/pypi/readme.md new file mode 100644 index 00000000000000..ae32c3ca683ef4 --- /dev/null +++ b/lib/modules/datasource/pypi/readme.md @@ -0,0 +1,6 @@ +This datasource uses the following logic to determine lookup URLs: + +- If the normalized registryUrl ends in `/simple/` or `/+simple/` then only the simple API will be tried +- Otherwise, the JSON API will be tried first +- If the JSON API returns a result, it will be used +- If the JSON API throws an error (e.g. 403, 404) then the simple API will be tried From c3672221fe76400672651dbaa652cbab566c0081 Mon Sep 17 00:00:00 2001 From: Rhys Arkins Date: Wed, 30 Oct 2024 09:21:08 +0100 Subject: [PATCH 3/3] add intentional fixture --- lib/modules/datasource/pypi/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/modules/datasource/pypi/index.ts b/lib/modules/datasource/pypi/index.ts index 7bc7f46983a37c..3e8812116e0555 100644 --- a/lib/modules/datasource/pypi/index.ts +++ b/lib/modules/datasource/pypi/index.ts @@ -68,7 +68,7 @@ export class PypiDatasource extends Datasource { } catch (err) { // error contacting json-style api -- attempt to fallback to a simple-style api logger.trace( - { packageName, hostUrl }, + { packageName, hostUrl, err }, 'Looking up pypi simple dependency via fallback', ); dependency = await this.getSimpleDependency(