feat(datasource/pypi): always fallback to try simple endpoints

lib/modules/datasource/pypi/__snapshots__/index.spec.ts.snap

@@ -1,47 +1,5 @@
 // Jest Snapshot v1, https://goo.gl/fbAQLP
 
-exports[`modules/datasource/pypi/index getReleases fall back from json and process data from simple endpoint 1`] = `
-{
-  "registryUrl": "https://custom.pypi.net/foo",
-  "releases": [
-    {
-      "version": "0.1.2",
-    },
-    {
-      "version": "0.1.3",
-    },
-    {
-      "version": "0.1.4",
-    },
-    {
-      "version": "0.2.0",
-    },
-    {
-      "version": "0.2.1",
-    },
-    {
-      "version": "0.2.2",
-    },
-    {
-      "version": "0.3.0",
-    },
-    {
-      "version": "0.4.0",
-    },
-    {
-      "version": "0.4.1",
-    },
-    {
-      "version": "0.4.2",
-    },
-    {
-      "isDeprecated": true,
-      "version": "0.5.0",
-    },
-  ],
-}
-`;
-
 exports[`modules/datasource/pypi/index getReleases parses data-requires-python and respects constraints from simple endpoint 1`] = `
 {
   "registryUrl": "https://some.registry.org/simple",

lib/modules/datasource/pypi/index.spec.ts

@@ -160,6 +160,10 @@ describe('modules/datasource/pypi/index', () => {
         .scope('https://custom.pypi.net/foo')
         .get('/azure-cli-monitor/json')
         .replyWithError('error');
+      httpMock
+        .scope('https://custom.pypi.net/foo')
+        .get('/azure-cli-monitor/')
+        .replyWithError('error');
       httpMock
         .scope('https://second-index/foo')
         .get('/azure-cli-monitor/json')
@@ -308,6 +312,11 @@ describe('modules/datasource/pypi/index', () => {
         .get('/not-normalized-package/json')
         .reply(200, htmlResponse);
 
+      httpMock
+        .scope(baseUrl)
+        .get('/not-normalized-package/')
+        .reply(200, htmlResponse);
+
       await getPkgReleases({
         datasource,
         registryUrls: [baseUrl],
@@ -716,25 +725,28 @@ describe('modules/datasource/pypi/index', () => {
       ).toBeNull();
     });
 
-    it('fall back from json and process data from simple endpoint', async () => {
-      httpMock
-        .scope('https://custom.pypi.net/foo')
-        .get('/dj-database-url/json')
-        .reply(404);
-      httpMock
-        .scope('https://custom.pypi.net/foo')
-        .get('/dj-database-url/')
-        .reply(200, htmlResponse);
-      const config = {
-        registryUrls: ['https://custom.pypi.net/foo'],
-      };
-      const result = await getPkgReleases({
-        datasource,
-        ...config,
-        packageName: 'dj-database-url',
-      });
-      expect(result).toMatchSnapshot();
-    });
+    it.each([404, 403])(
+      'fall back from json and process data from simple endpoint',
+      async (code: number) => {
+        httpMock
+          .scope('https://custom.pypi.net/foo')
+          .get('/dj-database-url/json')
+          .reply(code);
+        httpMock
+          .scope('https://custom.pypi.net/foo')
+          .get('/dj-database-url/')
+          .reply(200, htmlResponse);
+        const config = {
+          registryUrls: ['https://custom.pypi.net/foo'],
+        };
+        const result = await getPkgReleases({
+          datasource,
+          ...config,
+          packageName: 'dj-database-url',
+        });
+        expect(result).not.toBeNull();
+      },
+    );
 
     it('parses data-requires-python and respects constraints from simple endpoint', async () => {
       httpMock

lib/modules/datasource/pypi/index.ts

@@ -66,13 +66,9 @@ export class PypiDatasource extends Datasource {
         // we need to resolve early here so we can catch any 404s and fallback to a simple lookup
         dependency = await this.getDependency(normalizedLookupName, hostUrl);
       } catch (err) {
-        if (err.statusCode !== 404) {
-          throw err;
-        }
-
         // error contacting json-style api -- attempt to fallback to a simple-style api
         logger.trace(
-          { packageName, hostUrl },
+          { packageName, hostUrl, err },
           'Looking up pypi simple dependency via fallback',
         );
         dependency = await this.getSimpleDependency(

lib/modules/datasource/pypi/readme.md

@@ -0,0 +1,6 @@
+This datasource uses the following logic to determine lookup URLs:
+
+- If the normalized registryUrl ends in `/simple/` or `/+simple/` then only the simple API will be tried
+- Otherwise, the JSON API will be tried first
+- If the JSON API returns a result, it will be used
+- If the JSON API throws an error (e.g. 403, 404) then the simple API will be tried