Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

refactor(go): Change iteration over major versions #22900

Merged
merged 7 commits into from
Jun 21, 2023
Merged

refactor(go): Change iteration over major versions #22900

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
d98db19
refactor(go): Change iteration over major versions
zharinov Jun 20, 2023
d9cb1a1
Fix coverage
zharinov Jun 20, 2023
fddec2c
Merge branch 'main' into refactor/goproxy-query-versions
zharinov Jun 20, 2023
6e62f5d
Merge branch 'main' into refactor/goproxy-query-versions
zharinov Jun 20, 2023
ab89c70
Merge branch 'main' into refactor/goproxy-query-versions
zharinov Jun 21, 2023
5abc7be
Merge branch 'main' into refactor/goproxy-query-versions
zharinov Jun 21, 2023
8cb7ed2
Merge branch 'main' into refactor/goproxy-query-versions
zharinov Jun 21, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
53 changes: 45 additions & 8 deletions lib/modules/datasource/go/releases-goproxy.spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import { codeBlock } from 'common-tags';
import { Fixtures } from '../../../../test/fixtures';
import * as httpMock from '../../../../test/http-mock';
import { GithubReleasesDatasource } from '../github-releases';
Expand Down Expand Up @@ -340,7 +341,10 @@ describe('modules/datasource/go/releases-goproxy', () => {
.get('/@v/list')
.reply(
200,
['v1.0.0 2018-08-13T15:31:12Z', 'v1.0.1', ' \n'].join('\n')
codeBlock`
v1.0.0 2018-08-13T15:31:12Z
v1.0.1
`
)
.get('/@v/v1.0.1.info')
.reply(200, { Version: 'v1.0.1', Time: '2019-10-16T16:15:28Z' })
Expand All @@ -366,13 +370,21 @@ describe('modules/datasource/go/releases-goproxy', () => {
httpMock
.scope(`${baseUrl}/github.com/google/btree`)
.get('/@v/list')
.reply(200, 'v1.0.0\nv1.0.1\n')
.reply(
200,
codeBlock`
v1.0.0
v1.0.1
`
)
.get('/@v/v1.0.0.info')
.replyWithError('unknown')
.get('/@v/v1.0.1.info')
.reply(410)
.get('/v2/@v/list')
.reply(200);
.reply(200)
.get('/v3/@v/list')
.reply(404);

const res = await datasource.getReleases({
packageName: 'github.com/google/btree',
Expand All @@ -395,7 +407,13 @@ describe('modules/datasource/go/releases-goproxy', () => {
httpMock
.scope(`${baseUrl}/github.com/google/btree`)
.get('/@v/list')
.reply(200, 'v1.0.0\nv1.0.1\n')
.reply(
200,
codeBlock`
v1.0.0
v1.0.1
`
)
.get('/@v/v1.0.0.info')
.reply(200, { Version: 'v1.0.0', Time: '2018-08-13T15:31:12Z' })
.get('/@v/v1.0.1.info')
Expand Down Expand Up @@ -436,7 +454,13 @@ describe('modules/datasource/go/releases-goproxy', () => {
httpMock
.scope(`${baseUrl}/github.com/google/btree`)
.get('/@v/list')
.reply(200, 'v1.0.0\nv1.0.1\n')
.reply(
200,
codeBlock`
v1.0.0
v1.0.1
`
)
.get('/@v/v1.0.0.info')
.reply(200, { Version: 'v1.0.0', Time: '2018-08-13T15:31:12Z' })
.get('/@v/v1.0.1.info')
Expand Down Expand Up @@ -580,13 +604,24 @@ describe('modules/datasource/go/releases-goproxy', () => {
httpMock
.scope(`${baseUrl}/github.com/google/btree`)
.get('/@v/list')
.reply(200, 'v1.0.0\nv1.0.1\n')
.reply(
200,
codeBlock`
v1.0.0
v1.0.1
`
)
.get('/@v/v1.0.0.info')
.reply(200, { Version: 'v1.0.0', Time: '2018-08-13T15:31:12Z' })
.get('/@v/v1.0.1.info')
.reply(200, { Version: 'v1.0.1', Time: '2019-10-16T16:15:28Z' })
.get('/v2/@v/list')
.reply(200, 'v2.0.0\n')
.reply(
200,
codeBlock`
v2.0.0
`
)
.get('/v2/@v/v2.0.0.info')
.reply(200, { Version: 'v2.0.0', Time: '2020-10-16T16:15:28Z' })
.get('/v3/@v/list')
Expand Down Expand Up @@ -679,7 +714,9 @@ describe('modules/datasource/go/releases-goproxy', () => {
httpMock
.scope(`${baseUrl}/github.com/google/btree`)
.get('/@v/list')
.reply(200);
.reply(200)
.get('/v2/@v/list')
.reply(404);

const res = await datasource.getReleases({
packageName: 'github.com/google/btree',
Expand Down
97 changes: 49 additions & 48 deletions lib/modules/datasource/go/releases-goproxy.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import { DateTime } from 'luxon';
import moo from 'moo';
import { logger } from '../../../logger';
import { cache } from '../../../util/cache/package/decorator';
import { HttpError } from '../../../util/http';
import * as p from '../../../util/promises';
import { newlineRegex, regEx } from '../../../util/regex';
import { Datasource } from '../datasource';
Expand All @@ -14,7 +15,7 @@ import type { GoproxyItem, VersionInfo } from './types';

const parsedGoproxy: Record<string, GoproxyItem[]> = {};

const modRegex = regEx(`^(?<baseMod>.*?)(?:[./]v(?<majorVersion>\\d+))?$`);
const modRegex = regEx(/^(?<baseMod>.*?)(?:[./]v(?<majorVersion>\d+))?$/);
zharinov marked this conversation as resolved.
Show resolved Hide resolved

export class GoProxyDatasource extends Datasource {
static readonly id = 'go-proxy';
Expand Down Expand Up @@ -47,12 +48,6 @@ export class GoProxyDatasource extends Datasource {
return result;
}

const isGopkgin = packageName.startsWith('gopkg.in/');
const majorSuffixSeparator = isGopkgin ? '.' : '/';
const modParts = packageName.match(modRegex);
const baseMod = modParts?.groups?.baseMod ?? packageName;
const currentMajor = parseInt(modParts?.groups?.majorVersion ?? '0');

for (const { url, fallback } of proxyList) {
try {
if (url === 'off') {
Expand All @@ -62,43 +57,20 @@ export class GoProxyDatasource extends Datasource {
break;
}

const releases: Release[] = [];
for (let i = 0; ; i++) {
try {
const major = currentMajor + i;
let mod: string;
if (major < 2 && !isGopkgin) {
mod = baseMod;
i++; // v0 and v1 are the same module
} else {
mod = `${baseMod}${majorSuffixSeparator}v${major}`;
}
const result = await this.getVersionsWithInfo(url, mod);
if (result.length === 0) {
break;
}
releases.push(...result);
} catch (err) {
const statusCode = err?.response?.statusCode;
if (i > 0 && statusCode === 404) {
break;
}
throw err;
}
}

if (releases.length) {
const res = await this.getVersionsWithInfo(url, packageName);
if (res.releases.length) {
result = res;
try {
const datasource = await BaseGoDatasource.getDatasource(
packageName
);
const sourceUrl = getSourceUrl(datasource) ?? null;
result = { releases, sourceUrl };
const sourceUrl = getSourceUrl(datasource);
if (sourceUrl) {
result.sourceUrl = sourceUrl;
}
} catch (err) {
logger.trace({ err }, `Can't get datasource for ${packageName}`);
result = { releases };
}

break;
}
} catch (err) {
Expand Down Expand Up @@ -263,23 +235,52 @@ export class GoProxyDatasource extends Datasource {
async getVersionsWithInfo(
baseUrl: string,
packageName: string
): Promise<Release[]> {
const releasesIndex = await this.listVersions(baseUrl, packageName);
const releases = await p.map(releasesIndex, async (versionInfo) => {
const { version, releaseTimestamp } = versionInfo;
): Promise<ReleaseResult> {
const isGopkgin = packageName.startsWith('gopkg.in/');
const majorSuffixSeparator = isGopkgin ? '.' : '/';
const modParts = packageName.match(modRegex)?.groups;
const baseMod = modParts?.baseMod ?? /* istanbul ignore next */ packageName;
const packageMajor = parseInt(modParts?.majorVersion ?? '0');

if (releaseTimestamp) {
return { version, releaseTimestamp };
const result: ReleaseResult = { releases: [] };
for (let major = packageMajor; ; major += 1) {
let pkg = `${baseMod}${majorSuffixSeparator}v${major}`;
if (!isGopkgin && major < 2) {
pkg = baseMod;
major += 1; // v0 and v1 are the same module
}

try {
return await this.versionInfo(baseUrl, packageName, version);
const res = await this.listVersions(baseUrl, pkg);
const releases = await p.map(res, async (versionInfo) => {
const { version, releaseTimestamp } = versionInfo;

if (releaseTimestamp) {
return { version, releaseTimestamp };
}

try {
return await this.versionInfo(baseUrl, pkg, version);
} catch (err) {
logger.trace({ err }, `Can't obtain data from ${baseUrl}`);
return { version };
}
});
result.releases.push(...releases);
} catch (err) {
logger.trace({ err }, `Can't obtain data from ${baseUrl}`);
return { version };
if (
err instanceof HttpError &&
err.response?.statusCode === 404 &&
major !== packageMajor
) {
break;
}

throw err;
}
});
return releases;
}

return result;
}

static getCacheKey({ packageName }: GetReleasesConfig): string {
Expand Down