From 03cf03b2be287f73c6f482bfaa0637d698ce607a Mon Sep 17 00:00:00 2001 From: Nam Vu Date: Sat, 19 Oct 2024 15:57:08 +0200 Subject: [PATCH] feat(manager/uv): add support for git sources (#31928) Co-authored-by: Sebastian Poxhofer --- .../manager/pep621/processors/uv.spec.ts | 101 ++++++++++++++++-- lib/modules/manager/pep621/processors/uv.ts | 62 +++++++++-- lib/modules/manager/pep621/readme.md | 1 + lib/modules/manager/pep621/schema.ts | 36 +++++-- lib/modules/manager/pep621/utils.ts | 1 + 5 files changed, 176 insertions(+), 25 deletions(-) diff --git a/lib/modules/manager/pep621/processors/uv.spec.ts b/lib/modules/manager/pep621/processors/uv.spec.ts index ba53593d931e1b..41597b28503f08 100644 --- a/lib/modules/manager/pep621/processors/uv.spec.ts +++ b/lib/modules/manager/pep621/processors/uv.spec.ts @@ -4,6 +4,11 @@ import { fs, hostRules, mockedFunction } from '../../../../../test/util'; import { GlobalConfig } from '../../../../config/global'; import type { RepoGlobalConfig } from '../../../../config/types'; import { getPkgReleases as _getPkgReleases } from '../../../datasource'; +import { GitRefsDatasource } from '../../../datasource/git-refs'; +import { GitTagsDatasource } from '../../../datasource/git-tags'; +import { GithubTagsDatasource } from '../../../datasource/github-tags'; +import { GitlabTagsDatasource } from '../../../datasource/gitlab-tags'; +import { PypiDatasource } from '../../../datasource/pypi'; import type { UpdateArtifactsConfig } from '../../types'; import { depTypes } from '../utils'; import { UvProcessor } from './uv'; @@ -71,12 +76,10 @@ describe('modules/manager/pep621/processors/uv', () => { dep3: { path: '/local-dep.whl' }, dep4: { url: 'https://example.com' }, dep5: { workspace: true }, - dep6: { workspace: false }, - dep7: {}, }, }, }, - }; + } as const; const dependencies = [ {}, { depName: 'dep1' }, @@ -97,32 +100,102 @@ describe('modules/manager/pep621/processors/uv', () => { }, { depName: 'dep2', - skipReason: 'git-dependency', + depType: depTypes.uvSources, + datasource: GitRefsDatasource.id, + packageName: 'https://github.com/foo/bar', + currentValue: undefined, + skipReason: 'unspecified-version', }, { depName: 'dep3', + depType: depTypes.uvSources, skipReason: 'path-dependency', }, { depName: 'dep4', + depType: depTypes.uvSources, skipReason: 'unsupported-url', }, { depName: 'dep5', + depType: depTypes.uvSources, skipReason: 'inherited-dependency', }, { depName: 'dep6', - skipReason: 'invalid-dependency-specification', }, { depName: 'dep7', - skipReason: 'invalid-dependency-specification', }, ]); }); }); + it('applies git sources', () => { + const pyproject = { + tool: { + uv: { + 'dev-dependencies': ['dep3', 'dep4', 'dep5'], + sources: { + dep1: { git: 'https://github.com/foo/dep1', tag: '0.1.0' }, + dep2: { git: 'https://gitlab.com/foo/dep2', tag: '0.2.0' }, + dep3: { git: 'https://codeberg.org/foo/dep3.git', tag: '0.3.0' }, + dep4: { + git: 'https://github.com/foo/dep4', + rev: '1ca7d263f0f5038b53f74c5a757f18b8106c9390', + }, + dep5: { git: 'https://github.com/foo/dep5', branch: 'master' }, + }, + }, + }, + }; + const dependencies = [{ depName: 'dep1' }, { depName: 'dep2' }]; + + const result = processor.process(pyproject, dependencies); + + expect(result).toEqual([ + { + depName: 'dep1', + depType: depTypes.uvSources, + datasource: GithubTagsDatasource.id, + registryUrls: ['https://github.com'], + packageName: 'foo/dep1', + currentValue: '0.1.0', + }, + { + depName: 'dep2', + depType: depTypes.uvSources, + datasource: GitlabTagsDatasource.id, + registryUrls: ['https://gitlab.com'], + packageName: 'foo/dep2', + currentValue: '0.2.0', + }, + { + depName: 'dep3', + depType: depTypes.uvSources, + datasource: GitTagsDatasource.id, + packageName: 'https://codeberg.org/foo/dep3.git', + currentValue: '0.3.0', + }, + { + depName: 'dep4', + depType: depTypes.uvSources, + datasource: GitRefsDatasource.id, + packageName: 'https://github.com/foo/dep4', + currentDigest: '1ca7d263f0f5038b53f74c5a757f18b8106c9390', + replaceString: '1ca7d263f0f5038b53f74c5a757f18b8106c9390', + }, + { + depName: 'dep5', + depType: depTypes.uvSources, + datasource: GitRefsDatasource.id, + packageName: 'https://github.com/foo/dep5', + currentValue: 'master', + skipReason: 'git-dependency', + }, + ]); + }); + describe('updateArtifacts()', () => { it('returns null if there is no lock file', async () => { fs.getSiblingFileName.mockReturnValueOnce('uv.lock'); @@ -292,13 +365,27 @@ describe('modules/manager/pep621/processors/uv', () => { { packageName: 'dep1', depType: depTypes.dependencies, + datasource: PypiDatasource.id, registryUrls: ['https://foobar.com'], }, { packageName: 'dep2', depType: depTypes.dependencies, + datasource: PypiDatasource.id, registryUrls: ['https://example.com'], }, + { + packageName: 'dep3', + depType: depTypes.dependencies, + datasource: PypiDatasource.id, + registryUrls: ['invalidurl'], + }, + { + packageName: 'dep4', + depType: depTypes.dependencies, + datasource: GithubTagsDatasource.id, + registryUrls: ['https://github.com'], + }, ]; const result = await processor.updateArtifacts( { @@ -320,7 +407,7 @@ describe('modules/manager/pep621/processors/uv', () => { ]); expect(execSnapshots).toMatchObject([ { - cmd: 'uv lock --upgrade-package dep1 --upgrade-package dep2', + cmd: 'uv lock --upgrade-package dep1 --upgrade-package dep2 --upgrade-package dep3 --upgrade-package dep4', options: { env: { UV_EXTRA_INDEX_URL: diff --git a/lib/modules/manager/pep621/processors/uv.ts b/lib/modules/manager/pep621/processors/uv.ts index c328e687fdd3d6..b7aa2058a6949d 100644 --- a/lib/modules/manager/pep621/processors/uv.ts +++ b/lib/modules/manager/pep621/processors/uv.ts @@ -3,12 +3,18 @@ import { quote } from 'shlex'; import { TEMPORARY_ERROR } from '../../../../constants/error-messages'; import { logger } from '../../../../logger'; import type { HostRule } from '../../../../types'; +import { detectPlatform } from '../../../../util/common'; import { exec } from '../../../../util/exec'; import type { ExecOptions, ToolConstraint } from '../../../../util/exec/types'; import { getSiblingFileName, readLocalFile } from '../../../../util/fs'; +import { parseGitUrl } from '../../../../util/git/url'; import { find } from '../../../../util/host-rules'; import { Result } from '../../../../util/result'; import { parseUrl } from '../../../../util/url'; +import { GitRefsDatasource } from '../../../datasource/git-refs'; +import { GitTagsDatasource } from '../../../datasource/git-tags'; +import { GithubTagsDatasource } from '../../../datasource/github-tags'; +import { GitlabTagsDatasource } from '../../../datasource/gitlab-tags'; import { PypiDatasource } from '../../../datasource/pypi'; import type { PackageDependency, @@ -16,7 +22,7 @@ import type { UpdateArtifactsResult, Upgrade, } from '../../types'; -import { type PyProject, UvLockfileSchema } from '../schema'; +import { type PyProject, type UvGitSource, UvLockfileSchema } from '../schema'; import { depTypes, parseDependencyList } from '../utils'; import type { PyProjectProcessor } from './types'; @@ -37,7 +43,7 @@ export class UvProcessor implements PyProjectProcessor { ); // https://docs.astral.sh/uv/concepts/dependencies/#dependency-sources - // Skip sources that are either not yet handled by Renovate (e.g. git), or do not make sense to handle (e.g. path). + // Skip sources that do not make sense to handle (e.g. path). if (uv.sources) { for (const dep of deps) { if (!dep.depName) { @@ -46,16 +52,15 @@ export class UvProcessor implements PyProjectProcessor { const depSource = uv.sources[dep.depName]; if (depSource) { - if (depSource.git) { - dep.skipReason = 'git-dependency'; - } else if (depSource.url) { + dep.depType = depTypes.uvSources; + if ('url' in depSource) { dep.skipReason = 'unsupported-url'; - } else if (depSource.path) { + } else if ('path' in depSource) { dep.skipReason = 'path-dependency'; - } else if (depSource.workspace) { + } else if ('workspace' in depSource) { dep.skipReason = 'inherited-dependency'; } else { - dep.skipReason = 'invalid-dependency-specification'; + applyGitSource(dep, depSource); } } } @@ -175,6 +180,38 @@ export class UvProcessor implements PyProjectProcessor { } } +function applyGitSource(dep: PackageDependency, depSource: UvGitSource): void { + const { git, rev, tag, branch } = depSource; + if (tag) { + const platform = detectPlatform(git); + if (platform === 'github' || platform === 'gitlab') { + dep.datasource = + platform === 'github' + ? GithubTagsDatasource.id + : GitlabTagsDatasource.id; + const { protocol, source, full_name } = parseGitUrl(git); + dep.registryUrls = [`${protocol}://${source}`]; + dep.packageName = full_name; + } else { + dep.datasource = GitTagsDatasource.id; + dep.packageName = git; + } + dep.currentValue = tag; + dep.skipReason = undefined; + } else if (rev) { + dep.datasource = GitRefsDatasource.id; + dep.packageName = git; + dep.currentDigest = rev; + dep.replaceString = rev; + dep.skipReason = undefined; + } else { + dep.datasource = GitRefsDatasource.id; + dep.packageName = git; + dep.currentValue = branch; + dep.skipReason = branch ? 'git-dependency' : 'unspecified-version'; + } +} + function generateCMD(updatedDeps: Upgrade[]): string { const deps: string[] = []; @@ -184,7 +221,8 @@ function generateCMD(updatedDeps: Upgrade[]): string { deps.push(dep.depName!.split('/')[1]); break; } - case depTypes.uvDevDependencies: { + case depTypes.uvDevDependencies: + case depTypes.uvSources: { deps.push(dep.depName!); break; } @@ -205,7 +243,11 @@ function getMatchingHostRule(url: string | undefined): HostRule { } function getUvExtraIndexUrl(deps: Upgrade[]): NodeJS.ProcessEnv { - const registryUrls = new Set(deps.map((dep) => dep.registryUrls).flat()); + const pyPiRegistryUrls = deps + .filter((dep) => dep.datasource === PypiDatasource.id) + .map((dep) => dep.registryUrls) + .flat(); + const registryUrls = new Set(pyPiRegistryUrls); const extraIndexUrls: string[] = []; for (const registryUrl of registryUrls) { diff --git a/lib/modules/manager/pep621/readme.md b/lib/modules/manager/pep621/readme.md index 8d0992cf4dbab3..003c08102e8755 100644 --- a/lib/modules/manager/pep621/readme.md +++ b/lib/modules/manager/pep621/readme.md @@ -13,4 +13,5 @@ Available `depType`s: - `build-system.requires` - `tool.pdm.dev-dependencies` - `tool.uv.dev-dependencies` +- `tool.uv.sources` - `tool.hatch.envs.` diff --git a/lib/modules/manager/pep621/schema.ts b/lib/modules/manager/pep621/schema.ts index 913c43c87c526f..a13f466b21e555 100644 --- a/lib/modules/manager/pep621/schema.ts +++ b/lib/modules/manager/pep621/schema.ts @@ -1,5 +1,5 @@ import { z } from 'zod'; -import { LooseArray, Toml } from '../../../util/schema-utils'; +import { LooseArray, LooseRecord, Toml } from '../../../util/schema-utils'; export type PyProject = z.infer; @@ -35,17 +35,37 @@ const HatchSchema = z.object({ .optional(), }); -// https://docs.astral.sh/uv/concepts/dependencies/#dependency-sources -const UvSource = z.object({ - git: z.string().optional(), - path: z.string().optional(), - url: z.string().optional(), - workspace: z.boolean().optional(), +const UvGitSource = z.object({ + git: z.string(), + rev: z.string().optional(), + tag: z.string().optional(), + branch: z.string().optional(), +}); +export type UvGitSource = z.infer; + +const UvUrlSource = z.object({ + url: z.string(), +}); + +const UvPathSource = z.object({ + path: z.string(), }); +const UvWorkspaceSource = z.object({ + workspace: z.literal(true), +}); + +// https://docs.astral.sh/uv/concepts/dependencies/#dependency-sources +const UvSource = z.union([ + UvGitSource, + UvUrlSource, + UvPathSource, + UvWorkspaceSource, +]); + const UvSchema = z.object({ 'dev-dependencies': DependencyListSchema, - sources: z.record(z.string(), UvSource).optional(), + sources: LooseRecord(z.string(), UvSource).optional(), }); export const PyProjectSchema = z.object({ diff --git a/lib/modules/manager/pep621/utils.ts b/lib/modules/manager/pep621/utils.ts index 702c1afcdee89b..2e7aaa733d8c11 100644 --- a/lib/modules/manager/pep621/utils.ts +++ b/lib/modules/manager/pep621/utils.ts @@ -18,6 +18,7 @@ export const depTypes = { optionalDependencies: 'project.optional-dependencies', pdmDevDependencies: 'tool.pdm.dev-dependencies', uvDevDependencies: 'tool.uv.dev-dependencies', + uvSources: 'tool.uv.sources', buildSystemRequires: 'build-system.requires', };