diff --git a/chart/templates/server-deployment.yaml b/chart/templates/server-deployment.yaml index 0a6795bd84ff46..85315a574f8e5b 100644 --- a/chart/templates/server-deployment.yaml +++ b/chart/templates/server-deployment.yaml @@ -202,6 +202,14 @@ spec: value: {{ $comp.defaultBaseImageRegistryWhitelist | toJson | quote }} - name: GITPOD_DEFAULT_FEATURE_FLAGS value: {{ $comp.defaultFeatureFlags | toJson | quote }} + {{- if $comp.incrementalPrebuilds.repositoryPasslist }} + - name: INCREMENTAL_PREBUILDS_REPO_PASSLIST + value: {{ $comp.incrementalPrebuilds.repositoryPasslist | toJson | quote }} + {{- end }} + {{- if $comp.incrementalPrebuilds.commitHistory }} + - name: INCREMENTAL_PREBUILDS_COMMIT_HISTORY + value: {{ $comp.incrementalPrebuilds.commitHistory | quote }} + {{- end }} - name: AUTH_PROVIDERS_CONFIG valueFrom: configMapKeyRef: diff --git a/chart/values.yaml b/chart/values.yaml index d4f11113527fee..9a378a95255af3 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -280,6 +280,9 @@ components: wsman: [] defaultBaseImageRegistryWhitelist: [] defaultFeatureFlags: [] + incrementalPrebuilds: + repositoryPasslist: [] + commitHistory: 100 ports: http: expose: true diff --git a/components/gitpod-db/src/typeorm/migration/README.md b/components/gitpod-db/src/typeorm/migration/README.md index 1245909abfb466..c6ce558eb646ba 100644 --- a/components/gitpod-db/src/typeorm/migration/README.md +++ b/components/gitpod-db/src/typeorm/migration/README.md @@ -4,6 +4,7 @@ To create a new migration file, run this command in the `gitpod-db` component di ``` yarn typeorm migrations:create -n NameOfYourMigration +leeway run components:update-license-header ``` Then, simply populate the `up` and `down` methods in the generated migration file. diff --git a/components/gitpod-protocol/src/protocol.ts b/components/gitpod-protocol/src/protocol.ts index 341afaeb02b19f..3eda11672b33d0 100644 --- a/components/gitpod-protocol/src/protocol.ts +++ b/components/gitpod-protocol/src/protocol.ts @@ -813,6 +813,7 @@ export namespace SnapshotContext { export interface StartPrebuildContext extends WorkspaceContext { actual: WorkspaceContext; + commitHistory?: string[]; } export namespace StartPrebuildContext { diff --git a/components/server/ee/src/prebuilds/prebuild-manager.ts b/components/server/ee/src/prebuilds/prebuild-manager.ts index 6fe26745e9d497..06a386c36ae9ba 100644 --- a/components/server/ee/src/prebuilds/prebuild-manager.ts +++ b/components/server/ee/src/prebuilds/prebuild-manager.ts @@ -16,6 +16,7 @@ import { StartPrebuildResult } from './github-app'; import { WorkspaceFactory } from '../../../src/workspace/workspace-factory'; import { ConfigProvider } from '../../../src/workspace/config-provider'; import { WorkspaceStarter } from '../../../src/workspace/workspace-starter'; +import { Env } from '../../../src/env'; export class WorkspaceRunningError extends Error { constructor(msg: string, public instance: WorkspaceInstance) { @@ -30,6 +31,7 @@ export class PrebuildManager { @inject(WorkspaceStarter) protected readonly workspaceStarter: WorkspaceStarter; @inject(HostContextProvider) protected readonly hostContextProvider: HostContextProvider; @inject(ConfigProvider) protected readonly configProvider: ConfigProvider; + @inject(Env) protected env: Env; async hasAutomatedPrebuilds(ctx: TraceContext, cloneURL: string): Promise { const span = TraceContext.startSpan("hasPrebuilds", ctx); @@ -75,6 +77,11 @@ export class PrebuildManager { actual }; + if (this.shouldPrebuildIncrementally(actual.repository.cloneUrl)) { + const maxDepth = this.env.incrementalPrebuildsCommitHistory; + prebuildContext.commitHistory = await contextParser.fetchCommitHistory({ span }, user, contextURL, commit, maxDepth); + } + log.debug("Created prebuild context", prebuildContext); const workspace = await this.workspaceFactory.createForContext({span}, user, prebuildContext, contextURL); @@ -137,6 +144,12 @@ export class PrebuildManager { return true; } + protected shouldPrebuildIncrementally(cloneUrl: string): boolean { + const trimRepoUrl = (url: string) => url.replace(/\/$/, '').replace(/\.git$/, ''); + const repoUrl = trimRepoUrl(cloneUrl); + return this.env.incrementalPrebuildsRepositoryPassList.some(url => trimRepoUrl(url) === repoUrl); + } + async fetchConfig(ctx: TraceContext, user: User, contextURL: string): Promise { const span = TraceContext.startSpan("fetchConfig", ctx); span.setTag("contextURL", contextURL); diff --git a/components/server/ee/src/workspace/gitpod-server-impl.ts b/components/server/ee/src/workspace/gitpod-server-impl.ts index 854c24d8fbb2fd..05f7c514c1951a 100644 --- a/components/server/ee/src/workspace/gitpod-server-impl.ts +++ b/components/server/ee/src/workspace/gitpod-server-impl.ts @@ -549,7 +549,7 @@ export class GitpodServerEEImpl const logCtx: LogContext = { userId: user.id }; const cloneUrl = context.repository.cloneUrl; - const prebuiltWorkspace = await this.workspaceDb.trace({ span }).findPrebuiltWorkspaceByCommit(context.repository.cloneUrl, context.revision); + const prebuiltWorkspace = await this.workspaceDb.trace({ span }).findPrebuiltWorkspaceByCommit(cloneUrl, context.revision); const logPayload = { mode, cloneUrl, commit: context.revision, prebuiltWorkspace }; log.debug(logCtx, "Looking for prebuilt workspace: ", logPayload); if (!prebuiltWorkspace) { @@ -568,6 +568,10 @@ export class GitpodServerEEImpl if (mode === CreateWorkspaceMode.ForceNew) { // in force mode we ignore running prebuilds as we want to start a workspace as quickly as we can. return; + // TODO(janx): Fall back to parent prebuild instead, if it's available: + // const buildWorkspace = await this.workspaceDb.trace({span}).findById(prebuiltWorkspace.buildWorkspaceId); + // const parentPrebuild = await this.workspaceDb.trace({span}).findPrebuildByID(buildWorkspace.basedOnPrebuildId); + // Also, make sure to initialize it by both printing the parent prebuild logs AND re-runnnig the before/init/prebuild tasks. } let result: WorkspaceCreationResult = { diff --git a/components/server/ee/src/workspace/workspace-factory.ts b/components/server/ee/src/workspace/workspace-factory.ts index 1bf75bf3943fb6..efa42203e6f440 100644 --- a/components/server/ee/src/workspace/workspace-factory.ts +++ b/components/server/ee/src/workspace/workspace-factory.ts @@ -8,7 +8,7 @@ import * as uuidv4 from 'uuid/v4'; import { WorkspaceFactory } from "../../../src/workspace/workspace-factory"; import { injectable, inject } from "inversify"; import { TraceContext } from "@gitpod/gitpod-protocol/lib/util/tracing"; -import { User, StartPrebuildContext, Workspace, CommitContext, PrebuiltWorkspaceContext, WorkspaceContext, WithSnapshot, WithPrebuild } from "@gitpod/gitpod-protocol"; +import { User, StartPrebuildContext, Workspace, CommitContext, PrebuiltWorkspaceContext, WorkspaceContext, WithSnapshot, WithPrebuild, TaskConfig } from "@gitpod/gitpod-protocol"; import { log } from '@gitpod/gitpod-protocol/lib/util/logging'; import { LicenseEvaluator } from '@gitpod/licensor/lib'; import { Feature } from '@gitpod/licensor/lib/api'; @@ -56,7 +56,62 @@ export class WorkspaceFactoryEE extends WorkspaceFactory { } } - let ws = await this.createForCommit({span}, user, commitContext, normalizedContextURL); + const config = await this.configProvider.fetchConfig({span}, user, context.actual); + const imageSource = await this.imageSourceProvider.getImageSource(ctx, user, context.actual, config); + + // Walk back the commit history to find suitable parent prebuild to start an incremental prebuild on. + let ws; + for (const parent of (context.commitHistory || [])) { + const parentPrebuild = await this.db.trace({span}).findPrebuiltWorkspaceByCommit(commitContext.repository.cloneUrl, parent); + if (!parentPrebuild) { + continue; + } + if (parentPrebuild.state !== 'available') { + continue; + } + log.debug(`Considering parent prebuild for ${commitContext.revision}`, parentPrebuild); + const buildWorkspace = await this.db.trace({span}).findById(parentPrebuild.buildWorkspaceId); + if (!buildWorkspace) { + continue; + } + if (!!buildWorkspace.basedOnPrebuildId) { + continue; + } + if (JSON.stringify(imageSource) !== JSON.stringify(buildWorkspace.imageSource)) { + log.debug(`Skipping parent prebuild: Outdated image`, { + imageSource, + parentImageSource: buildWorkspace.imageSource, + }); + continue; + } + const filterPrebuildTasks = (tasks: TaskConfig[] = []) => (tasks + .map(task => Object.keys(task) + .filter(key => ['before', 'init', 'prebuild'].includes(key)) + // @ts-ignore + .reduce((obj, key) => ({ ...obj, [key]: task[key] }), {})) + .filter(task => Object.keys(task).length > 0)); + const prebuildTasks = filterPrebuildTasks(config.tasks); + const parentPrebuildTasks = filterPrebuildTasks(buildWorkspace.config.tasks); + if (JSON.stringify(prebuildTasks) !== JSON.stringify(parentPrebuildTasks)) { + log.debug(`Skipping parent prebuild: Outdated prebuild tasks`, { + prebuildTasks, + parentPrebuildTasks, + }); + continue; + } + const incrementalPrebuildContext: PrebuiltWorkspaceContext = { + title: `Incremental prebuild of "${commitContext.title}"`, + originalContext: commitContext, + prebuiltWorkspace: parentPrebuild, + } + ws = await this.createForPrebuiltWorkspace({span}, user, incrementalPrebuildContext, normalizedContextURL); + break; + } + + if (!ws) { + // No suitable parent prebuild was found -- create a (fresh) full prebuild. + ws = await this.createForCommit({span}, user, commitContext, normalizedContextURL); + } ws.type = "prebuild"; ws = await this.db.trace({span}).store(ws); @@ -82,7 +137,7 @@ export class WorkspaceFactoryEE extends WorkspaceFactory { protected async createForPrebuiltWorkspace(ctx: TraceContext, user: User, context: PrebuiltWorkspaceContext, normalizedContextURL: string): Promise { this.requireEELicense(Feature.FeaturePrebuild); - const span = TraceContext.startSpan("createForStartPrebuild", ctx); + const span = TraceContext.startSpan("createForPrebuiltWorkspace", ctx); const fallback = await this.fallbackIfOutPrebuildTime(ctx, user, context, normalizedContextURL); if (!!fallback) { diff --git a/components/server/src/bitbucket/bitbucket-context-parser.spec.ts b/components/server/src/bitbucket/bitbucket-context-parser.spec.ts index 8b948b0b4c67d3..32f4b4df610dcd 100644 --- a/components/server/src/bitbucket/bitbucket-context-parser.spec.ts +++ b/components/server/src/bitbucket/bitbucket-context-parser.spec.ts @@ -451,6 +451,13 @@ class TestBitbucketContextParser { "title": "gitpod/clu-sample-repo - master" }) } + + @test public async testFetchCommitHistory() { + const result = await this.parser.fetchCommitHistory({}, this.user, 'https://bitbucket.org/gitpod/sample-repository', 'dd0aef8097a7c521b8adfced795fcf96c9e598ef', 100); + expect(result).to.deep.equal([ + 'da2119f51b0e744cb6b36399f8433b477a4174ef', + ]) + } } module.exports = new TestBitbucketContextParser(); diff --git a/components/server/src/bitbucket/bitbucket-context-parser.ts b/components/server/src/bitbucket/bitbucket-context-parser.ts index b0b47523037625..7ff28a46690d90 100644 --- a/components/server/src/bitbucket/bitbucket-context-parser.ts +++ b/components/server/src/bitbucket/bitbucket-context-parser.ts @@ -259,4 +259,27 @@ export class BitbucketContextParser extends AbstractContextParser implements ICo return result; } + + public async fetchCommitHistory(ctx: TraceContext, user: User, contextUrl: string, sha: string, maxDepth: number): Promise { + const span = TraceContext.startSpan("BitbucketContextParser.fetchCommitHistory", ctx); + try { + // TODO(janx): To get more results than Bitbucket API's max pagelen (seems to be 100), pagination should be handled. + // The additional property 'page' may be helfpul. + const api = await this.api(user); + const { owner, repoName } = await this.parseURL(user, contextUrl); + const result = await api.repositories.listCommitsAt({ + workspace: owner, + repo_slug: repoName, + revision: sha, + pagelen: maxDepth, + }); + return result.data.values.slice(1).map((v: Schema.Commit) => v.hash); + } catch (e) { + span.log({ error: e }); + log.error({ userId: user.id }, "Error fetching Bitbucket commit history", e); + throw e; + } finally { + span.finish(); + } + } } diff --git a/components/server/src/env.ts b/components/server/src/env.ts index 959d4a569484fb..7bccd3fc036377 100644 --- a/components/server/src/env.ts +++ b/components/server/src/env.ts @@ -46,16 +46,25 @@ export class Env extends AbstractComponentEnv { })() readonly previewFeatureFlags: NamedWorkspaceFeatureFlag[] = (() => { - const value = process.env.EXPERIMENTAL_FEATURE_FLAGS; - if (!value) { - return []; + return this.parseStringArray('EXPERIMENTAL_FEATURE_FLAGS') as NamedWorkspaceFeatureFlag[]; + })(); + + protected parseStringArray(name: string): string[] { + const json = process.env[name]; + if (!json) { + return []; } - const flags = JSON.parse(value); - if (!Array.isArray(flags)) { - throw new Error(`EXPERIMENTAL_FEATURE_FLAGS should be an Array: ${value}`); + let value; + try { + value = JSON.parse(json); + } catch (error) { + throw new Error(`Could not parse ${name}: ${error}`); } - return flags; - })(); + if (!Array.isArray(value) || value.some(e => typeof e !== 'string')) { + throw `${name} should be an array of string: ${json}`; + } + return value; + } readonly gitpodRegion: string = process.env.GITPOD_REGION || 'unknown'; @@ -111,6 +120,15 @@ export class Env extends AbstractComponentEnv { // maxConcurrentPrebuildsPerRef is the maximum number of prebuilds we allow per ref type at any given time readonly maxConcurrentPrebuildsPerRef = Number.parseInt(process.env.MAX_CONCUR_PREBUILDS_PER_REF || '10', 10) || 10; + readonly incrementalPrebuildsRepositoryPassList: string[] = (() => { + try { + return this.parseStringArray('INCREMENTAL_PREBUILDS_REPO_PASSLIST'); + } catch (error) { + console.error(error); + return []; + } + })() + readonly incrementalPrebuildsCommitHistory: number = Number.parseInt(process.env.INCREMENTAL_PREBUILDS_COMMIT_HISTORY || '100', 10) || 100; protected gitpodLayernameFromFilesystem: string | null | undefined; protected readGitpodLayernameFromFilesystem(): string | undefined { @@ -140,19 +158,10 @@ export class Env extends AbstractComponentEnv { readonly blockNewUsers: boolean = this.parseBool("BLOCK_NEW_USERS"); readonly blockNewUsersPassList: string[] = (() => { - const l = process.env.BLOCK_NEW_USERS_PASSLIST; - if (!l) { - return []; - } try { - const res = JSON.parse(l); - if (!Array.isArray(res) || res.some(e => typeof e !== 'string')) { - console.error("BLOCK_NEW_USERS_PASSLIST is not an array of string"); - return []; - } - return res; - } catch (err) { - console.error("cannot parse BLOCK_NEW_USERS_PASSLIST", err); + return this.parseStringArray('BLOCK_NEW_USERS_PASSLIST'); + } catch (error) { + console.error(error); return []; } })(); @@ -164,26 +173,17 @@ export class Env extends AbstractComponentEnv { /** defaultBaseImageRegistryWhitelist is the list of registryies users get acces to by default */ readonly defaultBaseImageRegistryWhitelist: string[] = (() => { - const wljson = process.env.GITPOD_BASEIMG_REGISTRY_WHITELIST; - if (!wljson) { - return []; - } - - return JSON.parse(wljson); + return this.parseStringArray('GITPOD_BASEIMG_REGISTRY_WHITELIST'); })() readonly defaultFeatureFlags: NamedWorkspaceFeatureFlag[] = (() => { - const json = process.env.GITPOD_DEFAULT_FEATURE_FLAGS; - if (!json) { - return []; - } - - let r = JSON.parse(json); - if (!Array.isArray(r)) { + try { + const r = (this.parseStringArray('GITPOD_DEFAULT_FEATURE_FLAGS') as NamedWorkspaceFeatureFlag[]); + return r.filter(e => e in WorkspaceFeatureFlags); + } catch (error) { + console.error(error); return []; } - r = r.filter(e => e in WorkspaceFeatureFlags); - return r; })(); /** defaults to: false */ diff --git a/components/server/src/github/github-context-parser.spec.ts b/components/server/src/github/github-context-parser.spec.ts index 16f7c37aa9e1ac..f54c2cfe32e771 100644 --- a/components/server/src/github/github-context-parser.spec.ts +++ b/components/server/src/github/github-context-parser.spec.ts @@ -320,7 +320,6 @@ class TestGithubContextParser { }) } - @test public async testCommitContext_02_notExistingCommit() { try { await this.parser.handle({}, this.user, 'https://github.com/gitpod-io/gitpod-test-repo/commit/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'); @@ -537,5 +536,13 @@ class TestGithubContextParser { ) } + @test public async testFetchCommitHistory() { + const result = await this.parser.fetchCommitHistory({}, this.user, 'https://github.com/gitpod-io/gitpod-test-repo', '409ac2de49a53d679989d438735f78204f441634', 100); + expect(result).to.deep.equal([ + '506e5aed317f28023994ecf8ca6ed91430e9c1a4', + 'f5b041513bfab914b5fbf7ae55788d9835004d76', + ]) + } + } module.exports = new TestGithubContextParser() // Only to circumvent no usage warning :-/ \ No newline at end of file diff --git a/components/server/src/github/github-context-parser.ts b/components/server/src/github/github-context-parser.ts index acfb1e558f3000..ee913327b0eb63 100644 --- a/components/server/src/github/github-context-parser.ts +++ b/components/server/src/github/github-context-parser.ts @@ -33,7 +33,7 @@ export class GithubContextParser extends AbstractContextParser implements IConte if (moreSegments.length > 0) { switch (moreSegments[0]) { case 'pull': { - return await this.handlePullRequestContext({span}, user, host, owner, repoName, parseInt(moreSegments[1])); + return await this.handlePullRequestContext({span}, user, host, owner, repoName, parseInt(moreSegments[1], 10)); } case 'tree': case 'blob': @@ -41,7 +41,7 @@ export class GithubContextParser extends AbstractContextParser implements IConte return await this.handleTreeContext({span}, user, host, owner, repoName, moreSegments.slice(1)); } case 'issues': { - return await this.handleIssueContext({span}, user, host, owner, repoName, parseInt(moreSegments[1])); + return await this.handleIssueContext({span}, user, host, owner, repoName, parseInt(moreSegments[1], 10)); } case 'commit': { return await this.handleCommitContext({span}, user, host, owner, repoName, moreSegments[1]); @@ -72,7 +72,7 @@ export class GithubContextParser extends AbstractContextParser implements IConte try { const result: any = await this.githubQueryApi.runQuery(user, ` - query { + query { repository(name: "${repoName}", owner: "${owner}") { ${this.repoProperties()} defaultBranchRef { @@ -210,24 +210,24 @@ export class GithubContextParser extends AbstractContextParser implements IConte try { const result: any = await this.githubQueryApi.runQuery(user, ` - query { - repository(name: "${repoName}", owner: "${owner}") { - object(oid: "${sha}") { - oid, - ... on Commit { - messageHeadline - } + query { + repository(name: "${repoName}", owner: "${owner}") { + object(oid: "${sha}") { + oid, + ... on Commit { + messageHeadline } - ${this.repoProperties()} - defaultBranchRef { - name, - target { - oid - } - }, } + ${this.repoProperties()} + defaultBranchRef { + name, + target { + oid + } + }, } - `); + } + `); span.log({"request.finished": ""}); if (result.data.repository === null) { @@ -264,30 +264,30 @@ export class GithubContextParser extends AbstractContextParser implements IConte const result: any = await this.githubQueryApi.runQuery(user, ` query { repository(name: "${repoName}", owner: "${owner}") { - pullRequest(number: ${pullRequestNr}) { - title - headRef { - name - repository { - ${this.repoProperties()} - } - target { - oid - } - } - baseRef { - name - repository { - ${this.repoProperties()} - } - target { - oid - } + pullRequest(number: ${pullRequestNr}) { + title + headRef { + name + repository { + ${this.repoProperties()} + } + target { + oid + } + } + baseRef { + name + repository { + ${this.repoProperties()} + } + target { + oid + } + } } } - } } - `); + `); span.log({"request.finished": ""}); if (result.data.repository === null) { @@ -331,21 +331,21 @@ export class GithubContextParser extends AbstractContextParser implements IConte try { const result: any = await this.githubQueryApi.runQuery(user, ` - query { - repository(name: "${repoName}", owner: "${owner}") { - issue(number: ${issueNr}) { - title - } - ${this.repoProperties()} - defaultBranchRef { - name, - target { - oid - } - }, + query { + repository(name: "${repoName}", owner: "${owner}") { + issue(number: ${issueNr}) { + title } + ${this.repoProperties()} + defaultBranchRef { + name, + target { + oid + } + }, } - `); + } + `); span.log({"request.finished": ""}); if (result.data.repository === null) { @@ -416,4 +416,56 @@ export class GithubContextParser extends AbstractContextParser implements IConte }` : ''} `; } + + public async fetchCommitHistory(ctx: TraceContext, user: User, contextUrl: string, sha: string, maxDepth: number): Promise { + const span = TraceContext.startSpan("GithubContextParser.fetchCommitHistory", ctx); + + try { + if (sha.length != 40) { + throw new Error(`Invalid commit ID ${sha}.`); + } + + // TODO(janx): To get more results than GitHub API's max page size (seems to be 100), pagination should be handled. + // These additional history properties may be helfpul: + // totalCount, + // pageInfo { + // haxNextPage, + // }, + const { owner, repoName } = await this.parseURL(user, contextUrl); + const result: any = await this.githubQueryApi.runQuery(user, ` + query { + repository(name: "${repoName}", owner: "${owner}") { + object(oid: "${sha}") { + ... on Commit { + history(first: ${maxDepth}) { + edges { + node { + oid + } + } + } + } + } + } + } + `); + span.log({"request.finished": ""}); + + if (result.data.repository === null) { + throw await NotFoundError.create(await this.tokenHelper.getCurrentToken(user), user, this.config.host, owner, repoName); + } + + const commit = result.data.repository.object; + if (commit === null) { + throw new Error(`Couldn't find commit ${sha} in repository ${owner}/${repoName}.`); + } + + return commit.history.edges.slice(1).map((e: any) => e.node.oid) || []; + } catch (e) { + span.log({"error": e}); + throw e; + } finally { + span.finish(); + } + } } diff --git a/components/server/src/gitlab/gitlab-context-parser.spec.ts b/components/server/src/gitlab/gitlab-context-parser.spec.ts index 9d897571aeb354..339fcda4888ea3 100644 --- a/components/server/src/gitlab/gitlab-context-parser.spec.ts +++ b/components/server/src/gitlab/gitlab-context-parser.spec.ts @@ -547,6 +547,14 @@ class TestGitlabContextParser { }) } + @test public async testFetchCommitHistory() { + const result = await this.parser.fetchCommitHistory({}, this.user, 'https://gitlab.com/AlexTugarev/gp-test', '80948e8cc8f0e851e89a10bc7c2ee234d1a5fbe7', 100); + expect(result).to.deep.equal([ + '4447fbc4d46e6fd1ee41fb1b992702521ae078eb', + 'f2d9790f2752a794517b949c65a773eb864844cd', + ]) + } + } module.exports = new TestGitlabContextParser(); diff --git a/components/server/src/gitlab/gitlab-context-parser.ts b/components/server/src/gitlab/gitlab-context-parser.ts index 5fed7443878176..bc57b60bfd318f 100644 --- a/components/server/src/gitlab/gitlab-context-parser.ts +++ b/components/server/src/gitlab/gitlab-context-parser.ts @@ -339,4 +339,24 @@ export class GitlabContextParser extends AbstractContextParser implements IConte repository, }; } + + public async fetchCommitHistory(ctx: TraceContext, user: User, contextUrl: string, sha: string, maxDepth: number): Promise { + // TODO(janx): To get more results than GitLab API's max per_page (seems to be 100), pagination should be handled. + const { owner, repoName } = await this.parseURL(user, contextUrl); + const projectId = `${owner}/${repoName}`; + const result = await this.gitlabApi.run(user, async g => { + return g.Commits.all(projectId, { + ref_name: sha, + per_page: maxDepth, + page: 1, + }); + }); + if (GitLab.ApiError.is(result)) { + if (result.message === 'GitLab responded with code 404') { + throw new Error(`Couldn't find commit #${sha} in repository ${projectId}.`); + } + throw result; + } + return result.slice(1).map((c: GitLab.Commit) => c.id); + } } \ No newline at end of file diff --git a/components/server/src/workspace/context-parser.ts b/components/server/src/workspace/context-parser.ts index 2722f5479d2ac2..414211ad25667a 100644 --- a/components/server/src/workspace/context-parser.ts +++ b/components/server/src/workspace/context-parser.ts @@ -11,9 +11,10 @@ import { AuthProviderParams } from "../auth/auth-provider"; import { URLSearchParams, URL } from "url"; export interface IContextParser { - normalize?(contextURL: string): string | undefined - canHandle(user: User, context: string): boolean - handle(ctx: TraceContext, user: User, context: string): Promise + normalize?(contextUrl: string): string | undefined + canHandle(user: User, contextUrl: string): boolean + handle(ctx: TraceContext, user: User, contextUrl: string): Promise + fetchCommitHistory(ctx: TraceContext, user: User, contextUrl: string, commit: string, maxDepth: number): Promise } export const IContextParser = Symbol("IContextParser") @@ -73,7 +74,14 @@ export abstract class AbstractContextParser implements IContextParser { return lastSegment && urlSegment.endsWith('.git') ? urlSegment.substring(0, urlSegment.length - '.git'.length) : urlSegment; } - public abstract handle(ctx: TraceContext, user: User, context: string): Promise; + public abstract handle(ctx: TraceContext, user: User, contextUrl: string): Promise; + + /** + * Fetches the commit history of a commit (used to find a relevant parent prebuild for incremental prebuilds). + * + * @returns the linear commit history starting from (but excluding) the given commit, in the same order as `git log` + */ + public abstract fetchCommitHistory(ctx: TraceContext, user: User, contextUrl: string, commit: string, maxDepth: number): Promise; } export interface URLParts { diff --git a/components/server/src/workspace/snapshot-context-parser.ts b/components/server/src/workspace/snapshot-context-parser.ts index 72dcea72556135..2608372ec2ee13 100644 --- a/components/server/src/workspace/snapshot-context-parser.ts +++ b/components/server/src/workspace/snapshot-context-parser.ts @@ -33,4 +33,8 @@ export class SnapshotContextParser implements IContextParser { } } + public async fetchCommitHistory(ctx: TraceContext, user: User, contextUrl: string, commit: string, maxDepth: number): Promise { + throw new Error('SnapshotContextParser does not support fetching commit history'); + } + } \ No newline at end of file diff --git a/components/supervisor/pkg/supervisor/tasks.go b/components/supervisor/pkg/supervisor/tasks.go index b741f26d0bd007..87b96392a2e43a 100644 --- a/components/supervisor/pkg/supervisor/tasks.go +++ b/components/supervisor/pkg/supervisor/tasks.go @@ -378,6 +378,8 @@ func (tm *tasksManager) watch(task *task, terminal *terminal.Term) { defer stdout.Close() fileName := tm.prebuildLogFileName(task) + // TODO(janx): If the file already exists (from a parent prebuild), extract its "time saved", and log that below + // (instead, or in addition to, the incremental prebuild time). file, err := os.Create(fileName) var fileWriter *bufio.Writer if err != nil {