diff --git a/components/dashboard/package.json b/components/dashboard/package.json index ba5784260e7f2f..672c961f9693f6 100644 --- a/components/dashboard/package.json +++ b/components/dashboard/package.json @@ -7,6 +7,7 @@ "@gitpod/gitpod-protocol": "0.1.5", "countries-list": "^2.6.1", "moment": "^2.29.1", + "monaco-editor": "^0.25.2", "react": "^17.0.1", "react-dom": "^17.0.1", "react-router-dom": "^5.2.0", diff --git a/components/dashboard/src/App.tsx b/components/dashboard/src/App.tsx index cfcf1c695662b1..711c87c0f04214 100644 --- a/components/dashboard/src/App.tsx +++ b/components/dashboard/src/App.tsx @@ -32,6 +32,7 @@ const NewTeam = React.lazy(() => import(/* webpackPrefetch: true */ './teams/New const JoinTeam = React.lazy(() => import(/* webpackPrefetch: true */ './teams/JoinTeam')); const Members = React.lazy(() => import(/* webpackPrefetch: true */ './teams/Members')); const NewProject = React.lazy(() => import(/* webpackPrefetch: true */ './projects/NewProject')); +const ConfigureProject = React.lazy(() => import(/* webpackPrefetch: true */ './projects/ConfigureProject')); const Projects = React.lazy(() => import(/* webpackPrefetch: true */ './projects/Projects')); const Project = React.lazy(() => import(/* webpackPrefetch: true */ './projects/Project')); const Prebuilds = React.lazy(() => import(/* webpackPrefetch: true */ './projects/Prebuilds')); @@ -206,6 +207,9 @@ function App() { if (maybeProject === "members") { return ; } + if (subResource === "configure") { + return ; + } if (subResource === "prebuilds") { return ; } diff --git a/components/dashboard/src/Menu.tsx b/components/dashboard/src/Menu.tsx index d716436731a80b..62f2205e32c551 100644 --- a/components/dashboard/src/Menu.tsx +++ b/components/dashboard/src/Menu.tsx @@ -48,7 +48,7 @@ export default function Menu() { })(); const userFullName = user?.fullName || user?.name || '...'; - const showTeamsUI = user?.rolesOrPermissions?.includes('teams-and-projects') || window.location.hostname.endsWith('gitpod-dev.com') || window.location.hostname.endsWith('gitpod-io-dev.com'); + const showTeamsUI = user?.rolesOrPermissions?.includes('teams-and-projects'); const team = getCurrentTeam(location, teams); // Hide most of the top menu when in a full-page form. @@ -97,6 +97,10 @@ export default function Menu() { { title: 'Settings', link: `/${team.slug}/${projectName}/settings` + }, + { + title: 'Configure', + link: `/${team.slug}/${projectName}/configure` } ] : [ { diff --git a/components/dashboard/src/components/MonacoEditor.tsx b/components/dashboard/src/components/MonacoEditor.tsx new file mode 100644 index 00000000000000..844ca0274d371d --- /dev/null +++ b/components/dashboard/src/components/MonacoEditor.tsx @@ -0,0 +1,44 @@ +/** + * Copyright (c) 2021 Gitpod GmbH. All rights reserved. + * Licensed under the GNU Affero General Public License (AGPL). + * See License-AGPL.txt in the project root for license information. + */ + +import { useEffect, useRef } from "react"; +import * as monaco from "monaco-editor"; + +export default function MonacoEditor(props: { classes: string, disabled?: boolean, language: string, value: string, onChange: (value: string) => void }) { + const containerRef = useRef(null); + const editorRef = useRef(); + + useEffect(() => { + if (containerRef.current) { + editorRef.current = monaco.editor.create(containerRef.current, { + value: props.value, + language: props.language, + minimap: { + enabled: false, + }, + renderLineHighlight: 'none', + }); + editorRef.current.onDidChangeModelContent(() => { + props.onChange(editorRef.current!.getValue()); + }); + } + return () => editorRef.current?.dispose(); + }, []); + + useEffect(() => { + if (editorRef.current && editorRef.current.getValue() !== props.value) { + editorRef.current.setValue(props.value); + } + }, [ props.value ]); + + useEffect(() => { + if (editorRef.current) { + editorRef.current.updateOptions({ readOnly: props.disabled }); + } + }, [ props.disabled ]); + + return
; +} \ No newline at end of file diff --git a/components/dashboard/src/components/PrebuildLogs.tsx b/components/dashboard/src/components/PrebuildLogs.tsx new file mode 100644 index 00000000000000..f57a3306f8407f --- /dev/null +++ b/components/dashboard/src/components/PrebuildLogs.tsx @@ -0,0 +1,205 @@ +/** + * Copyright (c) 2021 Gitpod GmbH. All rights reserved. + * Licensed under the GNU Affero General Public License (AGPL). + * See License-AGPL.txt in the project root for license information. + */ + +import EventEmitter from "events"; +import React, { Suspense, useEffect, useState } from "react"; +import { Workspace, WorkspaceInstance, DisposableCollection, WorkspaceImageBuild, GitpodServer, HEADLESS_LOG_STREAM_STATUS_CODE_REGEX } from "@gitpod/gitpod-protocol"; +import { getGitpodService } from "../service/service"; + +const WorkspaceLogs = React.lazy(() => import('./WorkspaceLogs')); + +export default function PrebuildLogs(props: { workspaceId?: string }) { + const [ workspace, setWorkspace ] = useState(); + const [ workspaceInstance, setWorkspaceInstance ] = useState(); + const [ error, setError ] = useState(); + const logsEmitter = new EventEmitter(); + const service = getGitpodService(); + + useEffect(() => { + const disposables = new DisposableCollection(); + (async () => { + if (!props.workspaceId) { + return; + } + try { + const info = await service.server.getWorkspace(props.workspaceId); + if (info.latestInstance) { + setWorkspace(info.workspace); + setWorkspaceInstance(info.latestInstance); + } + disposables.push(service.registerClient({ + onInstanceUpdate: setWorkspaceInstance, + onWorkspaceImageBuildLogs: (info: WorkspaceImageBuild.StateInfo, content?: WorkspaceImageBuild.LogContent) => { + if (!content) { + return; + } + logsEmitter.emit('logs', content.text); + }, + })); + if (info.latestInstance) { + disposables.push(watchHeadlessLogs(service.server, info.latestInstance.id, chunk => { + logsEmitter.emit('logs', chunk); + }, async () => workspaceInstance?.status.phase === 'stopped')); + } + } catch (err) { + console.error(err); + setError(err); + } + })(); + return function cleanUp() { + disposables.dispose(); + } + }, [ props.workspaceId ]); + + useEffect(() => { + switch (workspaceInstance?.status.phase) { + // unknown indicates an issue within the system in that it cannot determine the actual phase of + // a workspace. This phase is usually accompanied by an error. + case "unknown": + break; + + // Preparing means that we haven't actually started the workspace instance just yet, but rather + // are still preparing for launch. This means we're building the Docker image for the workspace. + case "preparing": + service.server.watchWorkspaceImageBuildLogs(workspace!.id); + break; + + // Pending means the workspace does not yet consume resources in the cluster, but rather is looking for + // some space within the cluster. If for example the cluster needs to scale up to accomodate the + // workspace, the workspace will be in Pending state until that happened. + case "pending": + break; + + // Creating means the workspace is currently being created. That includes downloading the images required + // to run the workspace over the network. The time spent in this phase varies widely and depends on the current + // network speed, image size and cache states. + case "creating": + break; + + // Initializing is the phase in which the workspace is executing the appropriate workspace initializer (e.g. Git + // clone or backup download). After this phase one can expect the workspace to either be Running or Failed. + case "initializing": + break; + + // Running means the workspace is able to actively perform work, either by serving a user through Theia, + // or as a headless workspace. + case "running": + break; + + // Interrupted is an exceptional state where the container should be running but is temporarily unavailable. + // When in this state, we expect it to become running or stopping anytime soon. + case "interrupted": + break; + + // Stopping means that the workspace is currently shutting down. It could go to stopped every moment. + case "stopping": + break; + + // Stopped means the workspace ended regularly because it was shut down. + case "stopped": + service.server.watchWorkspaceImageBuildLogs(workspace!.id); + break; + } + if (workspaceInstance?.status.conditions.failed) { + setError(new Error(workspaceInstance.status.conditions.failed)); + } + }, [ workspaceInstance?.status.phase ]); + + return <> +
{workspaceInstance?.status.phase}
+ }> + + +
+ {workspaceInstance?.status.phase === 'stopped' + ? + : } +
+ ; +} + +export function watchHeadlessLogs(server: GitpodServer, instanceId: string, onLog: (chunk: string) => void, checkIsDone: () => Promise): DisposableCollection { + const disposables = new DisposableCollection(); + + const startWatchingLogs = async () => { + if (await checkIsDone()) { + return; + } + + const retry = async (reason: string, err?: Error) => { + console.debug("re-trying headless-logs because: " + reason, err); + await new Promise((resolve) => { + setTimeout(resolve, 2000); + }); + startWatchingLogs().catch(console.error); + }; + + let response: Response | undefined = undefined; + let reader: ReadableStreamDefaultReader | undefined = undefined; + try { + const logSources = await server.getHeadlessLog(instanceId); + // TODO(gpl) Only listening on first stream for now + const streamIds = Object.keys(logSources.streams); + if (streamIds.length < 1) { + await retry("no streams"); + return; + } + + const streamUrl = logSources.streams[streamIds[0]]; + console.log("fetching from streamUrl: " + streamUrl); + response = await fetch(streamUrl, { + method: 'GET', + cache: 'no-cache', + credentials: 'include', + keepalive: true, + headers: { + 'TE': 'trailers', // necessary to receive stream status code + }, + }); + reader = response.body?.getReader(); + if (!reader) { + await retry("no reader"); + return; + } + disposables.push({ dispose: () => reader?.cancel() }); + + const decoder = new TextDecoder('utf-8'); + let chunk = await reader.read(); + while (!chunk.done) { + const msg = decoder.decode(chunk.value, { stream: true }); + + // In an ideal world, we'd use res.addTrailers()/response.trailer here. But despite being introduced with HTTP/1.1 in 1999, trailers are not supported by popular proxies (nginx, for example). + // So we resort to this hand-written solution: + const matches = msg.match(HEADLESS_LOG_STREAM_STATUS_CODE_REGEX); + if (matches) { + if (matches.length < 2) { + console.debug("error parsing log stream status code. msg: " + msg); + } else { + const streamStatusCode = matches[1]; + if (streamStatusCode !== "200") { + throw new Error("received status code: " + streamStatusCode); + } + } + } else { + onLog(msg); + } + + chunk = await reader.read(); + } + reader.cancel() + + if (await checkIsDone()) { + return; + } + } catch(err) { + reader?.cancel().catch(console.debug); + await retry("error while listening to stream", err); + } + }; + startWatchingLogs().catch(console.error); + + return disposables; +} \ No newline at end of file diff --git a/components/dashboard/src/components/WorkspaceLogs.tsx b/components/dashboard/src/components/WorkspaceLogs.tsx new file mode 100644 index 00000000000000..56607596e87676 --- /dev/null +++ b/components/dashboard/src/components/WorkspaceLogs.tsx @@ -0,0 +1,89 @@ +/** + * Copyright (c) 2021 Gitpod GmbH. All rights reserved. + * Licensed under the GNU Affero General Public License (AGPL). + * See License-AGPL.txt in the project root for license information. + */ + +import EventEmitter from 'events'; +import React from 'react'; +import { Terminal, ITerminalOptions, ITheme } from 'xterm'; +import { FitAddon } from 'xterm-addon-fit' +import 'xterm/css/xterm.css'; +import { DisposableCollection } from '@gitpod/gitpod-protocol'; + +export interface WorkspaceLogsProps { + logsEmitter: EventEmitter; + errorMessage?: string; + classes?: string; +} + +export interface WorkspaceLogsState { +} + +export default class WorkspaceLogs extends React.Component { + protected xTermParentRef: React.RefObject; + protected terminal: Terminal | undefined; + protected fitAddon: FitAddon | undefined; + + constructor(props: WorkspaceLogsProps) { + super(props); + this.xTermParentRef = React.createRef(); + } + + private readonly toDispose = new DisposableCollection(); + componentDidMount() { + const element = this.xTermParentRef.current; + if (element === null) { + return; + } + const theme: ITheme = {}; + const options: ITerminalOptions = { + cursorBlink: false, + disableStdin: true, + fontSize: 14, + theme, + scrollback: 9999999, + }; + this.terminal = new Terminal(options); + this.fitAddon = new FitAddon(); + this.terminal.loadAddon(this.fitAddon); + this.terminal.open(element); + this.props.logsEmitter.on('logs', logs => { + if (this.fitAddon && this.terminal && logs) { + this.terminal.write(logs); + } + }); + this.toDispose.push(this.terminal); + this.fitAddon.fit(); + + // Fit terminal on window resize (debounced) + let timeout: NodeJS.Timeout | undefined; + const onWindowResize = () => { + clearTimeout(timeout!); + timeout = setTimeout(() => this.fitAddon!.fit(), 20); + }; + window.addEventListener('resize', onWindowResize); + this.toDispose.push({ + dispose: () => { + clearTimeout(timeout!); + window.removeEventListener('resize', onWindowResize); + } + }); + } + + componentDidUpdate() { + if (this.terminal && this.props.errorMessage) { + this.terminal.write(`\n\u001b[38;5;196m${this.props.errorMessage}\u001b[0m`); + } + } + + componentWillUnmount() { + this.toDispose.dispose(); + } + + render() { + return
+
+
; + } +} diff --git a/components/dashboard/src/projects/ConfigureProject.tsx b/components/dashboard/src/projects/ConfigureProject.tsx new file mode 100644 index 00000000000000..d2f4a4c39f81a8 --- /dev/null +++ b/components/dashboard/src/projects/ConfigureProject.tsx @@ -0,0 +1,140 @@ +/** + * Copyright (c) 2021 Gitpod GmbH. All rights reserved. + * Licensed under the GNU Affero General Public License (AGPL). + * See License-AGPL.txt in the project root for license information. + */ + +import React, { Suspense, useContext, useEffect, useState } from "react"; +import { useLocation, useRouteMatch } from "react-router"; +import { CreateWorkspaceMode, ProjectInfo, WorkspaceCreationResult } from "@gitpod/gitpod-protocol"; +import PrebuildLogs from "../components/PrebuildLogs"; +import TabMenuItem from "../components/TabMenuItem"; +import { getGitpodService } from "../service/service"; +import { getCurrentTeam, TeamsContext } from "../teams/teams-context"; +import AlertBox from "../components/AlertBox"; + +const MonacoEditor = React.lazy(() => import('../components/MonacoEditor')); + +const TASKS = { + NPM: `tasks: + - init: npm install + command: npm run start`, + Yarn: `tasks: + - init: yarn install + command: yarn run start`, + Go: `tasks: + - init: go get && go build ./... && go test ./... + command: go run`, + Rails: `tasks: + - init: bin/setup + command: bin/rails server`, + Rust: `tasks: + - init: cargo build + command: cargo watch -x run`, + Python: `tasks: + - init: pip install -r requirements.txt + command: python main.py`, + Other: `tasks: + - init: # TODO: install dependencies, build project + command: # TODO: start app` +} + +// const IMAGES = { +// Default: 'gitpod/workspace-full', +// '.NET': 'gitpod/workspace-dotnet', +// MongoDB: 'gitpod/workspace-mongodb', +// MySQL: 'gitpod/workspace-mysql', +// PostgreSQL: 'gitpod/workspace-postgres', +// 'Virtual Desktop (VNC)': 'gitpod/workspace-full-vnc', +// } + +export default function () { + const { teams } = useContext(TeamsContext); + const location = useLocation(); + const team = getCurrentTeam(location, teams); + const routeMatch = useRouteMatch<{ teamSlug: string, projectSlug: string }>("/:teamSlug/:projectSlug/configure"); + const [ project, setProject ] = useState(); + const [ gitpodYml, setGitpodYml ] = useState(''); + const [ dockerfile, setDockerfile ] = useState(''); + const [ editorError, setEditorError ] = useState(null); + const [ selectedEditor, setSelectedEditor ] = useState<'.gitpod.yml'|'.gitpod.Dockerfile'>('.gitpod.yml'); + const [ isEditorDisabled, setIsEditorDisabled ] = useState(true); + const [ workspaceCreationResult, setWorkspaceCreationResult ] = useState(); + + useEffect(() => { + if (!team) { + return; + } + // Disable editing while loading, or when the config comes from Git. + setIsEditorDisabled(true); + setEditorError(null); + (async () => { + const projects = await getGitpodService().server.getProjects(team.id); + const project = projects.find(p => p.name === routeMatch?.params.projectSlug); + if (project) { + setProject(project); + const configString = await getGitpodService().server.fetchProjectRepositoryConfiguration(project.id); + if (configString) { + // TODO(janx): Link to .gitpod.yml directly instead of just the cloneUrl. + setEditorError(A Gitpod configuration already exists in the project's repository. Please edit it in Gitpod instead.); + setGitpodYml(configString); + } else { + setIsEditorDisabled(false); + setGitpodYml(project.config && project.config['.gitpod.yml'] || ''); + } + } + })(); + }, [ team ]); + + const buildProject = async (event: React.MouseEvent) => { + if (!project) { + return; + } + // (event.target as HTMLButtonElement).disabled = true; + setEditorError(null); + if (!!workspaceCreationResult) { + setWorkspaceCreationResult(undefined); + } + try { + await getGitpodService().server.setProjectConfiguration(project.id, gitpodYml); + const result = await getGitpodService().server.createWorkspace({ + contextUrl: `prebuild/${project.cloneUrl}`, + mode: CreateWorkspaceMode.ForceNew, + }); + setWorkspaceCreationResult(result); + } catch (error) { + setEditorError({String(error)}); + } + } + + return
+

Configure Project

+

Fully-automate your project's dev setup. Learn more

+
+
+ {editorError && {editorError}} + + {!!dockerfile &&
+ setSelectedEditor('.gitpod.yml')} /> + setSelectedEditor('.gitpod.Dockerfile')} /> +
} + }> + {selectedEditor === '.gitpod.yml' && + } + {selectedEditor === '.gitpod.Dockerfile' && + } + +
+ +
+
+
+

Output

+ {!!workspaceCreationResult && } +
+
+
; +} diff --git a/components/dashboard/src/projects/NewProject.tsx b/components/dashboard/src/projects/NewProject.tsx index dbb703c76ac973..a2469469080462 100644 --- a/components/dashboard/src/projects/NewProject.tsx +++ b/components/dashboard/src/projects/NewProject.tsx @@ -144,7 +144,7 @@ export default function NewProject() { teamId: team.id }); - history.push(`/${team.slug}/projects`); + history.push(`/${team.slug}/${repo.name}/configure`); } const toSimpleName = (fullName: string) => { diff --git a/components/dashboard/src/start/CreateWorkspace.tsx b/components/dashboard/src/start/CreateWorkspace.tsx index e79c2ae9876551..e05c83c04735bf 100644 --- a/components/dashboard/src/start/CreateWorkspace.tsx +++ b/components/dashboard/src/start/CreateWorkspace.tsx @@ -16,9 +16,9 @@ import StartWorkspace from "./StartWorkspace"; import { openAuthorizeWindow } from "../provider-utils"; import { SelectAccountPayload } from "@gitpod/gitpod-protocol/lib/auth"; import { SelectAccountModal } from "../settings/SelectAccountModal"; -import { watchHeadlessLogs } from "./WorkspaceLogs"; +import { watchHeadlessLogs } from "../components/PrebuildLogs"; -const WorkspaceLogs = React.lazy(() => import('./WorkspaceLogs')); +const WorkspaceLogs = React.lazy(() => import('../components/WorkspaceLogs')); export interface CreateWorkspaceProps { contextUrl: string; diff --git a/components/dashboard/src/start/StartWorkspace.tsx b/components/dashboard/src/start/StartWorkspace.tsx index 77394832600728..f5e47bca0e5e13 100644 --- a/components/dashboard/src/start/StartWorkspace.tsx +++ b/components/dashboard/src/start/StartWorkspace.tsx @@ -11,9 +11,9 @@ import { ErrorCodes } from "@gitpod/gitpod-protocol/lib/messaging/error"; import PendingChangesDropdown from "../components/PendingChangesDropdown"; import { getGitpodService, gitpodHostUrl } from "../service/service"; import { StartPage, StartPhase, StartWorkspaceError } from "./StartPage"; -import { watchHeadlessLogs } from "./WorkspaceLogs"; +import { watchHeadlessLogs } from "../components/PrebuildLogs"; -const WorkspaceLogs = React.lazy(() => import('./WorkspaceLogs')); +const WorkspaceLogs = React.lazy(() => import('../components/WorkspaceLogs')); export interface StartWorkspaceProps { workspaceId: string; diff --git a/components/dashboard/src/start/WorkspaceLogs.tsx b/components/dashboard/src/start/WorkspaceLogs.tsx deleted file mode 100644 index 81acdeea5950e9..00000000000000 --- a/components/dashboard/src/start/WorkspaceLogs.tsx +++ /dev/null @@ -1,171 +0,0 @@ -/** - * Copyright (c) 2021 Gitpod GmbH. All rights reserved. - * Licensed under the GNU Affero General Public License (AGPL). - * See License-AGPL.txt in the project root for license information. - */ - -import EventEmitter from 'events'; -import React from 'react'; -import { Terminal, ITerminalOptions, ITheme } from 'xterm'; -import { FitAddon } from 'xterm-addon-fit' -import 'xterm/css/xterm.css'; -import { DisposableCollection, GitpodServer, HEADLESS_LOG_STREAM_STATUS_CODE_REGEX } from '@gitpod/gitpod-protocol'; - -export interface WorkspaceLogsProps { - logsEmitter: EventEmitter; - errorMessage?: string; -} - -export interface WorkspaceLogsState { -} - -export default class WorkspaceLogs extends React.Component { - protected xTermParentRef: React.RefObject; - protected terminal: Terminal | undefined; - protected fitAddon: FitAddon | undefined; - - constructor(props: WorkspaceLogsProps) { - super(props); - this.xTermParentRef = React.createRef(); - } - - private readonly toDispose = new DisposableCollection(); - componentDidMount() { - const element = this.xTermParentRef.current; - if (element === null) { - return; - } - const theme: ITheme = {}; - const options: ITerminalOptions = { - cursorBlink: false, - disableStdin: true, - fontSize: 14, - theme, - scrollback: 9999999, - }; - this.terminal = new Terminal(options); - this.fitAddon = new FitAddon(); - this.terminal.loadAddon(this.fitAddon); - this.terminal.open(element); - this.props.logsEmitter.on('logs', logs => { - if (this.fitAddon && this.terminal && logs) { - this.terminal.write(logs); - } - }); - this.toDispose.push(this.terminal); - this.fitAddon.fit(); - - // Fit terminal on window resize (debounced) - let timeout: NodeJS.Timeout | undefined; - const onWindowResize = () => { - clearTimeout(timeout!); - timeout = setTimeout(() => this.fitAddon!.fit(), 20); - }; - window.addEventListener('resize', onWindowResize); - this.toDispose.push({ - dispose: () => { - clearTimeout(timeout!); - window.removeEventListener('resize', onWindowResize); - } - }); - } - - componentDidUpdate() { - if (this.terminal && this.props.errorMessage) { - this.terminal.write(`\n\u001b[38;5;196m${this.props.errorMessage}\u001b[0m`); - } - } - - componentWillUnmount() { - this.toDispose.dispose(); - } - - render() { - return
-
-
; - } -} - -export function watchHeadlessLogs(server: GitpodServer, instanceId: string, onLog: (chunk: string) => void, checkIsDone: () => Promise): DisposableCollection { - const disposables = new DisposableCollection(); - - const startWatchingLogs = async () => { - if (await checkIsDone()) { - return; - } - - const retry = async (reason: string, err?: Error) => { - console.debug("re-trying headless-logs because: " + reason, err); - await new Promise((resolve) => { - setTimeout(resolve, 2000); - }); - startWatchingLogs().catch(console.error); - }; - - let response: Response | undefined = undefined; - let reader: ReadableStreamDefaultReader | undefined = undefined; - try { - const logSources = await server.getHeadlessLog(instanceId); - // TODO(gpl) Only listening on first stream for now - const streamIds = Object.keys(logSources.streams); - if (streamIds.length < 1) { - await retry("no streams"); - return; - } - - const streamUrl = logSources.streams[streamIds[0]]; - console.log("fetching from streamUrl: " + streamUrl); - response = await fetch(streamUrl, { - method: 'GET', - cache: 'no-cache', - credentials: 'include', - keepalive: true, - headers: { - 'TE': 'trailers', // necessary to receive stream status code - }, - }); - reader = response.body?.getReader(); - if (!reader) { - await retry("no reader"); - return; - } - disposables.push({ dispose: () => reader?.cancel() }); - - const decoder = new TextDecoder('utf-8'); - let chunk = await reader.read(); - while (!chunk.done) { - const msg = decoder.decode(chunk.value, { stream: true }); - - // In an ideal world, we'd use res.addTrailers()/response.trailer here. But despite being introduced with HTTP/1.1 in 1999, trailers are not supported by popular proxies (nginx, for example). - // So we resort to this hand-written solution: - const matches = msg.match(HEADLESS_LOG_STREAM_STATUS_CODE_REGEX); - if (matches) { - if (matches.length < 2) { - console.debug("error parsing log stream status code. msg: " + msg); - } else { - const streamStatusCode = matches[1]; - if (streamStatusCode !== "200") { - throw new Error("received status code: " + streamStatusCode); - } - } - } else { - onLog(msg); - } - - chunk = await reader.read(); - } - reader.cancel() - - if (await checkIsDone()) { - return; - } - } catch(err) { - reader?.cancel().catch(console.debug); - await retry("error while listening to stream", err); - } - }; - startWatchingLogs().catch(console.error); - - return disposables; -} \ No newline at end of file diff --git a/components/gitpod-db/src/project-db.ts b/components/gitpod-db/src/project-db.ts index 09261a5bc32d08..4fbf9df8b2d15c 100644 --- a/components/gitpod-db/src/project-db.ts +++ b/components/gitpod-db/src/project-db.ts @@ -4,12 +4,14 @@ * See License.enterprise.txt in the project root folder. */ -import { Project } from "@gitpod/gitpod-protocol"; +import { Project, ProjectConfig } from "@gitpod/gitpod-protocol"; export const ProjectDB = Symbol('ProjectDB'); export interface ProjectDB { + findProjectById(projectId: string): Promise; findProjectByCloneUrl(cloneUrl: string): Promise; findProjectByInstallationId(installationId: string): Promise; findProjectsByTeam(teamId: string): Promise; storeProject(project: Project): Promise; + setProjectConfiguration(projectId: string, config: ProjectConfig): Promise; } \ No newline at end of file diff --git a/components/gitpod-db/src/typeorm/entity/db-project.ts b/components/gitpod-db/src/typeorm/entity/db-project.ts index ec7768eb44ac61..101f5d68d67b05 100644 --- a/components/gitpod-db/src/typeorm/entity/db-project.ts +++ b/components/gitpod-db/src/typeorm/entity/db-project.ts @@ -6,6 +6,7 @@ import { Entity, Column, PrimaryColumn, Index } from "typeorm"; import { TypeORM } from "../typeorm"; +import { ProjectConfig } from "@gitpod/gitpod-protocol"; @Entity() // on DB but not Typeorm: @Index("ind_lastModified", ["_lastModified"]) // DBSync @@ -26,6 +27,9 @@ export class DBProject { @Column() appInstallationId: string; + @Column("simple-json", { nullable: true }) + config?: ProjectConfig; + @Column("varchar") creationTime: string; diff --git a/components/gitpod-db/src/typeorm/migration/1626853418369-ProjectConfiguration.ts b/components/gitpod-db/src/typeorm/migration/1626853418369-ProjectConfiguration.ts new file mode 100644 index 00000000000000..f3b54882ec793d --- /dev/null +++ b/components/gitpod-db/src/typeorm/migration/1626853418369-ProjectConfiguration.ts @@ -0,0 +1,21 @@ +/** + * Copyright (c) 2021 Gitpod GmbH. All rights reserved. + * Licensed under the GNU Affero General Public License (AGPL). + * See License-AGPL.txt in the project root for license information. + */ + +import { MigrationInterface, QueryRunner } from "typeorm"; +import { columnExists } from "./helper/helper"; + +export class ProjectConfiguration1626853418369 implements MigrationInterface { + + public async up(queryRunner: QueryRunner): Promise { + if (!(await columnExists(queryRunner, "d_b_project", "config"))) { + await queryRunner.query("ALTER TABLE d_b_project ADD COLUMN config text"); + } + } + + public async down(queryRunner: QueryRunner): Promise { + } + +} diff --git a/components/gitpod-db/src/typeorm/project-db-impl.ts b/components/gitpod-db/src/typeorm/project-db-impl.ts index ad36bb2fd6ebd7..9581b004a5526d 100644 --- a/components/gitpod-db/src/typeorm/project-db-impl.ts +++ b/components/gitpod-db/src/typeorm/project-db-impl.ts @@ -9,7 +9,7 @@ import { TypeORM } from "./typeorm"; import { Repository } from "typeorm"; import { ProjectDB } from "../project-db"; import { DBProject } from "./entity/db-project"; -import { Project } from "@gitpod/gitpod-protocol"; +import { Project, ProjectConfig } from "@gitpod/gitpod-protocol"; @injectable() export class ProjectDBImpl implements ProjectDB { @@ -23,12 +23,17 @@ export class ProjectDBImpl implements ProjectDB { return (await this.getEntityManager()).getRepository(DBProject); } - async findProjectByCloneUrl(cloneUrl: string): Promise { + public async findProjectById(projectId: string): Promise { + const repo = await this.getRepo(); + return repo.findOne({ id: projectId }); + } + + public async findProjectByCloneUrl(cloneUrl: string): Promise { const repo = await this.getRepo(); return repo.findOne({ cloneUrl }); } - async findProjectByInstallationId(appInstallationId: string): Promise { + public async findProjectByInstallationId(appInstallationId: string): Promise { const repo = await this.getRepo(); return repo.findOne({ appInstallationId }); } @@ -42,4 +47,14 @@ export class ProjectDBImpl implements ProjectDB { const repo = await this.getRepo(); return repo.save(project); } + + public async setProjectConfiguration(projectId: string, config: ProjectConfig): Promise { + const repo = await this.getRepo(); + const project = await repo.findOne({ id: projectId, deleted: false }); + if (!project) { + throw new Error('A project with this ID could not be found'); + } + project.config = config; + await repo.save(project); + } } diff --git a/components/gitpod-protocol/src/gitpod-service.ts b/components/gitpod-protocol/src/gitpod-service.ts index bd6ca031f89221..68bbd85ff5eb91 100644 --- a/components/gitpod-protocol/src/gitpod-service.ts +++ b/components/gitpod-protocol/src/gitpod-service.ts @@ -107,13 +107,13 @@ export interface GitpodServer extends JsonRpcServer, AdminServer, getUserStorageResource(options: GitpodServer.GetUserStorageResourceOptions): Promise; updateUserStorageResource(options: GitpodServer.UpdateUserStorageResourceOptions): Promise; - // user env vars + // User env vars getEnvVars(): Promise; getAllEnvVars(): Promise; setEnvVar(variable: UserEnvVarValue): Promise; deleteEnvVar(variable: UserEnvVarValue): Promise; - // teams and projects + // Teams getTeams(): Promise; getTeamMembers(teamId: string): Promise; createTeam(name: string): Promise; @@ -123,6 +123,14 @@ export interface GitpodServer extends JsonRpcServer, AdminServer, getGenericInvite(teamId: string): Promise; resetGenericInvite(inviteId: string): Promise; + // Projects + getProviderRepositoriesForUser(params: GetProviderRepositoriesParams): Promise; + createProject(params: CreateProjectParams): Promise; + getProjects(teamId: string): Promise; + getPrebuilds(teamId: string, projectId: string): Promise; + setProjectConfiguration(projectId: string, configString: string): Promise; + fetchProjectRepositoryConfiguration(projectId: string): Promise; + // content service getContentBlobUploadUrl(name: string): Promise getContentBlobDownloadUrl(name: string): Promise @@ -209,14 +217,6 @@ export interface GitpodServer extends JsonRpcServer, AdminServer, getGithubUpgradeUrls(): Promise; - /** - * projects - */ - getProviderRepositoriesForUser(params: GetProviderRepositoriesParams): Promise; - createProject(params: CreateProjectParams): Promise; - getProjects(teamId: string): Promise; - getPrebuilds(teamId: string, project: string): Promise; - /** * Analytics */ diff --git a/components/gitpod-protocol/src/protocol.ts b/components/gitpod-protocol/src/protocol.ts index b54083e1a82fb7..66afe2cb7c3db0 100644 --- a/components/gitpod-protocol/src/protocol.ts +++ b/components/gitpod-protocol/src/protocol.ts @@ -563,12 +563,13 @@ export interface WorkspaceConfig { * Where the config object originates from. * * repo - from the repository + * project-db - from the "Project" stored in the database * definitly-gp - from github.com/gitpod-io/definitely-gp * derived - computed based on analyzing the repository * additional-content - config comes from additional content, usually provided through the project's configuration * default - our static catch-all default config */ - _origin?: 'repo' | 'definitely-gp' | 'derived' | 'additional-content' | 'default'; + _origin?: 'repo' | 'project-db' | 'definitely-gp' | 'derived' | 'additional-content' | 'default'; /** * Set of automatically infered feature flags. That's not something the user can set, but @@ -913,7 +914,7 @@ export interface Commit { refType?: RefType } -export interface AdditionalContentContext { +export interface AdditionalContentContext extends WorkspaceContext { /** * utf-8 encoded contents that will be copied on top of the workspace's filesystem diff --git a/components/gitpod-protocol/src/teams-projects-protocol.ts b/components/gitpod-protocol/src/teams-projects-protocol.ts index 15a1967312a19d..37b2f9e84867ac 100644 --- a/components/gitpod-protocol/src/teams-projects-protocol.ts +++ b/components/gitpod-protocol/src/teams-projects-protocol.ts @@ -7,12 +7,17 @@ import { PrebuiltWorkspaceState } from "./protocol"; import uuidv4 = require("uuid/v4"); +export interface ProjectConfig { + '.gitpod.yml': string; +} + export interface Project { id: string; name: string; cloneUrl: string; teamId: string; appInstallationId: string; + config?: ProjectConfig; creationTime: string; /** This is a flag that triggers the HARD DELETION of this entity */ deleted?: boolean; diff --git a/components/server/ee/src/workspace/gitpod-server-impl.ts b/components/server/ee/src/workspace/gitpod-server-impl.ts index 0ae8c806cf206d..bdcd165c832ab6 100644 --- a/components/server/ee/src/workspace/gitpod-server-impl.ts +++ b/components/server/ee/src/workspace/gitpod-server-impl.ts @@ -1446,8 +1446,7 @@ export class GitpodServerEEImpl extends GitpodServerImpl { const user = this.checkAndBlockUser("getProviderRepositoriesForUser"); @@ -1457,11 +1456,14 @@ export class GitpodServerEEImpl extends GitpodServerImpl { this.checkAndBlockUser("getPrebuilds"); + const span = opentracing.globalTracer().startSpan("getPrebuilds"); + span.setTag("teamId", teamId); + span.setTag("projectName", projectName); const result: PrebuildInfo[] = []; const project = (await this.projectDB.findProjectsByTeam(teamId)).find(p => p.name === projectName); if (project) { - const pwss = await this.workspaceDb.trace({}).findPrebuiltWorkspacesByProject(project.id); + const pwss = await this.workspaceDb.trace({ span }).findPrebuiltWorkspacesByProject(project.id); for (const pws of pwss) { result.push({ @@ -1479,6 +1481,4 @@ export class GitpodServerEEImpl extends GitpodServerImpl { @@ -140,13 +142,24 @@ export class ConfigProvider { } const services = hostContext.services; const contextRepoConfig = services.fileProvider.getGitpodFileContent(commit, user); + const projectDBConfig = this.projectDB.findProjectByCloneUrl(commit.repository.cloneUrl).then(project => project?.config); const definitelyGpConfig = this.fetchExternalGitpodFileContent({ span }, commit.repository); const inferredConfig = this.inferingConfigProvider.infer(user, commit); let customConfigString = await contextRepoConfig; + let fromProjectDB = false; + if (!customConfigString) { + // We haven't found a Gitpod configuration file in the context repo - check the "Project" in the DB. + const config = await projectDBConfig; + if (config) { + customConfigString = config['.gitpod.yml']; + fromProjectDB = true; + } + } + let fromDefinitelyGp = false; if (!customConfigString) { - /* We haven't found a Gitpod configuration file in the context repo - check definitely-gp. + /* We haven't found a Gitpod configuration file in the context repo or "Project" - check definitely-gp. * * In case we had found a config file here, we'd still be checking the definitely GP repo, just to save some time. * While all those checks will be in vain, they should not leak memory either as they'll simply @@ -169,7 +182,7 @@ export class ConfigProvider { log.info(logContext, err.message, { repoCloneUrl: commit.repository.cloneUrl, revision: commit.revision, customConfigString }); throw err; } - customConfig._origin = fromDefinitelyGp ? 'definitely-gp' : 'repo'; + customConfig._origin = fromProjectDB ? 'project-db' : (fromDefinitelyGp ? 'definitely-gp' : 'repo'); } else { /* There is no configuration for this repository. Before we fall back to the default config, * let's see if there is language specific configuration we might want to apply. diff --git a/components/server/src/workspace/gitpod-server-impl.ts b/components/server/src/workspace/gitpod-server-impl.ts index 47fdb0bf4c95e8..46622c5b0e388d 100644 --- a/components/server/src/workspace/gitpod-server-impl.ts +++ b/components/server/src/workspace/gitpod-server-impl.ts @@ -11,6 +11,7 @@ import { AuthProviderEntry, AuthProviderInfo, Branding, CommitContext, Configura import { AccountStatement } from "@gitpod/gitpod-protocol/lib/accounting-protocol"; import { AdminBlockUserRequest, AdminGetListRequest, AdminGetListResult, AdminGetWorkspacesRequest, AdminModifyPermanentWorkspaceFeatureFlagRequest, AdminModifyRoleOrPermissionRequest, WorkspaceAndInstance } from '@gitpod/gitpod-protocol/lib/admin-protocol'; import { GetLicenseInfoResult, LicenseFeature, LicenseValidationResult } from '@gitpod/gitpod-protocol/lib/license-protocol'; +import { GitpodFileParser } from '@gitpod/gitpod-protocol/lib/gitpod-file-parser'; import { ErrorCodes } from '@gitpod/gitpod-protocol/lib/messaging/error'; import { GithubUpgradeURL, PlanCoupon } from "@gitpod/gitpod-protocol/lib/payment-protocol"; import { TeamSubscription, TeamSubscriptionSlot, TeamSubscriptionSlotResolved } from "@gitpod/gitpod-protocol/lib/team-subscription-protocol"; @@ -61,6 +62,7 @@ export class GitpodServerImpl { // Note: this operation is per-user only, hence needs no resource guard - const user = this.checkUser("setEnvVar"); + const user = this.checkAndBlockUser("setEnvVar"); const userId = user.id; variable.repositoryPattern = UserEnvVar.normalizeRepoPattern(variable.repositoryPattern); @@ -1394,7 +1396,7 @@ export class GitpodServerImpl { // Note: this operation is per-user only, hence needs no resource guard - const user = this.checkUser("deleteEnvVar"); + const user = this.checkAndBlockUser("deleteEnvVar"); const userId = user.id; if (!variable.id && variable.name && variable.repositoryPattern) { @@ -1438,12 +1440,13 @@ export class GitpodServerImpl { // Note: this operation is per-user only, hence needs no resource guard - const user = this.checkUser("createTeam"); + const user = this.checkAndBlockUser("createTeam"); return this.teamDB.createTeam(user.id, name); } public async joinTeam(inviteId: string): Promise { - const user = this.checkUser("joinTeam"); + const user = this.checkAndBlockUser("joinTeam"); + // Invites can be used by anyone, as long as they know the invite ID, hence needs no resource guard const invite = await this.teamDB.findTeamMembershipInviteById(inviteId); if (!invite || invite.invalidationTime !== '') { throw new ResponseError(ErrorCodes.NOT_FOUND, "The invite link is no longer valid."); @@ -1454,13 +1457,13 @@ export class GitpodServerImpl { - this.checkUser("setTeamMemberRole"); + this.checkAndBlockUser("setTeamMemberRole"); await this.guardTeamOperation(teamId, "update"); await this.teamDB.setTeamMemberRole(userId, teamId, role); } public async removeTeamMember(teamId: string, userId: string): Promise { - const user = this.checkUser("removeTeamMember"); + const user = this.checkAndBlockUser("removeTeamMember"); // Users are free to leave any team themselves, but only owners can remove others from their teams. await this.guardTeamOperation(teamId, user.id === userId ? "get" : "update"); await this.teamDB.removeMemberFromTeam(userId, teamId); @@ -1477,7 +1480,7 @@ export class GitpodServerImpl { - this.checkUser("resetGenericInvite"); + this.checkAndBlockUser("resetGenericInvite"); await this.guardTeamOperation(teamId, "update"); return this.teamDB.resetGenericInvite(teamId); } @@ -1491,6 +1494,71 @@ export class GitpodServerImpl { + this.checkUser("getProviderRepositoriesForUser"); + // Note: this operation is per-user only, hence needs no resource guard + return []; + } + + public async createProject(params: CreateProjectParams): Promise { + this.checkAndBlockUser("createProject"); + const { name, cloneUrl, teamId, appInstallationId } = params; + // Anyone who can read a team's information (i.e. any team member) can create a new project. + await this.guardTeamOperation(teamId, "get"); + return this.projectDB.storeProject(Project.create({name, cloneUrl, teamId, appInstallationId})); + } + + public async getProjects(teamId: string): Promise { + this.checkUser("getProjects"); + await this.guardTeamOperation(teamId, "get"); + return this.projectDB.findProjectsByTeam(teamId); + } + + public async getPrebuilds(teamId: string, projectId: string): Promise { + this.checkUser("getPrebuilds"); + await this.guardTeamOperation(teamId, "get"); + return []; + } + + public async setProjectConfiguration(projectId: string, configString: string): Promise { + this.checkAndBlockUser("setProjectConfiguration"); + const project = await this.projectDB.findProjectById(projectId); + if (!project) { + throw new ResponseError(ErrorCodes.NOT_FOUND, "Project not found"); + } + // Anyone who can read a team's information (i.e. any team member) can (re-)configure a project. + await this.guardTeamOperation(project.teamId, "get"); + const parseResult = this.gitpodParser.parse(configString); + if (parseResult.validationErrors) { + throw new Error(`This configuration could not be parsed: ${parseResult.validationErrors.join(', ')}`); + } + await this.projectDB.setProjectConfiguration(projectId, { '.gitpod.yml': configString }); + } + + public async fetchProjectRepositoryConfiguration(projectId: string): Promise { + const user = this.checkUser("fetchProjectRepositoryConfiguration"); + const span = opentracing.globalTracer().startSpan("fetchProjectRepositoryConfiguration"); + span.setTag("projectId", projectId); + + const project = await this.projectDB.findProjectById(projectId); + if (!project) { + throw new ResponseError(ErrorCodes.NOT_FOUND, "Project not found"); + } + // Anyone who can read a team's information (i.e. any team member) can (re-)configure a project. + await this.guardTeamOperation(project.teamId, "get"); + + const normalizedContextUrl = this.contextParser.normalizeContextURL(project.cloneUrl); + const context = (await this.contextParser.handle({ span }, user, normalizedContextUrl)) as CommitContext; + const { host } = context.repository; + const hostContext = this.hostContextProvider.get(host); + if (!hostContext || !hostContext.services) { + throw new Error(`Cannot fetch repository configuration for host: ${host}`); + } + const repoHost = hostContext.services; + const configString = await repoHost.fileProvider.getGitpodFileContent(context, user); + return configString; + } + public async getContentBlobUploadUrl(name: string): Promise { const user = this.checkAndBlockUser("getContentBlobUploadUrl"); await this.guardAccess({ kind: "contentBlob", name: name, userID: user.id }, "create"); @@ -1906,35 +1974,4 @@ export class GitpodServerImpl { - this.checkAndBlockUser("getProviderRepositoriesForUser"); - return []; - } - public async createProject(params: CreateProjectParams): Promise { - this.checkUser("createProject"); - const { name, cloneUrl, teamId, appInstallationId } = params; - // Anyone who can read a team's information (i.e. any team member) can create a new project. - await this.guardTeamOperation(teamId, "get"); - return this.projectDB.storeProject(Project.create({name, cloneUrl, teamId, appInstallationId})); - } - public async getProjects(teamId: string): Promise { - this.checkUser("getProjects"); - await this.guardTeamOperation(teamId, "get"); - const result: Project[] = []; - const toProjectInfo = (p: Project) => ({ ...p }); - result.push(...(await this.projectDB.findProjectsByTeam(teamId)).map(toProjectInfo)); - return result; - } - public async getPrebuilds(teamId: string, project: string): Promise { - this.checkAndBlockUser("getPrebuilds"); - await this.guardTeamOperation(teamId, "get"); - return []; - } - // - //#endregion } diff --git a/components/server/src/workspace/workspace-factory.ts b/components/server/src/workspace/workspace-factory.ts index ba7a5aca298296..10d7f8ca0398dd 100644 --- a/components/server/src/workspace/workspace-factory.ts +++ b/components/server/src/workspace/workspace-factory.ts @@ -5,7 +5,7 @@ */ import { DBWithTracing, TracedWorkspaceDB, WorkspaceDB, ProjectDB } from '@gitpod/gitpod-db/lib'; -import { CommitContext, IssueContext, PullRequestContext, Repository, SnapshotContext, User, Workspace, WorkspaceConfig, WorkspaceContext, WorkspaceProbeContext } from '@gitpod/gitpod-protocol'; +import { AdditionalContentContext, CommitContext, IssueContext, PullRequestContext, Repository, SnapshotContext, User, Workspace, WorkspaceConfig, WorkspaceContext, WorkspaceProbeContext } from '@gitpod/gitpod-protocol'; import { ErrorCodes } from '@gitpod/gitpod-protocol/lib/messaging/error'; import { generateWorkspaceID } from '@gitpod/gitpod-protocol/lib/util/generate-workspace-id'; import { log } from '@gitpod/gitpod-protocol/lib/util/logging'; @@ -136,11 +136,21 @@ export class WorkspaceFactory { const span = TraceContext.startSpan("createForCommit", ctx); try { + // TODO(janx): We potentially fetch the same Project twice in this flow (once here, and once in `configProvider`, + // to parse a potential custom config from the Project DB). It would be cool to fetch the Project only once (and + // e.g. pass it to `configProvider.fetchConfig` here). const [ config, project ] = await Promise.all([ this.configProvider.fetchConfig({ span }, user, context), this.projectDB.findProjectByCloneUrl(context.repository.cloneUrl), ]); const imageSource = await this.imageSourceProvider.getImageSource(ctx, user, context, config); + if (config._origin === 'project-db') { + // If the project is configured via the Project DB, place the uncommitted configuration into the workspace, + // thus encouraging Git-based configurations. + if (project?.config) { + (context as any as AdditionalContentContext).additionalFiles = { ...project.config }; + } + } const id = await generateWorkspaceID(); const newWs: Workspace = { diff --git a/yarn.lock b/yarn.lock index 3a4fe85d937bac..8b90ec0ebd1246 100644 --- a/yarn.lock +++ b/yarn.lock @@ -14839,6 +14839,11 @@ moment@^2.6.0: version "2.22.2" resolved "https://registry.yarnpkg.com/moment/-/moment-2.22.2.tgz#3c257f9839fc0e93ff53149632239eb90783ff66" +monaco-editor@^0.25.2: + version "0.25.2" + resolved "https://registry.yarnpkg.com/monaco-editor/-/monaco-editor-0.25.2.tgz#119e2b15bbd968a1a99c03cac9c329316d7c37e9" + integrity sha512-5iylzSJevCnzJn9UVsW8yOZ3yHjmAs4TfvH3zsbftKiFKmHG0xirGN6DK9Kk04VSWxYCZZAIafYJoNJJMAU1KA== + move-concurrently@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/move-concurrently/-/move-concurrently-1.0.1.tgz#be2c005fda32e0b29af1f05d7c4b33214c701f92"