[prebuilds] Introduce 'failed' state for prebuilds

Distinguish between failed tasks but finished prebuilds
and fully failed prebuilds (no snapshot)

fixes #8592

components/dashboard/src/projects/Prebuilds.tsx

@@ -103,7 +103,7 @@ export default function (props: { project?: Project, isAdminDashboard?: boolean
     }, [prebuilds])
 
     const prebuildContextMenu = (p: PrebuildWithStatus) => {
-        const isFailed = p.status === "aborted" || p.status === "timeout" || !!p.error;
+        const isFailed = p.status === "aborted" || p.status === "timeout" || p.status === "failed"|| !!p.error;
         const isRunning = p.status === "building";
         const entries: ContextMenuEntry[] = [];
         if (isFailed) {
@@ -248,8 +248,10 @@ export function prebuildStatusLabel(prebuild?: PrebuildWithStatus) {
             return (<span className="font-medium text-blue-500 uppercase">running</span>);
         case "aborted":
             return (<span className="font-medium text-gray-500 uppercase">canceled</span>);
+        case "failed":
+            return (<span className="font-medium text-red-500 uppercase">system error</span>);
         case "timeout":
-            return (<span className="font-medium text-red-500 uppercase">failed</span>);
+            return (<span className="font-medium text-red-500 uppercase">timed out</span>);
         case "available":
             if (prebuild?.error) {
                 return (<span className="font-medium text-red-500 uppercase">failed</span>);
@@ -267,6 +269,8 @@ export function prebuildStatusIcon(prebuild?: PrebuildWithStatus) {
             return <img alt="" className="h-4 w-4" src={StatusRunning} />;
         case "aborted":
             return <img alt="" className="h-4 w-4" src={StatusCanceled} />;
+        case "failed":
+            return <img alt="" className="h-4 w-4" src={StatusFailed} />;
         case "timeout":
             return <img alt="" className="h-4 w-4" src={StatusFailed} />;
         case "available":

components/dashboard/src/projects/Project.tsx

@@ -250,7 +250,7 @@ export default function () {
                                 <a href={gitpodHostUrl.withContext(`${branch.url}`).toString()}>
                                     <button className={`primary mr-2 py-2 opacity-0 group-hover:opacity-100`}>New Workspace</button>
                                 </a>
-                                <ItemFieldContextMenu className="py-0.5" menuEntries={(!prebuild || prebuild.status === 'aborted' || prebuild.status === 'timeout' || !!prebuild.error)
+                                <ItemFieldContextMenu className="py-0.5" menuEntries={(!prebuild || prebuild.status === 'aborted' || prebuild.status === 'failed' || prebuild.status === 'timeout' || !!prebuild.error)
                                     ? [{
                                         title: `${prebuild ? 'Rerun' : 'Run'} Prebuild (${branch.name})`,
                                         onClick: () => triggerPrebuild(branch),

components/gitpod-protocol/src/headless-workspace-log.ts

@@ -11,6 +11,7 @@ export enum HeadlessWorkspaceEventType {
     FinishedButFailed = "finish-fail",
     AbortedTimedOut = "aborted-timeout",
     Aborted = "aborted",
+    Failed = "failed",
     Started = "started"
 }
 export namespace HeadlessWorkspaceEventType {

components/gitpod-protocol/src/protocol.ts

@@ -645,12 +645,14 @@ export type PrebuiltWorkspaceState
     = "queued"
     // the workspace prebuild is currently running (i.e. there's a workspace pod deployed)
     | "building"
-    // the prebuild failed due to some issue with the system (e.g. missed a message, could not start workspace)
+    // the prebuild was aborted
     | "aborted"
     // the prebuild timed out
     | "timeout"
-    // the prebuild has finished and a snapshot is available
-    | "available";
+    // the prebuild has finished (even if a headless task failed) and a snapshot is available
+    | "available"
+    // the prebuild (headless workspace) failed due to some system error
+    | "failed";
 
 export interface PrebuiltWorkspace {
     id: string;

components/server/ee/src/prebuilds/prebuild-manager.ts

@@ -72,7 +72,7 @@ export class PrebuildManager {
             }
             const existingPB = await this.workspaceDB.trace({ span }).findPrebuiltWorkspaceByCommit(cloneURL, commit);
             // If the existing prebuild is failed, we want to retrigger it.
-            if (!!existingPB && existingPB.state !== 'aborted' && existingPB.state !== 'timeout' && !existingPB.error) {
+            if (!!existingPB && existingPB.state !== 'aborted' && existingPB.state !== 'failed' && existingPB.state !== 'timeout') {
                 // If the existing prebuild is based on an outdated project config, we also want to retrigger it.
                 const existingPBWS = await this.workspaceDB.trace({ span }).findById(existingPB.buildWorkspaceId);
                 const existingConfig = existingPBWS?.config;

components/server/ee/src/prebuilds/prebuilt-status-maintainer.ts

@@ -102,16 +102,18 @@ export class PrebuildStatusMaintainer implements Disposable {
     protected getConclusionFromPrebuildState(pws: PrebuiltWorkspace): "error" | "failure" | "pending" | "success" {
         if (pws.state === "aborted") {
             return "error";
-        } else if (pws.state === "queued") {
-            return "pending";
+        } else if (pws.state === "failed") {
+            return "error";
         } else if (pws.state === "timeout") {
             return "error";
+        } else if (pws.state === "queued") {
+            return "pending";
+        } else if (pws.state === "building") {
+            return "pending";
         } else if (pws.state === "available" && !pws.error) {
             return "success";
         } else if (pws.state === "available" && !!pws.error) {
             return "failure";
-        } else if (pws.state === "building") {
-            return "pending";
         } else {
             log.warn("Should have updated prebuilt workspace updatable, but don't know how. Resorting to error conclusion.", { pws });
             return "error";

components/server/src/workspace/workspace-starter.ts

@@ -292,13 +292,13 @@ export class WorkspaceStarter {
             // If we just attempted to start a workspace for a prebuild - and that failed, we have to fail the prebuild itself.
             if (workspace.type === 'prebuild') {
                 const prebuild = await this.workspaceDb.trace({ span }).findPrebuildByWorkspaceID(workspace.id);
-                if (prebuild && prebuild.state !== 'aborted') {
-                    prebuild.state = "aborted";
+                if (prebuild && prebuild.state !== 'failed') {
+                    prebuild.state = "failed";
                     prebuild.error = err.toString();
 
                     await this.workspaceDb.trace({ span }).storePrebuiltWorkspace(prebuild)
                     await this.messageBus.notifyHeadlessUpdate({ span }, workspace.ownerId, workspace.id, <HeadlessWorkspaceEvent>{
-                        type: HeadlessWorkspaceEventType.Aborted,
+                        type: HeadlessWorkspaceEventType.Failed,
                         // TODO: `workspaceID: workspace.id` not needed here? (found in ee/src/prebuilds/prebuild-queue-maintainer.ts and ee/src/bridge.ts)
                     });
                 }

components/ws-manager-bridge/ee/src/bridge.ts

@@ -77,18 +77,22 @@ export class WorkspaceManagerBridgeEE extends WorkspaceManagerBridge {
                     prebuild.error = status.conditions!.timeout;
                     headlessUpdateType = HeadlessWorkspaceEventType.AbortedTimedOut;
                 } else if (!!status.conditions!.failed) {
-                    prebuild.state = "aborted";
+                    prebuild.state = "failed";
                     prebuild.error = status.conditions!.failed;
-                    headlessUpdateType = HeadlessWorkspaceEventType.Aborted;
+                    headlessUpdateType = HeadlessWorkspaceEventType.Failed;
                 } else if (!!status.conditions!.stoppedByRequest) {
                     prebuild.state = "aborted";
                     prebuild.error = "Cancelled";
                     headlessUpdateType = HeadlessWorkspaceEventType.Aborted;
                 } else if (!!status.conditions!.headlessTaskFailed) {
                     prebuild.state = "available";
-                    prebuild.error = status.conditions!.headlessTaskFailed;
+                    if (status.conditions!.headlessTaskFailed)
+                        prebuild.error = status.conditions!.headlessTaskFailed;
                     prebuild.snapshot = status.conditions!.snapshot;
                     headlessUpdateType = HeadlessWorkspaceEventType.FinishedButFailed;
+                } else if (!status.conditions!.snapshot) {
+                    prebuild.state = "failed";
+                    headlessUpdateType = HeadlessWorkspaceEventType.Failed;
                 } else {
                     prebuild.state = "available";
                     prebuild.snapshot = status.conditions!.snapshot;