Merge branch 'main' into jetbrains/intellij-213-6777-52

.github/workflows/changelog.yml

@@ -10,7 +10,7 @@ jobs:
 
     runs-on: ubuntu-latest
 
-    container: eu.gcr.io/gitpod-core-dev/dev/changelog:0.0.34
+    container: eu.gcr.io/gitpod-core-dev/dev/changelog:0.0.36
 
     steps:
       - uses: actions/checkout@v2
@@ -24,10 +24,11 @@ jobs:
             git add CHANGELOG.md
             git commit -m "[changelog] updated changelog"
             git push origin $PR_BRANCH
-            /app/changelog pr -t $TOKEN -o gitpod-io -r gitpod -b $GITHUB_REF -H $PR_BRANCH
+            /app/changelog pr -t $TOKEN -a $APPROVAL_TOKEN -o gitpod-io -r gitpod -b $GITHUB_REF -H $PR_BRANCH
           fi
         env:
           GITHUB_USER: roboquat
           GITHUB_EMAIL: [email protected]
           TOKEN: ${{ secrets.ROBOQUAT_AUTOMATIC_CHANGELOG }}
+          APPROVAL_TOKEN: ${{ secrets.GITOPS_BOT_PR_APPROVAL_TOKEN }}
         shell: bash

.werft/build.ts

@@ -86,7 +86,8 @@ const vmSlices = {
     START_KUBECTL_PORT_FORWARDS: 'Start kubectl port forwards',
     COPY_CERT_MANAGER_RESOURCES: 'Copy CertManager resources from core-dev',
     INSTALL_LETS_ENCRYPT_ISSUER: 'Install Lets Encrypt issuer',
-    KUBECONFIG: 'Getting kubeconfig'
+    KUBECONFIG: 'Getting kubeconfig',
+    EXTERNAL_LOGGING: 'Install credentials to send logs from fluent-bit to GCP'
 }
 
 export function parseVersion(context) {
@@ -353,6 +354,10 @@ export async function build(context, version) {
         exec(`kubectl apply -f clouddns-dns01-solver-svc-acct.yaml -f letsencrypt-issuer.yaml`, { slice: vmSlices.INSTALL_LETS_ENCRYPT_ISSUER, dontCheckRc: true })
         werft.done(vmSlices.INSTALL_LETS_ENCRYPT_ISSUER)
 
+        VM.installFluentBit({namespace: 'default', slice: vmSlices.EXTERNAL_LOGGING})
+        werft.done(vmSlices.EXTERNAL_LOGGING)
+
+
         issueMetaCerts(PROXY_SECRET_NAME, "default", domain, withVM)
         werft.done('certificate')
         installMonitoring(deploymentConfig.namespace, 9100, deploymentConfig.domain, STACKDRIVER_SERVICEACCOUNT, true);

.werft/build.yaml

@@ -45,6 +45,9 @@ pod:
   - name: harvester-vm-ssh-keys
     secret:
       secretName: harvester-vm-ssh-keys
+  - name: fluent-bit-external
+    secret:
+      secretName: fluent-bit-external
   # - name: deploy-key
   #   secret:
   #     secretName: deploy-key
@@ -98,6 +101,8 @@ pod:
       mountPath: /mnt/secrets/harvester-kubeconfig
     - name: harvester-vm-ssh-keys
       mountPath: /mnt/secrets/harvester-vm-ssh-keys
+    - name: fluent-bit-external
+      mountPath: /mnt/fluent-bit-external
     # - name: deploy-key
     #   mountPath: /mnt/secrets/deploy-key
     #   readOnly: true
@@ -188,7 +193,7 @@ pod:
         sudo chown gitpod:gitpod $GOCACHE
         export GITHUB_TOKEN=$(echo $GITHUB_TOKEN | xargs)
 
-        export DOCKER_HOST=tcp://$NODENAME:2475
+        export DOCKER_HOST=tcp://$NODENAME:2375
         sudo chown -R gitpod:gitpod /workspace
 
         mkdir /workspace/.ssh

.werft/clean-up-werft-build-nodes.yaml

@@ -36,7 +36,7 @@ pod:
           sleep 1
           set -Eeuo pipefail
 
-          export DOCKER_HOST=tcp://$NODENAME:2475
+          export DOCKER_HOST=tcp://$NODENAME:2375
 
           werft log phase docker-engine-prune "Cleaning up Docker Engine used by Werft builds"
           docker system prune --force --all --filter 'until=72h' 2>&1 | werft log slice docker-engine-prune

.werft/debug.yaml

@@ -153,7 +153,7 @@ pod:
         sudo chown gitpod:gitpod $GOCACHE
         export GITHUB_TOKEN=$(echo $GITHUB_TOKEN | xargs)
 
-        export DOCKER_HOST=tcp://$NODENAME:2475
+        export DOCKER_HOST=tcp://$NODENAME:2375
         sudo chown -R gitpod:gitpod /workspace
 
         (cd .werft && yarn install && mv node_modules ..) | werft log slice prep

.werft/vm/charts/fluentbit/values.yaml

@@ -0,0 +1,20 @@
+config:
+  outputs: |
+    [OUTPUT]
+        Name stackdriver
+        Match *
+
+env:
+  - name: GOOGLE_SERVICE_CREDENTIALS
+    value: /gcp/credentials.json
+
+
+extraVolumes:
+  - name: fluent-bit-external
+    secret:
+      secretName: fluent-bit-external
+      defaultMode: 420
+
+extraVolumeMounts:
+  - name: fluent-bit-external
+    mountPath: /gcp

.werft/vm/vm.ts

@@ -179,3 +179,13 @@ export function startSSHProxy(options: { name: string, slice: string }) {
 export function stopKubectlPortForwards() {
     exec(`sudo killall kubectl || true`)
 }
+
+/**
+ * Install Fluent-Bit sending logs to GCP
+ */
+export function installFluentBit(options: {namespace: string, slice: string}) {
+    exec(`kubectl create secret generic fluent-bit-external --save-config --dry-run=client --from-file=credentials.json=/mnt/fluent-bit-external/credentials.json -o yaml | kubectl apply -n ${options.namespace} -f -`, { slice: options.slice, dontCheckRc: true})
+    exec(`helm3 repo add fluent https://fluent.github.io/helm-charts`, { slice: options.slice, dontCheckRc: true})
+    exec(`helm3 repo update`, { slice: options.slice, dontCheckRc: true})
+    exec(`helm3 upgrade --install fluent-bit fluent/fluent-bit -n ${options.namespace} -f .werft/vm/charts/fluentbit/values.yaml`, { slice: options.slice, dontCheckRc: true})
+}

CHANGELOG.md

@@ -1,6 +1,9 @@
 # Change Log
 
 ## February 2022
+- Fix missing status updates for prebuilds. ([#7968](https://github.com/gitpod-io/gitpod/pull/7968)) - [@AlexTugarev](https://github.com/AlexTugarev)
+- reduce idle DB load on SH installations ([#7940](https://github.com/gitpod-io/gitpod/pull/7940)) - [@geropl](https://github.com/geropl)
+- [gitlab] user-scoped env vars can now be filtered for nested repos on Gitlab ([#7978](https://github.com/gitpod-io/gitpod/pull/7978)) - [@JanKoehnlein](https://github.com/JanKoehnlein)
 - Update PyCharm IDE image to version 213.6777.50. ([#7943](https://github.com/gitpod-io/gitpod/pull/7943)) - [@roboquat](https://github.com/roboquat)
 - Update PhpStorm IDE image to version 213.6777.58. ([#7944](https://github.com/gitpod-io/gitpod/pull/7944)) - [@roboquat](https://github.com/roboquat)
 - [installer]: add jaeger sampling options to the tracing object ([#7727](https://github.com/gitpod-io/gitpod/pull/7727)) - [@MrSimonEmms](https://github.com/MrSimonEmms)

WORKSPACE.yaml

@@ -7,7 +7,7 @@ defaultArgs:
   jbMarketplacePublishTrigger: "false"
   publishToJBMarketplace: true
   localAppVersion: unknown
-  codeCommit: 40c9a1fb052740734bf465b98032f1a50404c042
+  codeCommit: 4035f15a03e6d92cb9815e366b856e43f1caabe7
   intellijDownloadUrl: "https://download.jetbrains.com/idea/ideaIU-2021.3.2.tar.gz"
   golandDownloadUrl: "https://download.jetbrains.com/go/goland-2021.3.2.tar.gz"
   pycharmDownloadUrl: "https://download.jetbrains.com/python/pycharm-professional-2021.3.2.tar.gz"

chart/values.yaml

@@ -356,7 +356,7 @@ components:
       version: "latest"
     codeImage:
       imageName: "ide/code"
-      stableVersion: "commit-dc3be5ab7e8dd88d7b3d9c40fa8a2b2130106c2d"
+      stableVersion: "commit-adaa5e19c625d55704417e59083d0b38a5c0f0f4"
       insidersVersion: "nightly"
     desktopIdeImages:
       codeDesktop:

components/dashboard/src/admin/ProjectDetail.tsx

@@ -30,7 +30,12 @@ export default function ProjectDetail(props: { project: Project, owner: string |
                         </>
                     </Property>
                     :
-                    <Property name="Owner">{`${props.owner} (Team)`}</Property>}
+                    <Property name="Owner">
+                        <>
+                            {props.owner}
+                            <span className="text-gray-400 dark:text-gray-500"> (Team)</span>
+                        </>
+                    </Property>}
             </div>
             <div className="flex w-full mt-6">
                 <Property name="Incremental Prebuilds">{props.project.settings?.useIncrementalPrebuilds ? "Yes" : "No"}</Property>

components/docker-up/docker-up/main.go

@@ -68,6 +68,8 @@ func main() {
 		logger.SetLevel(logrus.DebugLevel)
 	}
 
+	log = logrus.NewEntry(logger)
+
 	listenFD := os.Getenv("LISTEN_FDS") != ""
 	if _, err := os.Stat(dockerSocketFN); !listenFD && (err == nil || !os.IsNotExist(err)) {
 		logger.Fatalf("Docker socket already exists at %s.\nIn a Gitpod workspace Docker will start automatically when used.\nIf all else fails, please remove %s and try again.", dockerSocketFN, dockerSocketFN)

components/gitpod-db/src/periodic-deleter.ts

@@ -10,6 +10,7 @@ import { log } from '@gitpod/gitpod-protocol/lib/util/logging';
 
 import { GitpodTableDescriptionProvider, TableDescription } from "./tables";
 import { TypeORM } from "./typeorm/typeorm";
+import { repeat } from "@gitpod/gitpod-protocol/lib/util/repeat";
 
 @injectable()
 export class PeriodicDbDeleter {
@@ -23,26 +24,22 @@ export class PeriodicDbDeleter {
     }
 
     protected async sync() {
-        await this.doSync();
-        await new Promise(resolve => setTimeout(resolve, 1001));
-        this.sync().catch(err => log.error("[PeriodicDbDeleter] sync failed", err));
-    }
-    protected async doSync() {
-        const sortedTables = this.tableProvider.getSortedTables();
-        const toBeDeleted: { table: string, deletions: string[] }[] = [];
-        for (const table of sortedTables) {
-            toBeDeleted.push(await this.collectRowsToBeDeleted(table));
-        }
-        // when collecting the deletions do so in the inverse order as during update (delete dependency targes first)
-        for (const { deletions } of toBeDeleted.reverse()) {
-            for (const deletion of deletions) {
-                try {
-                    await this.query(deletion);
-                } catch (error) {
-                    log.error(`[PeriodicDbDeleter] sync error`, error);
+        const doSync = async() => {
+            const sortedTables = this.tableProvider.getSortedTables();
+            const toBeDeleted: { table: string, deletions: string[] }[] = [];
+            for (const table of sortedTables) {
+                toBeDeleted.push(await this.collectRowsToBeDeleted(table));
+            }
+            // when collecting the deletions do so in the inverse order as during update (delete dependency targes first)
+            const pendingDeletions: Promise<void>[] = [];
+            for (const { deletions } of toBeDeleted.reverse()) {
+                for (const deletion of deletions) {
+                    pendingDeletions.push(this.query(deletion).catch(err => log.error(`[PeriodicDbDeleter] sync error`, err)));
                 }
             }
-        }
+            await Promise.all(pendingDeletions);
+        };
+        repeat(doSync, 30000);  // deletion is never time-critical, so we should ensure we do not spam ourselves
     }
     protected async collectRowsToBeDeleted(table: TableDescription): Promise<{ table: string, deletions: string[] }> {
         try {

components/gitpod-protocol/src/protocol.ts

@@ -258,8 +258,8 @@ export namespace UserEnvVar {
     // DEPRECATED: Use ProjectEnvVar instead of repositoryPattern - https://github.com/gitpod-com/gitpod/issues/5322
     export function splitRepositoryPattern(repositoryPattern: string): string[] {
         const patterns = repositoryPattern.split('/');
-        const repoPattern = patterns.pop() || "";
-        const ownerPattern = patterns.join('/');
+        const repoPattern = patterns.slice(1).join('/')
+        const ownerPattern = patterns[0];
         return [ownerPattern, repoPattern];
     }
 }

components/ide/code/supervisor-ide-config.json

@@ -1,6 +1,6 @@
 {
     "entrypoint": "/ide/startup.sh",
-    "entrypointArgs": [ "--port", "{IDEPORT}", "--host", "0.0.0.0", "--connection-token", "00000", "--server-data-dir", "/workspace/.vscode-remote" ],
+    "entrypointArgs": [ "--port", "{IDEPORT}", "--host", "0.0.0.0", "--without-connection-token", "--server-data-dir", "/workspace/.vscode-remote" ],
     "readinessProbe": {
         "type": "http",
         "http": {

components/workspacekit/go.mod

@@ -2,7 +2,7 @@ module github.com/gitpod-io/gitpod/workspacekit
 
 go 1.17
 
-replace github.com/seccomp/libseccomp-golang => github.com/kinvolk/libseccomp-golang v0.9.2-0.20201113182948-883917843313
+replace github.com/seccomp/libseccomp-golang => github.com/gitpod-io/libseccomp-golang v0.9.2-0.20220203100026-45179215fdb1
 
 require (
 	github.com/gitpod-io/gitpod/common-go v0.0.0-00010101000000-000000000000

components/workspacekit/go.sum

@@ -64,6 +64,8 @@ github.com/fatih/structtag v1.2.0 h1:/OdNE99OxoI/PqaW/SuSK9uxxT3f/tcSZgon/ssNSx4
 github.com/fatih/structtag v1.2.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4/aAZl94=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
+github.com/gitpod-io/libseccomp-golang v0.9.2-0.20220203100026-45179215fdb1 h1:Z97P8/nLkIJf7uKQ1hjz3MdIfZyX3t8zWspNFj3Ox5M=
+github.com/gitpod-io/libseccomp-golang v0.9.2-0.20220203100026-45179215fdb1/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=

components/ws-daemon/nsinsider/go.mod

@@ -2,7 +2,7 @@ module github.com/gitpod-io/gitpod/ws-daemon/nsinsider
 
 go 1.17
 
-replace github.com/seccomp/libseccomp-golang => github.com/kinvolk/libseccomp-golang v0.9.2-0.20201113182948-883917843313 // leeway indirect from components/workspacekit:lib
+replace github.com/seccomp/libseccomp-golang => github.com/gitpod-io/libseccomp-golang v0.9.2-0.20220203100026-45179215fdb1 // leeway indirect from components/workspacekit:lib
 
 require (
 	github.com/gitpod-io/gitpod/common-go v0.0.0-00010101000000-000000000000

components/ws-manager-api/typescript/src/client-provider.spec.ts

@@ -54,8 +54,8 @@ class TestClientProvider {
 
     @test
     public async getStartClusterSets() {
-        await this.expectInstallations(["a1", "a2", "a3"], await this.provider.getStartClusterSets({} as User, {} as Workspace, {} as WorkspaceInstance), "default case");
-        this.expectInstallations(["a1", "a2", "a3", "con1"], await this.provider.getStartClusterSets({
+        await this.expectInstallations([["a2", "a3"]], await this.provider.getStartClusterSets({} as User, {} as Workspace, {} as WorkspaceInstance), "default case");
+        await this.expectInstallations([["con1"], ["a2", "a3"]], await this.provider.getStartClusterSets({
             rolesOrPermissions: ["new-workspace-cluster"]
         } as User, {} as Workspace, {} as WorkspaceInstance), "new workspace cluster");
     }
@@ -102,13 +102,27 @@ class TestClientProvider {
         expect(constraintInverseMoreResources(clusters, {getsMoreResources: true} as ExtendedUser, {} as Workspace, {} as WorkspaceInstance).map(c => c.name), "inverse more resources").to.be.eql(["b1"]);
     }
 
-    private async expectInstallations(expected: string[], actual: IWorkspaceClusterStartSet, msg: string) {
+    private async expectInstallations(expectedSets: string[][], actual: IWorkspaceClusterStartSet, msg: string) {
         const a: string[] = [];
         for await (const c of actual) {
             a.push(c.installation);
         }
 
-        expect(a.sort(), msg).to.be.eql(expected);
+        // we check:
+        //  - the order of returned sets
+        //  - identical content of said sets
+        //  - NOT the order of clusters within a set
+        let i = 0;
+        for (; i < expectedSets.length; i++) {
+            const eSet = expectedSets[i];
+            const aSet = a.splice(0, eSet.length);
+            expect(aSet.sort(), `returned set does not match (${msg})`).to.be.eql(eSet.sort());
+        }
+
+        const eOverage = expectedSets.slice(i).reduce((p, c) => [...p, ...c], []);
+        const aOverage = a;
+        expect(eOverage, `missing some cluster(s)/sets (${msg})`).to.be.empty;
+        expect(aOverage, `got too many cluster(s)/sets (${msg})`).to.be.empty;
     }
 
 }

components/ws-manager-api/typescript/src/client-provider.ts

@@ -43,7 +43,7 @@ export class WorkspaceManagerClientProvider implements Disposable {
      */
     public async getStartClusterSets(user: ExtendedUser, workspace: Workspace, instance: WorkspaceInstance): Promise<IWorkspaceClusterStartSet> {
         const allClusters = await this.source.getAllWorkspaceClusters();
-        const availableClusters = allClusters.filter(c => c.score >= 0 && c.state === "available");
+        const availableClusters = allClusters.filter(c => c.score > 0 && c.state === "available");
 
         const sets = workspaceClusterSets.map(constraints => {
             const r = constraints.constraint(availableClusters, user, workspace, instance);
@@ -62,9 +62,9 @@ export class WorkspaceManagerClientProvider implements Disposable {
                                 return {done: true, value: undefined};
                             }
 
-                            let res = await sets[sets.length - 1].next();
+                            let res = await sets[0].next();
                             if (!!res.done) {
-                                sets.pop();
+                                sets.splice(0, 1);
                                 continue;
                             }
 

components/ws-manager-bridge/ee/src/bridge.ts

@@ -114,12 +114,21 @@ export class WorkspaceManagerBridgeEE extends WorkspaceManagerBridge {
         }
     }
 
-    protected async controlPrebuildInstance(instance: WorkspaceInstance): Promise<void> {
+    protected async stopPrebuildInstance(ctx: TraceContext, instance: WorkspaceInstance): Promise<void> {
+        const span = TraceContext.startSpan("stopPrebuildInstance", ctx);
+
         const prebuild = await this.workspaceDB.trace({}).findPrebuildByWorkspaceID(instance.workspaceId);
-        if (prebuild && prebuild.state == 'building') {
+        if (prebuild) {
             // this is a prebuild - set it to aborted
             prebuild.state = 'aborted';
             await this.workspaceDB.trace({}).storePrebuiltWorkspace(prebuild);
+
+            { // notify about prebuild updated
+                const info = (await this.workspaceDB.trace({span}).findPrebuildInfos([prebuild.id]))[0];
+                if (info) {
+                    this.messagebus.notifyOnPrebuildUpdate({ info, status: prebuild.state });
+                }
+            }
         }
     }
 

components/ws-manager-bridge/src/bridge.ts

@@ -312,7 +312,7 @@ export class WorkspaceManagerBridge implements Disposable {
             instance.stoppedTime = new Date().toISOString();
             promises.push(this.workspaceDB.trace({}).storeInstance(instance));
             promises.push(this.onInstanceStopped({}, ri.workspace.ownerId, instance));
-            promises.push(this.controlPrebuildInstance(instance));
+            promises.push(this.stopPrebuildInstance(ctx, instance));
         }
         await Promise.all(promises);
     }
@@ -325,7 +325,7 @@ export class WorkspaceManagerBridge implements Disposable {
         // prebuilds are an EE feature - we just need the hook here
     }
 
-    protected async controlPrebuildInstance(instance: WorkspaceInstance): Promise<void> {
+    protected async stopPrebuildInstance(ctx: TraceContext, instance: WorkspaceInstance): Promise<void> {
         // prebuilds are an EE feature - we just need the hook here
     }