feat: serve airgap samples (#1142)

* feat: serve airgap samples

Signed-off-by: Anatolii Bazko <[email protected]>

build/dockerfiles/Dockerfile

@@ -11,7 +11,8 @@
 FROM docker.io/node:18.19.1-alpine3.19 as builder
 
 # hadolint ignore=DL3018
-RUN apk add --no-cache python3 make g++
+RUN apk add --no-cache python3 py3-pip make g++ jq curl
+RUN pip3 install yq --break-system-packages
 
 # hadolint ignore=DL3059,SC1072
 RUN if ! [ type "yarn" &> /dev/null ]; then \
@@ -40,6 +41,11 @@ RUN yarn build
 # leave only production dependencies
 RUN yarn workspace @eclipse-che/dashboard-backend install --production
 
+# Prepare air-gapped resources
+# ARG GITHUB_TOKEN=$GITHUB_TOKEN
+COPY build/dockerfiles/airgap.sh /dashboard/airgap.sh
+RUN /dashboard/airgap.sh -d /dashboard/packages/devfile-registry/air-gap
+
 FROM docker.io/node:18.19.1-alpine3.19
 
 RUN apk --no-cache add curl

build/dockerfiles/airgap.sh

@@ -0,0 +1,118 @@
+#!/bin/sh
+#
+# Copyright (c) 2021-2024 Red Hat, Inc.
+# This program and the accompanying materials are made
+# available under the terms of the Eclipse Public License 2.0
+# which is available at https://www.eclipse.org/legal/epl-2.0/
+#
+# SPDX-License-Identifier: EPL-2.0
+#
+
+# The script is used to download resources (projects and devfiles)
+# for air-gapped (offline) environments. Only https://github.com is supported for now.
+
+set -e
+
+init() {
+  unset AIRGAP_RESOURCES_DIR
+
+  while [ "$#" -gt 0 ]; do
+    case $1 in
+      '--airgap-resources-dir'|'-d') AIRGAP_RESOURCES_DIR=$2; shift 1;;
+      '--help'|'-h') usage; exit;;
+    esac
+    shift 1
+  done
+
+  [ -z "${AIRGAP_RESOURCES_DIR}" ] && { usage; exit; }
+  SAMPLES_JSON_PATH="${AIRGAP_RESOURCES_DIR}/index.json"
+}
+
+usage() {
+  cat <<EOF
+  Usage: $0 [OPTIONS]
+
+  Options:
+    --airgap-resources-dir, -d  Directory where airgap resources are stored
+    --help, -h                  Show this help message
+EOF
+}
+
+run() {
+  samplesNum=$(jq -r '. | length' "${SAMPLES_JSON_PATH}")
+
+  i=0
+  while [ "${i}" -lt "${samplesNum}" ]; do
+    url=$(jq -r '.['${i}'].url' "${SAMPLES_JSON_PATH}")
+    name=$(jq -r '.['${i}'].displayName' "${SAMPLES_JSON_PATH}")
+    encodedName=$(echo "${name}" | jq -Rr @uri)
+
+    if [ "${url}" != "null" ]; then
+      strippedURL="${url#https://github.com/}"
+      organization="$(echo "${strippedURL}" | cut -d '/' -f 1)"
+      repository="$(echo "${strippedURL}" | cut -d '/' -f 2)"
+      ref="$(echo "${strippedURL}" | cut -d '/' -f 4)"
+
+      if [ -n "${ref}" ]; then
+        archiveFileName="${organization}-${repository}-${ref}.zip"
+        devfileFileName="${organization}-${repository}-${ref}-devfile.yaml"
+        projectDownloadLink="https://api.github.com/repos/${organization}/${repository}/zipball/${ref}"
+        devfileDownloadLink="https://api.github.com/repos/${organization}/${repository}/contents/devfile.yaml?ref=${ref}"
+      else
+        archiveFileName="${organization}-${repository}.zip"
+        devfileFileName="${organization}-${repository}-devfile.yaml"
+        projectDownloadLink="https://api.github.com/repos/${organization}/${repository}/zipball"
+        devfileDownloadLink="https://api.github.com/repos/${organization}/${repository}/contents/devfile.yaml"
+      fi
+
+      echo "[INFO] Downloading ${url} into ${archiveFileName}"
+      processSample \
+        "${archiveFileName}" \
+        "${devfileFileName}" \
+        "${projectDownloadLink}" \
+        "${devfileDownloadLink}" \
+        "${encodedName}" \
+        "${repository}"
+    fi
+
+    i=$((i+1))
+  done
+}
+
+processSample() {
+  archiveFileName=$1
+  devfileFileName=$2
+  projectDownloadLink=$3
+  devfileDownloadLink=$4
+  encodedName=$5
+  repository=$6
+
+  curl -L \
+      -H "Accept: application/vnd.github.raw+json" \
+      -H "X-GitHub-Api-Version: 2022-11-28" \
+      -H "Authorization: token ${GITHUB_TOKEN}" \
+      "${devfileDownloadLink}" \
+      -o "${AIRGAP_RESOURCES_DIR}/${devfileFileName}"
+
+  curl -L \
+      -H "Accept: application/vnd.github+json" \
+      -H "X-GitHub-Api-Version: 2022-11-28" \
+      -H "Authorization: token ${GITHUB_TOKEN}" \
+      "${projectDownloadLink}" \
+      -o "${AIRGAP_RESOURCES_DIR}/${archiveFileName}"
+
+  # CHE_DASHBOARD_INTERNAL_URL is a placeholder that will be replaced
+  # by the actual URL in entrypoint.sh
+  devfileLink="CHE_DASHBOARD_INTERNAL_URL/dashboard/api/airgap-sample/devfile/download?name=${encodedName}"
+  projectLink="CHE_DASHBOARD_INTERNAL_URL/dashboard/api/airgap-sample/project/download?name=${encodedName}"
+
+  echo "$(jq '(.['${i}'].url) = '\"${devfileLink}\" ${SAMPLES_JSON_PATH})" > "${SAMPLES_JSON_PATH}"
+  echo "$(jq '(.['${i}'].project.zip.filename) = '\"${archiveFileName}\" ${SAMPLES_JSON_PATH})" > "${SAMPLES_JSON_PATH}"
+  echo "$(jq '(.['${i}'].devfile.filename) = '\"${devfileFileName}\" ${SAMPLES_JSON_PATH})" > "${SAMPLES_JSON_PATH}"
+
+  # Update the devfile with the project link
+  yq -riY '.projects=[{name: "'${repository}'", zip: {location: "'${projectLink}'"}}]' "${AIRGAP_RESOURCES_DIR}/${devfileFileName}"
+}
+
+init "$@"
+run

build/dockerfiles/entrypoint.sh

@@ -9,7 +9,9 @@
 #
 
 set -e
+
 echo 'Starting Dashboard backend server...'
+find /public/dashboard/devfile-registry/air-gap -type f \( -name "*.json" -o -name "*.yaml" \) -exec sed -i 's|CHE_DASHBOARD_INTERNAL_URL|'${CHE_DASHBOARD_INTERNAL_URL}'|g' {} \;
 start_server="node /backend/server/backend.js --publicFolder /public"
 $start_server &
 wait

build/dockerfiles/rhel.entrypoint.sh

@@ -17,6 +17,7 @@ fi
 set -x
 
 echo 'Starting Dashboard backend server...'
+find /public/dashboard/devfile-registry/air-gap -type f \( -name "*.json" -o -name "*.yaml" \) -exec sed -i 's|CHE_DASHBOARD_INTERNAL_URL|'${CHE_DASHBOARD_INTERNAL_URL}'|g' {} \;
 start_server="node /backend/server/backend.js --publicFolder /public"
 $start_server &
 wait

packages/common/src/dto/api/index.ts

@@ -17,6 +17,7 @@ import {
 } from '@devfile/api';
 import { CoreV1EventList, V1PodList } from '@kubernetes/client-node';
 import * as webSocket from './webSocket';
+import { ReadStream } from 'fs';
 
 export { webSocket };
 
@@ -189,3 +190,13 @@ export interface IGettingStartedSample {
   url: string;
   tags?: Array<string>;
 }
+
+export interface IAirGapSample extends IGettingStartedSample {
+  project?: { zip?: { filename?: string } };
+  devfile?: { filename?: string };
+}
+
+export interface IStreamedFile {
+  stream: ReadStream;
+  size: number;
+}

packages/dashboard-backend/package.json

@@ -37,6 +37,7 @@
     "@fastify/swagger": "^8.14.0",
     "@fastify/swagger-ui": "3.0.0",
     "@fastify/websocket": "^10.0.1",
+    "@fastify/rate-limit": "^9.1.0",
     "@kubernetes/client-node": "^0.21.0",
     "args": "^5.0.3",
     "axios": "^1.7.0",

packages/dashboard-backend/src/app.ts

@@ -21,6 +21,7 @@ import { registerCors } from '@/plugins/cors';
 import { registerStaticServer } from '@/plugins/staticServer';
 import { registerSwagger } from '@/plugins/swagger';
 import { registerWebSocket } from '@/plugins/webSocket';
+import { registerAirGapSampleRoute } from '@/routes/api/airGapSample';
 import { registerClusterConfigRoute } from '@/routes/api/clusterConfig';
 import { registerClusterInfoRoute } from '@/routes/api/clusterInfo';
 import { registerDataResolverRoute } from '@/routes/api/dataResolver';
@@ -68,6 +69,8 @@ export default async function buildApp(server: FastifyInstance): Promise<unknown
     },
   );
 
+  server.register(import('@fastify/rate-limit'));
+
   return Promise.allSettled([
     registerWebSocket(server),
 
@@ -123,5 +126,7 @@ export default async function buildApp(server: FastifyInstance): Promise<unknown
     registerSShKeysRoutes(server),
 
     registerWorkspacePreferencesRoute(server),
+
+    registerAirGapSampleRoute(server),
   ]);
 }

packages/dashboard-backend/src/devworkspaceClient/__mocks__/index.ts

@@ -11,6 +11,7 @@
  */
 
 import {
+  IAirGapSampleApi,
   IDevWorkspaceApi,
   IDevWorkspacePreferencesApi,
   IDevWorkspaceTemplateApi,
@@ -64,6 +65,11 @@ export class DevWorkspaceClient implements IDevWorkspaceClient {
   get gettingStartedSampleApi(): IGettingStartedSampleApi {
     throw new Error('Method not implemented.');
   }
+
+  get airGapSampleApi(): IAirGapSampleApi {
+    throw new Error('Method not implemented.');
+  }
+
   get gitConfigApi(): IGitConfigApi {
     throw new Error('Method not implemented.');
   }

packages/dashboard-backend/src/devworkspaceClient/index.ts

@@ -12,6 +12,7 @@
 
 import * as k8s from '@kubernetes/client-node';
 
+import { AirGapSampleApiService } from '@/devworkspaceClient/services/airGapSampleApi';
 import { DevWorkspaceApiService } from '@/devworkspaceClient/services/devWorkspaceApi';
 import { DevWorkspacePreferencesApiService } from '@/devworkspaceClient/services/devWorkspacePreferencesApi';
 import { DevWorkspaceTemplateApiService } from '@/devworkspaceClient/services/devWorkspaceTemplateApi';
@@ -29,6 +30,7 @@ import { ServerConfigApiService } from '@/devworkspaceClient/services/serverConf
 import { SshKeysService } from '@/devworkspaceClient/services/sshKeysApi';
 import { UserProfileApiService } from '@/devworkspaceClient/services/userProfileApi';
 import {
+  IAirGapSampleApi,
   IDevWorkspaceApi,
   IDevWorkspaceClient,
   IDevWorkspacePreferencesApi,
@@ -109,6 +111,10 @@ export class DevWorkspaceClient implements IDevWorkspaceClient {
     return new GettingStartedSamplesApiService(this.kubeConfig);
   }
 
+  get airGapSampleApi(): IAirGapSampleApi {
+    return new AirGapSampleApiService();
+  }
+
   get editorsApi(): IEditorsApi {
     return new EditorsApiService();
   }

packages/dashboard-backend/src/devworkspaceClient/services/__tests__/airGapSamplesApi.spec.ts

@@ -0,0 +1,114 @@
+/*
+ * Copyright (c) 2018-2024 Red Hat, Inc.
+ * This program and the accompanying materials are made
+ * available under the terms of the Eclipse Public License 2.0
+ * which is available at https://www.eclipse.org/legal/epl-2.0/
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ *
+ * Contributors:
+ *   Red Hat, Inc. - initial API and implementation
+ */
+
+import path from 'path';
+
+import { AirGapSampleApiService } from '@/devworkspaceClient/services/airGapSampleApi';
+
+describe('Getting Started Samples API Service', () => {
+  let airGapSampleApiService: AirGapSampleApiService;
+
+  beforeEach(() => {
+    jest.resetModules();
+    airGapSampleApiService = new AirGapSampleApiService(
+      path.join(__dirname, 'fixtures', 'air-gap'),
+    );
+  });
+
+  afterEach(() => {
+    jest.clearAllMocks();
+  });
+
+  test('fetching metadata', async () => {
+    const res = await airGapSampleApiService.list();
+    expect(res.length).toEqual(6);
+  });
+
+  test('reading devfile', async () => {
+    const airGapResource = await airGapSampleApiService.downloadDevfile('Sample_devfile');
+    const devfileContent = await streamToString(airGapResource.stream);
+    expect(devfileContent.length).toEqual(airGapResource.size);
+    expect(devfileContent).toEqual(
+      'schemaVersion: 2.2.0\n' + 'metadata:\n' + '  generateName: empty\n',
+    );
+  });
+
+  test('reading project', async () => {
+    const airGapResource = await airGapSampleApiService.downloadProject('Sample_project');
+    const devfileContent = await streamToString(airGapResource.stream);
+    expect(devfileContent.length).toEqual(airGapResource.size);
+    expect(devfileContent).toEqual('project');
+  });
+
+  test('error reading devfile, if field not specified', async () => {
+    try {
+      await airGapSampleApiService.downloadDevfile('Sample_no_devfile_filename');
+      fail('should throw an error');
+    } catch (e: any) {
+      expect(e.message).toEqual('filename not defined');
+    }
+  });
+
+  test('error reading project, if field not specified', async () => {
+    try {
+      await airGapSampleApiService.downloadProject('Sample_no_project_filename');
+      fail('should throw an error');
+    } catch (e: any) {
+      expect(e.message).toEqual('filename not defined');
+    }
+  });
+
+  test('error reading devfile, if devfile does not exist', async () => {
+    try {
+      await airGapSampleApiService.downloadDevfile('Sample_devfile_not_exists');
+      fail('should throw an error');
+    } catch (e: any) {
+      expect(e.message).toEqual('File not found');
+    }
+  });
+
+  test('error reading project, if project does not exist', async () => {
+    try {
+      await airGapSampleApiService.downloadProject('Sample_project_not_exists');
+      fail('should throw an error');
+    } catch (e: any) {
+      expect(e.message).toEqual('File not found');
+    }
+  });
+
+  test('error reading devfile, sample not found', async () => {
+    try {
+      await airGapSampleApiService.downloadDevfile('Sample_not_exists');
+      fail('should throw an error');
+    } catch (e: any) {
+      expect(e.message).toEqual('Sample not found');
+    }
+  });
+
+  test('error reading project, sample not found', async () => {
+    try {
+      await airGapSampleApiService.downloadProject('Sample_not_exists');
+      fail('should throw an error');
+    } catch (e: any) {
+      expect(e.message).toEqual('Sample not found');
+    }
+  });
+});
+
+function streamToString(stream: NodeJS.ReadableStream): Promise<string> {
+  const chunks: any[] = [];
+  return new Promise((resolve, reject) => {
+    stream.on('data', chunk => chunks.push(chunk));
+    stream.on('error', reject);
+    stream.on('end', () => resolve(Buffer.concat(chunks).toString()));
+  });
+}