[Serverless] Select project type via config (#155754)

(cherry picked from commit de64ff5)

# Conflicts:
#	config/serverless.es.yml
#	config/serverless.security.yml
#	config/serverless.yml
#	src/cli/serve/serve.js
#	x-pack/plugins/serverless/server/plugin.ts

config/serverless.oblt.yml

@@ -1 +1,2 @@
+uiSettings.overrides.defaultRoute: /app/observability/overview
 xpack.infra.logs.app_target: discover

packages/core/root/core-root-server-internal/src/bootstrap.test.mocks.ts

@@ -0,0 +1,25 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { Env } from '@kbn/config';
+import { rawConfigServiceMock, configServiceMock } from '@kbn/config-mocks';
+
+export const mockConfigService = configServiceMock.create();
+export const mockRawConfigService = rawConfigServiceMock.create();
+export const mockRawConfigServiceConstructor = jest.fn(() => mockRawConfigService);
+jest.doMock('@kbn/config', () => ({
+  ConfigService: jest.fn(() => mockConfigService),
+  Env,
+  RawConfigService: jest.fn(mockRawConfigServiceConstructor),
+}));
+
+jest.doMock('./root', () => ({
+  Root: jest.fn(() => ({
+    shutdown: jest.fn(),
+  })),
+}));

packages/core/root/core-root-server-internal/src/bootstrap.test.ts

@@ -0,0 +1,61 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { of } from 'rxjs';
+import type { CliArgs } from '@kbn/config';
+
+import { mockRawConfigService, mockRawConfigServiceConstructor } from './bootstrap.test.mocks';
+
+jest.mock('@kbn/core-logging-server-internal');
+
+import { bootstrap } from './bootstrap';
+
+const bootstrapCfg = {
+  configs: ['config/kibana.yml'],
+  cliArgs: {} as unknown as CliArgs,
+  applyConfigOverrides: () => ({}),
+};
+
+describe('bootstrap', () => {
+  describe('serverless', () => {
+    beforeEach(() => {
+      jest.clearAllMocks();
+    });
+
+    test('should load additional serverless files for a valid project', async () => {
+      mockRawConfigService.getConfig$.mockReturnValue(of({ serverless: 'es' }));
+      await bootstrap(bootstrapCfg);
+      expect(mockRawConfigServiceConstructor).toHaveBeenCalledTimes(2);
+      expect(mockRawConfigServiceConstructor).toHaveBeenNthCalledWith(
+        1,
+        bootstrapCfg.configs,
+        bootstrapCfg.applyConfigOverrides
+      );
+      expect(mockRawConfigServiceConstructor).toHaveBeenNthCalledWith(
+        2,
+        [
+          expect.stringContaining('config/serverless.yml'),
+          expect.stringContaining('config/serverless.es.yml'),
+          ...bootstrapCfg.configs,
+        ],
+        bootstrapCfg.applyConfigOverrides
+      );
+    });
+
+    test('should skip loading the serverless files for an invalid project', async () => {
+      mockRawConfigService.getConfig$.mockReturnValue(of({ serverless: 'not-valid' }));
+      await bootstrap(bootstrapCfg);
+      expect(mockRawConfigServiceConstructor).toHaveBeenCalledTimes(1);
+      expect(mockRawConfigServiceConstructor).toHaveBeenNthCalledWith(
+        1,
+        bootstrapCfg.configs,
+        bootstrapCfg.applyConfigOverrides
+      );
+    });
+  });
+});

packages/core/root/core-root-server-internal/src/bootstrap.ts

@@ -7,9 +7,14 @@
  */
 
 import chalk from 'chalk';
+import { firstValueFrom } from 'rxjs';
 import { getPackages } from '@kbn/repo-packages';
 import { CliArgs, Env, RawConfigService } from '@kbn/config';
 import { CriticalError } from '@kbn/core-base-server-internal';
+import { resolve } from 'path';
+import { getConfigDirectory } from '@kbn/utils';
+import { statSync } from 'fs';
+import { VALID_SERVERLESS_PROJECT_TYPES } from './root/serverless_config';
 import { Root } from './root';
 import { MIGRATION_EXCEPTION_CODE } from './constants';
 
@@ -38,15 +43,40 @@ export async function bootstrap({ configs, cliArgs, applyConfigOverrides }: Boot
   // eslint-disable-next-line @typescript-eslint/no-var-requires
   const { REPO_ROOT } = require('@kbn/repo-info');
 
-  const env = Env.createDefault(REPO_ROOT, {
+  let env = Env.createDefault(REPO_ROOT, {
     configs,
     cliArgs,
     repoPackages: getPackages(REPO_ROOT),
   });
 
-  const rawConfigService = new RawConfigService(env.configs, applyConfigOverrides);
+  let rawConfigService = new RawConfigService(env.configs, applyConfigOverrides);
   rawConfigService.loadConfig();
 
+  // Hack to load the extra serverless config files if `serverless: {projectType}` is found in it.
+  const rawConfig = await firstValueFrom(rawConfigService.getConfig$());
+  const serverlessProjectType = rawConfig?.serverless;
+  if (
+    typeof serverlessProjectType === 'string' &&
+    VALID_SERVERLESS_PROJECT_TYPES.includes(serverlessProjectType)
+  ) {
+    const extendedConfigs = [
+      ...['serverless.yml', `serverless.${serverlessProjectType}.yml`]
+        .map((name) => resolve(getConfigDirectory(), name))
+        .filter(configFileExists),
+      ...configs,
+    ];
+
+    env = Env.createDefault(REPO_ROOT, {
+      configs: extendedConfigs,
+      cliArgs: { ...cliArgs, serverless: true },
+      repoPackages: getPackages(REPO_ROOT),
+    });
+
+    rawConfigService.stop();
+    rawConfigService = new RawConfigService(env.configs, applyConfigOverrides);
+    rawConfigService.loadConfig();
+  }
+
   const root = new Root(rawConfigService, env, onRootShutdown);
 
   process.on('SIGHUP', () => reloadConfiguration());
@@ -128,3 +158,15 @@ function onRootShutdown(reason?: any) {
 
   process.exit(0);
 }
+
+function configFileExists(path: string) {
+  try {
+    return statSync(path).isFile();
+  } catch (err) {
+    if (err.code === 'ENOENT') {
+      return false;
+    }
+
+    throw err;
+  }
+}

packages/core/root/core-root-server-internal/src/register_service_config.ts

@@ -28,6 +28,7 @@ import { uiSettingsConfig } from '@kbn/core-ui-settings-server-internal';
 
 import { config as pluginsConfig } from '@kbn/core-plugins-server-internal';
 import { elasticApmConfig } from './root/elastic_config';
+import { serverlessConfig } from './root/serverless_config';
 
 const rootConfigPath = '';
 
@@ -49,6 +50,7 @@ export function registerServiceConfig(configService: ConfigService) {
     pluginsConfig,
     savedObjectsConfig,
     savedObjectsMigrationConfig,
+    serverlessConfig,
     statusConfig,
     uiSettingsConfig,
   ];

packages/core/root/core-root-server-internal/src/root/serverless_config.ts

@@ -0,0 +1,34 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { schema, TypeOf, Type } from '@kbn/config-schema';
+import { ServiceConfigDescriptor } from '@kbn/core-base-server-internal';
+
+// Config validation for how to run Kibana in Serverless mode.
+// Clients need to specify the project type to run in.
+// Going for a simple `serverless` string because it serves as
+// a direct replacement to the legacy --serverless CLI flag.
+// If we even decide to extend this further, and converting it into an object,
+// BWC can be ensured by adding the object definition as another alternative to `schema.oneOf`.
+
+export const VALID_SERVERLESS_PROJECT_TYPES = ['es', 'oblt', 'security'];
+
+const serverlessConfigSchema = schema.maybe(
+  schema.oneOf(
+    VALID_SERVERLESS_PROJECT_TYPES.map((projectName) => schema.literal(projectName)) as [
+      Type<typeof VALID_SERVERLESS_PROJECT_TYPES[number]> // This cast is needed because it's different to Type<T>[] :sight:
+    ]
+  )
+);
+
+export type ServerlessConfigType = TypeOf<typeof serverlessConfigSchema>;
+
+export const serverlessConfig: ServiceConfigDescriptor<ServerlessConfigType> = {
+  path: 'serverless',
+  schema: serverlessConfigSchema,
+};

src/cli/serve/integration_tests/serverless_config_flag.test.ts

@@ -0,0 +1,88 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { spawn, spawnSync } from 'child_process';
+import { readFileSync } from 'fs';
+import { resolve } from 'path';
+import { filter, firstValueFrom, from, take, concatMap } from 'rxjs';
+
+import { REPO_ROOT } from '@kbn/repo-info';
+import { getConfigDirectory } from '@kbn/utils';
+
+describe('cli serverless project type', () => {
+  it(
+    'exits with statusCode 1 and logs an error when serverless project type is invalid',
+    () => {
+      const { error, status, stdout } = spawnSync(
+        process.execPath,
+        ['scripts/kibana', '--serverless=non-existing-project-type'],
+        {
+          cwd: REPO_ROOT,
+        }
+      );
+      expect(error).toBe(undefined);
+
+      expect(stdout.toString('utf8')).toContain(
+        'FATAL CLI ERROR Error: invalid --serverless value, must be one of es, oblt, security'
+      );
+
+      expect(status).toBe(1);
+    },
+    20 * 1000
+  );
+
+  // Skipping this one because on CI it fails to read the config file
+  it.skip.each(['es', 'oblt', 'security'])(
+    'writes the serverless project type %s in config/serverless.recent.yml',
+    async (mode) => {
+      // Making sure `--serverless` translates into the `serverless` config entry, and validates against the accepted values
+      const child = spawn(process.execPath, ['scripts/kibana', `--serverless=${mode}`], {
+        cwd: REPO_ROOT,
+      });
+
+      // Wait for 5 lines in the logs
+      await firstValueFrom(from(child.stdout).pipe(take(5)));
+
+      expect(
+        readFileSync(resolve(getConfigDirectory(), 'serverless.recent.yml'), 'utf-8')
+      ).toContain(`serverless: ${mode}\n`);
+
+      child.kill('SIGKILL');
+    }
+  );
+
+  it.each(['es', 'oblt', 'security'])(
+    'Kibana does not crash when running project type %s',
+    async (mode) => {
+      const child = spawn(process.execPath, ['scripts/kibana', `--serverless=${mode}`], {
+        cwd: REPO_ROOT,
+      });
+
+      // Wait until Kibana starts listening to the port
+      let leftover = '';
+      const found = await firstValueFrom(
+        from(child.stdout).pipe(
+          concatMap((chunk: Buffer) => {
+            const data = leftover + chunk.toString('utf-8');
+            const msgs = data.split('\n');
+            leftover = msgs.pop() ?? '';
+            return msgs;
+          }),
+          filter(
+            (msg) =>
+              msg.includes('http server running at http://localhost:5601') || msg.includes('FATAL')
+          )
+        )
+      );
+
+      child.kill('SIGKILL');
+
+      expect(found).not.toContain('FATAL');
+    }
+  );
+});