Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Non-Inclusive Language] Replace isDevClusterMaster with isDevClusterManager #1690

Closed
tmarkley opened this issue Jun 7, 2022 · 0 comments · Fixed by #1719
Closed

[Non-Inclusive Language] Replace isDevClusterMaster with isDevClusterManager #1690

tmarkley opened this issue Jun 7, 2022 · 0 comments · Fixed by #1719

Comments

@tmarkley
Copy link
Contributor

tmarkley commented Jun 7, 2022

Parent: #1318

Doesn't need to be deprecated; can be replaced.

Instances of isDevClusterMaster

isDevClusterMaster: boolean;
}
/** @internal */
export interface CliArgs {
dev: boolean;
envName?: string;
quiet: boolean;
silent: boolean;
watch: boolean;
repl: boolean;
basePath: boolean;
/** @deprecated use disableOptimizer to know if the @osd/optimizer is disabled in development */
optimize?: boolean;
runExamples: boolean;
disableOptimizer: boolean;
cache: boolean;
dist: boolean;
}
/** @internal */
export interface RawPackageInfo {
branch: string;
version: string;
build: {
distributable?: boolean;
number: number;
sha: string;
};
}
export class Env {
/**
* @internal
*/
public static createDefault(repoRoot: string, options: EnvOptions, pkg?: RawPackageInfo): Env {
if (!pkg) {
pkg = loadJsonFile.sync(join(repoRoot, 'package.json')) as RawPackageInfo;
}
return new Env(repoRoot, pkg, options);
}
/** @internal */
public readonly configDir: string;
/** @internal */
public readonly binDir: string;
/** @internal */
public readonly logDir: string;
/** @internal */
public readonly pluginSearchPaths: readonly string[];
/**
* Information about OpenSearch Dashboards package (version, build number etc.).
*/
public readonly packageInfo: Readonly<PackageInfo>;
/**
* Mode OpenSearch Dashboards currently run in (development or production).
*/
public readonly mode: Readonly<EnvironmentMode>;
/**
* Arguments provided through command line.
* @internal
*/
public readonly cliArgs: Readonly<CliArgs>;
/**
* Paths to the configuration files.
* @internal
*/
public readonly configs: readonly string[];
/**
* Indicates that this OpenSearch Dashboards instance is run as development Node Cluster master.
* @internal
*/
public readonly isDevClusterMaster: boolean;
/**
* @internal
*/
constructor(public readonly homeDir: string, pkg: RawPackageInfo, options: EnvOptions) {
this.configDir = resolve(this.homeDir, 'config');
this.binDir = resolve(this.homeDir, 'bin');
this.logDir = resolve(this.homeDir, 'log');
/**
* BEWARE: this needs to stay roughly synchronized with the @osd/optimizer
* `packages/osd-optimizer/src/optimizer_config.ts` determines the paths
* that should be searched for plugins to build
*/
this.pluginSearchPaths = [
resolve(this.homeDir, 'src', 'plugins'),
resolve(this.homeDir, 'plugins'),
...(options.cliArgs.runExamples ? [resolve(this.homeDir, 'examples')] : []),
resolve(this.homeDir, '..', 'opensearch-dashboards-extra'),
];
this.cliArgs = Object.freeze(options.cliArgs);
this.configs = Object.freeze(options.configs);
this.isDevClusterMaster = options.isDevClusterMaster;

isDevClusterMaster:
options.isDevClusterMaster !== undefined ? options.isDevClusterMaster : false,

"isDevClusterMaster": false,
"logDir": "/test/opensearchDashboardsRoot/log",
"mode": Object {
"dev": true,
"name": "development",
"prod": false,
},
"packageInfo": Object {
"branch": "feature-v1",
"buildNum": 9007199254740991,
"buildSha": "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX",
"dist": false,
"version": "v1",
},
"pluginSearchPaths": Array [
"/test/opensearchDashboardsRoot/src/plugins",
"/test/opensearchDashboardsRoot/plugins",
"/test/opensearchDashboardsRoot/../opensearch-dashboards-extra",
],
}
`;
exports[`correctly creates default environment if \`--env.name\` is supplied.: prod env properties 1`] = `
Env {
"binDir": "/test/opensearchDashboardsRoot/bin",
"cliArgs": Object {
"basePath": false,
"cache": true,
"dev": false,
"disableOptimizer": true,
"dist": false,
"envName": "production",
"quiet": false,
"repl": false,
"runExamples": false,
"silent": false,
"watch": false,
},
"configDir": "/test/opensearchDashboardsRoot/config",
"configs": Array [
"/some/other/path/some-opensearch-dashboards.yml",
],
"homeDir": "/test/opensearchDashboardsRoot",
"isDevClusterMaster": false,
"logDir": "/test/opensearchDashboardsRoot/log",
"mode": Object {
"dev": false,
"name": "production",
"prod": true,
},
"packageInfo": Object {
"branch": "feature-v1",
"buildNum": 9007199254740991,
"buildSha": "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX",
"dist": false,
"version": "v1",
},
"pluginSearchPaths": Array [
"/test/opensearchDashboardsRoot/src/plugins",
"/test/opensearchDashboardsRoot/plugins",
"/test/opensearchDashboardsRoot/../opensearch-dashboards-extra",
],
}
`;
exports[`correctly creates default environment in dev mode.: env properties 1`] = `
Env {
"binDir": "/test/opensearchDashboardsRoot/bin",
"cliArgs": Object {
"basePath": false,
"cache": true,
"dev": true,
"disableOptimizer": true,
"dist": false,
"quiet": false,
"repl": false,
"runExamples": false,
"silent": false,
"watch": false,
},
"configDir": "/test/opensearchDashboardsRoot/config",
"configs": Array [
"/test/cwd/config/opensearch_dashboards.yml",
],
"homeDir": "/test/opensearchDashboardsRoot",
"isDevClusterMaster": true,
"logDir": "/test/opensearchDashboardsRoot/log",
"mode": Object {
"dev": true,
"name": "development",
"prod": false,
},
"packageInfo": Object {
"branch": "some-branch",
"buildNum": 9007199254740991,
"buildSha": "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX",
"dist": false,
"version": "some-version",
},
"pluginSearchPaths": Array [
"/test/opensearchDashboardsRoot/src/plugins",
"/test/opensearchDashboardsRoot/plugins",
"/test/opensearchDashboardsRoot/../opensearch-dashboards-extra",
],
}
`;
exports[`correctly creates default environment in prod distributable mode.: env properties 1`] = `
Env {
"binDir": "/test/opensearchDashboardsRoot/bin",
"cliArgs": Object {
"basePath": false,
"cache": true,
"dev": false,
"disableOptimizer": true,
"dist": false,
"quiet": false,
"repl": false,
"runExamples": false,
"silent": false,
"watch": false,
},
"configDir": "/test/opensearchDashboardsRoot/config",
"configs": Array [
"/some/other/path/some-opensearch-dashboards.yml",
],
"homeDir": "/test/opensearchDashboardsRoot",
"isDevClusterMaster": false,
"logDir": "/test/opensearchDashboardsRoot/log",
"mode": Object {
"dev": false,
"name": "production",
"prod": true,
},
"packageInfo": Object {
"branch": "feature-v1",
"buildNum": 100,
"buildSha": "feature-v1-build-sha",
"dist": true,
"version": "v1",
},
"pluginSearchPaths": Array [
"/test/opensearchDashboardsRoot/src/plugins",
"/test/opensearchDashboardsRoot/plugins",
"/test/opensearchDashboardsRoot/../opensearch-dashboards-extra",
],
}
`;
exports[`correctly creates default environment in prod non-distributable mode.: env properties 1`] = `
Env {
"binDir": "/test/opensearchDashboardsRoot/bin",
"cliArgs": Object {
"basePath": false,
"cache": true,
"dev": false,
"disableOptimizer": true,
"dist": false,
"quiet": false,
"repl": false,
"runExamples": false,
"silent": false,
"watch": false,
},
"configDir": "/test/opensearchDashboardsRoot/config",
"configs": Array [
"/some/other/path/some-opensearch-dashboards.yml",
],
"homeDir": "/test/opensearchDashboardsRoot",
"isDevClusterMaster": false,
"logDir": "/test/opensearchDashboardsRoot/log",
"mode": Object {
"dev": false,
"name": "production",
"prod": true,
},
"packageInfo": Object {
"branch": "feature-v1",
"buildNum": 9007199254740991,
"buildSha": "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX",
"dist": false,
"version": "v1",
},
"pluginSearchPaths": Array [
"/test/opensearchDashboardsRoot/src/plugins",
"/test/opensearchDashboardsRoot/plugins",
"/test/opensearchDashboardsRoot/../opensearch-dashboards-extra",
],
}
`;
exports[`correctly creates environment with constructor.: env properties 1`] = `
Env {
"binDir": "/some/home/dir/bin",
"cliArgs": Object {
"basePath": false,
"cache": true,
"dev": false,
"disableOptimizer": true,
"dist": false,
"quiet": false,
"repl": false,
"runExamples": false,
"silent": false,
"watch": false,
},
"configDir": "/some/home/dir/config",
"configs": Array [
"/some/other/path/some-opensearch-dashboards.yml",
],
"homeDir": "/some/home/dir",
"isDevClusterMaster": false,

import { isMaster } from 'cluster';
import { CliArgs, Env, RawConfigService } from './config';
import { Root } from './root';
import { CriticalError } from './errors';
interface OpenSearchDashboardsFeatures {
// Indicates whether we can run OpenSearch Dashboards in a so called cluster mode in which
// OpenSearch Dashboards is run as a "worker" process together with optimizer "worker" process
// that are orchestrated by the "master" process (dev mode only feature).
isClusterModeSupported: boolean;
// Indicates whether we can run OpenSearch Dashboards in REPL mode (dev mode only feature).
isReplModeSupported: boolean;
}
interface BootstrapArgs {
configs: string[];
cliArgs: CliArgs;
applyConfigOverrides: (config: Record<string, any>) => Record<string, any>;
features: OpenSearchDashboardsFeatures;
}
/**
*
* @internal
* @param param0 - options
*/
export async function bootstrap({
configs,
cliArgs,
applyConfigOverrides,
features,
}: BootstrapArgs) {
if (cliArgs.repl && !features.isReplModeSupported) {
onRootShutdown('OpenSearch Dashboards REPL mode can only be run in development mode.');
}
if (cliArgs.optimize) {
// --optimize is deprecated and does nothing now, avoid starting up and just shutdown
return;
}
// `bootstrap` is exported from the `src/core/server/index` module,
// meaning that any test importing, implicitly or explicitly, anything concrete
// from `core/server` will load `dev-utils`. As some tests are mocking the `fs` package,
// and as `REPO_ROOT` is initialized on the fly when importing `dev-utils` and requires
// the `fs` package, it causes failures. This is why we use a dynamic `require` here.
// eslint-disable-next-line @typescript-eslint/no-var-requires
const { REPO_ROOT } = require('@osd/utils');
const env = Env.createDefault(REPO_ROOT, {
configs,
cliArgs,
isDevClusterMaster: isMaster && cliArgs.dev && features.isClusterModeSupported,

test('does not start http server if process is dev cluster master', async () => {
const configService = createConfigService();
const httpServer = {
isListening: () => false,
setup: jest.fn().mockReturnValue({}),
start: jest.fn(),
stop: noop,
};
mockHttpServer.mockImplementation(() => httpServer);
const service = new HttpService({
coreId,
configService,
env: Env.createDefault(REPO_ROOT, getEnvOptions({ isDevClusterMaster: true })),

* 2. When the process is run as dev cluster master in which case cluster manager
* will fork a dedicated process where http service will be set up instead.
* @internal
* */
private shouldListen(config: HttpConfig) {
return !this.coreContext.env.isDevClusterMaster && config.autoListen;

describe('once LegacyService is set up in `devClusterMaster` mode', () => {
beforeEach(() => {
configService.atPath.mockImplementation((path) => {
return new BehaviorSubject(
path === 'dev' ? { basePathProxyTargetPort: 100500 } : { basePath: '/abc' }
);
});
});
test('creates ClusterManager without base path proxy.', async () => {
const devClusterLegacyService = new LegacyService({
coreId,
env: Env.createDefault(
REPO_ROOT,
getEnvOptions({
cliArgs: { silent: true, basePath: false },
isDevClusterMaster: true,
})
),
logger,
configService: configService as any,
});
await devClusterLegacyService.setupLegacyConfig();
await devClusterLegacyService.setup(setupDeps);
await devClusterLegacyService.start(startDeps);
expect(MockClusterManager).toHaveBeenCalledTimes(1);
expect(MockClusterManager).toHaveBeenCalledWith(
expect.objectContaining({ silent: true, basePath: false }),
expect.objectContaining({
get: expect.any(Function),
set: expect.any(Function),
}),
undefined
);
});
test('creates ClusterManager with base path proxy.', async () => {
const devClusterLegacyService = new LegacyService({
coreId,
env: Env.createDefault(
REPO_ROOT,
getEnvOptions({
cliArgs: { quiet: true, basePath: true },
isDevClusterMaster: true,

if (this.coreContext.env.isDevClusterMaster) {

const initialize = config.initialize && !this.coreContext.env.isDevClusterMaster;

import { isMaster, isWorker } from 'cluster';
import { Server } from '@hapi/hapi';
import { LogRotator } from './log_rotator';
import { OpenSearchDashboardsConfig } from '../../osd_server';
let logRotator: LogRotator;
export async function setupLoggingRotate(server: Server, config: OpenSearchDashboardsConfig) {
// If log rotate is not enabled we skip
if (!config.get('logging.rotate.enabled')) {
return;
}
// We just want to start the logging rotate service once
// and we choose to use the master (prod) or the worker server (dev)
if (!isMaster && isWorker && process.env.osdWorkerType !== 'server') {
return;
}
// We don't want to run logging rotate server if
// we are not logging to a file
if (config.get('logging.dest') === 'stdout') {
server.log(
['warning', 'logging:rotate'],
'Log rotation is enabled but logging.dest is configured for stdout. Set logging.dest to a file for this setting to take effect.'
);
return;
}
// Enable Logging Rotate Service
// We need the master process and it can

import { isMaster } from 'cluster';
import fs from 'fs';
import { Server } from '@hapi/hapi';
import { throttle } from 'lodash';
import { tmpdir } from 'os';
import { basename, dirname, join, sep } from 'path';
import { Observable } from 'rxjs';
import { first } from 'rxjs/operators';
import { promisify } from 'util';
import { OpenSearchDashboardsConfig } from '../../osd_server';
const mkdirAsync = promisify(fs.mkdir);
const readdirAsync = promisify(fs.readdir);
const renameAsync = promisify(fs.rename);
const statAsync = promisify(fs.stat);
const unlinkAsync = promisify(fs.unlink);
const writeFileAsync = promisify(fs.writeFile);
export class LogRotator {
private readonly config: OpenSearchDashboardsConfig;
private readonly log: Server['log'];
public logFilePath: string;
public everyBytes: number;
public keepFiles: number;
public running: boolean;
private logFileSize: number;
public isRotating: boolean;
public throttledRotate: () => void;
public stalker: chokidar.FSWatcher | null;
public usePolling: boolean;
public pollingInterval: number;
private stalkerUsePollingPolicyTestTimeout: NodeJS.Timeout | null;
public shouldUsePolling: boolean;
constructor(config: OpenSearchDashboardsConfig, server: Server) {
this.config = config;
this.log = server.log.bind(server);
this.logFilePath = config.get('logging.dest');
this.everyBytes = config.get('logging.rotate.everyBytes');
this.keepFiles = config.get('logging.rotate.keepFiles');
this.running = false;
this.logFileSize = 0;
this.isRotating = false;
this.throttledRotate = throttle(async () => await this._rotate(), 5000);
this.stalker = null;
this.usePolling = config.get('logging.rotate.usePolling');
this.pollingInterval = config.get('logging.rotate.pollingInterval');
this.shouldUsePolling = false;
this.stalkerUsePollingPolicyTestTimeout = null;
}
async start() {
if (this.running) {
return;
}
this.running = true;
// create exit listener for cleanup purposes
this._createExitListener();
// call rotate on startup
await this._callRotateOnStartup();
// init log file size monitor
await this._startLogFileSizeMonitor();
}
stop = () => {
if (!this.running) {
return;
}
// cleanup exit listener
this._deleteExitListener();
// stop log file size monitor
this._stopLogFileSizeMonitor();
this.running = false;
};
async _shouldUsePolling() {
try {
// Setup a test file in order to try the fs env
// and understand if we need to usePolling or not
const tempFileDir = tmpdir();
const tempFile = join(tempFileDir, 'osd_log_rotation_use_polling_test_file.log');
await mkdirAsync(tempFileDir, { recursive: true });
await writeFileAsync(tempFile, '');
// setup fs.watch for the temp test file
const testWatcher = fs.watch(tempFile, { persistent: false });
// await writeFileAsync(tempFile, 'test');
const usePollingTest$ = new Observable<boolean>((observer) => {
// observable complete function
const completeFn = (completeStatus: boolean) => {
if (this.stalkerUsePollingPolicyTestTimeout) {
clearTimeout(this.stalkerUsePollingPolicyTestTimeout);
}
testWatcher.close();
observer.next(completeStatus);
observer.complete();
};
// setup conditions that would fire the observable
this.stalkerUsePollingPolicyTestTimeout = setTimeout(() => completeFn(true), 15000);
testWatcher.on('change', () => completeFn(false));
testWatcher.on('error', () => completeFn(true));
// fire test watcher events
setTimeout(() => {
fs.writeFileSync(tempFile, 'test');
}, 0);
});
// wait for the first observable result and consider it as the result
// for our use polling test
const usePollingTestResult = await usePollingTest$.pipe(first()).toPromise();
// delete the temp file used for the test
await unlinkAsync(tempFile);
return usePollingTestResult;
} catch {
return true;
}
}
async _startLogFileSizeMonitor() {
this.usePolling = this.config.get('logging.rotate.usePolling');
this.shouldUsePolling = await this._shouldUsePolling();
if (this.usePolling && !this.shouldUsePolling) {
this.log(
['warning', 'logging:rotate'],
'Looks like your current environment support a faster algorithm then polling. You can try to disable `usePolling`'
);
}
if (!this.usePolling && this.shouldUsePolling) {
this.log(
['error', 'logging:rotate'],
'Looks like within your current environment you need to use polling in order to enable log rotator. Please enable `usePolling`'
);
}
this.stalker = chokidar.watch(this.logFilePath, {
ignoreInitial: true,
awaitWriteFinish: false,
useFsEvents: false,
usePolling: this.usePolling,
interval: this.pollingInterval,
binaryInterval: this.pollingInterval,
alwaysStat: true,
atomic: false,
});
this.stalker.on('change', this._logFileSizeMonitorHandler);
}
_logFileSizeMonitorHandler = async (filename: string, stats: fs.Stats) => {
if (!filename || !stats) {
return;
}
this.logFileSize = stats.size || 0;
await this.throttledRotate();
};
_stopLogFileSizeMonitor() {
if (!this.stalker) {
return;
}
this.stalker.close();
if (this.stalkerUsePollingPolicyTestTimeout) {
clearTimeout(this.stalkerUsePollingPolicyTestTimeout);
}
}
_createExitListener() {
process.on('exit', this.stop);
}
_deleteExitListener() {
process.removeListener('exit', this.stop);
}
async _getLogFileSizeAndCreateIfNeeded() {
try {
const logFileStats = await statAsync(this.logFilePath);
return logFileStats.size;
} catch {
// touch the file to make the watcher being able to register
// change events
await writeFileAsync(this.logFilePath, '');
return 0;
}
}
async _callRotateOnStartup() {
this.logFileSize = await this._getLogFileSizeAndCreateIfNeeded();
await this._rotate();
}
_shouldRotate() {
// should rotate evaluation
// 1. should rotate if current log size exceeds
// the defined one on everyBytes
// 2. should not rotate if is already rotating or if any
// of the conditions on 1. do not apply
if (this.isRotating) {
return false;
}
return this.logFileSize >= this.everyBytes;
}
async _rotate() {
if (!this._shouldRotate()) {
return;
}
await this._rotateNow();
}
async _rotateNow() {
// rotate process
// 1. get rotated files metadata (list of log rotated files present on the log folder, numerical sorted)
// 2. delete last file
// 3. rename all files to the correct index +1
// 4. rename + compress current log into 1
// 5. send SIGHUP to reload log config
// rotate process is starting
this.isRotating = true;
// get rotated files metadata
const foundRotatedFiles = await this._readRotatedFilesMetadata();
// delete number of rotated files exceeding the keepFiles limit setting
const rotatedFiles: string[] = await this._deleteFoundRotatedFilesAboveKeepFilesLimit(
foundRotatedFiles
);
// delete last file
await this._deleteLastRotatedFile(rotatedFiles);
// rename all files to correct index + 1
// and normalize numbering if by some reason
// (for example log file deletion) that numbering
// was interrupted
await this._renameRotatedFilesByOne(rotatedFiles);
// rename current log into 0
await this._rotateCurrentLogFile();
// send SIGHUP to reload log configuration
this._sendReloadLogConfigSignal();
// Reset log file size
this.logFileSize = 0;
// rotate process is finished
this.isRotating = false;
}
async _readRotatedFilesMetadata() {
const logFileBaseName = basename(this.logFilePath);
const logFilesFolder = dirname(this.logFilePath);
const foundLogFiles: string[] = await readdirAsync(logFilesFolder);
return (
foundLogFiles
.filter((file) => new RegExp(`${logFileBaseName}\\.\\d`).test(file))
// we use .slice(-1) here in order to retrieve the last number match in the read filenames
.sort((a, b) => Number(a.match(/(\d+)/g)!.slice(-1)) - Number(b.match(/(\d+)/g)!.slice(-1)))
.map((filename) => `${logFilesFolder}${sep}${filename}`)
);
}
async _deleteFoundRotatedFilesAboveKeepFilesLimit(foundRotatedFiles: string[]) {
if (foundRotatedFiles.length <= this.keepFiles) {
return foundRotatedFiles;
}
const finalRotatedFiles = foundRotatedFiles.slice(0, this.keepFiles);
const rotatedFilesToDelete = foundRotatedFiles.slice(
finalRotatedFiles.length,
foundRotatedFiles.length
);
await Promise.all(
rotatedFilesToDelete.map((rotatedFilePath: string) => unlinkAsync(rotatedFilePath))
);
return finalRotatedFiles;
}
async _deleteLastRotatedFile(rotatedFiles: string[]) {
if (rotatedFiles.length < this.keepFiles) {
return;
}
const lastFilePath: string = rotatedFiles.pop() as string;
await unlinkAsync(lastFilePath);
}
async _renameRotatedFilesByOne(rotatedFiles: string[]) {
const logFileBaseName = basename(this.logFilePath);
const logFilesFolder = dirname(this.logFilePath);
for (let i = rotatedFiles.length - 1; i >= 0; i--) {
const oldFilePath = rotatedFiles[i];
const newFilePath = `${logFilesFolder}${sep}${logFileBaseName}.${i + 1}`;
await renameAsync(oldFilePath, newFilePath);
}
}
async _rotateCurrentLogFile() {
const newFilePath = `${this.logFilePath}.0`;
await renameAsync(this.logFilePath, newFilePath);
}
_sendReloadLogConfigSignal() {
if (isMaster) {

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Projects
None yet
2 participants