diff --git a/CHANGELOG.md b/CHANGELOG.md index 948b9e63..ea2a8c0d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ [Full Changelog](https://github.com/SquirrelCorporation/SquirrelServersManager/compare/v0.1.22...HEAD) +**Implemented enhancements:** + +- \[FEAT\] Add advanced diagnostic checks for device connections [\#482](https://github.com/SquirrelCorporation/SquirrelServersManager/pull/482) ([SquirrelDeveloper](https://github.com/SquirrelDeveloper)) + **Merged pull requests:** - \[CHORE\] Increase JSON request size limit to 50mb [\#480](https://github.com/SquirrelCorporation/SquirrelServersManager/pull/480) ([SquirrelDeveloper](https://github.com/SquirrelDeveloper)) diff --git a/client/src/components/DeviceConfiguration/diagnostic/ExistingDeviceAdvancedDiagnostic.tsx b/client/src/components/DeviceConfiguration/diagnostic/ExistingDeviceAdvancedDiagnostic.tsx new file mode 100644 index 00000000..b7913d71 --- /dev/null +++ b/client/src/components/DeviceConfiguration/diagnostic/ExistingDeviceAdvancedDiagnostic.tsx @@ -0,0 +1,144 @@ +import { MedicalSearchDiagnosisSolid } from '@/components/Icons/CustomIcons'; +import { postDeviceDiagnostic } from '@/services/rest/device'; +import { socket } from '@/socket'; +import { history } from '@umijs/max'; +import { + Avatar, + Button, + Card, + Col, + message, + Row, + StepProps, + Steps, +} from 'antd'; +import React, { useEffect } from 'react'; +import { API, SsmDeviceDiagnostic, SsmEvents } from 'ssm-shared-lib'; + +type ExistingDeviceAdvancedDiagnosticProps = { + device: Partial; +}; + +const items: (StepProps & any)[] = [ + { + title: 'Ssh Connect', + description: 'Waiting...', + key: SsmDeviceDiagnostic.Checks.SSH_CONNECT, + }, + { + title: 'Ssh Docker Connect', + description: 'Waiting...', + key: SsmDeviceDiagnostic.Checks.SSH_DOCKER_CONNECT, + }, + { + title: 'Ssh Docker Socket Connectivity', + description: 'Waiting...', + key: SsmDeviceDiagnostic.Checks.DOCKER_SOCKET, + }, + { + title: 'Disk Space', + description: 'Waiting...', + key: SsmDeviceDiagnostic.Checks.DISK_SPACE, + }, + { + title: 'Memory & CPU', + description: 'Waiting...', + key: SsmDeviceDiagnostic.Checks.CPU_MEMORY_INFO, + }, +]; + +const ExistingDeviceAdvancedDiagnostic: React.FC< + ExistingDeviceAdvancedDiagnosticProps +> = ({ device }) => { + const [diagInProgress, setDiagInProgress] = React.useState(false); + const [steps, setSteps] = React.useState(items); + const onDiagnosticProgress = (payload: any) => { + setSteps((prevSteps) => + prevSteps.map((step) => + step.key === payload?.data?.check + ? { + ...step, + description: payload.message || step.description, // Update description if provided + status: !payload.success ? 'error' : 'success', + } + : step, + ), + ); + }; + + useEffect(() => { + socket.connect(); + socket.on(SsmEvents.Diagnostic.PROGRESS, onDiagnosticProgress); + + return () => { + socket.off(SsmEvents.Diagnostic.PROGRESS, onDiagnosticProgress); + socket.disconnect(); + }; + }, []); + + const onStartDiag = async () => { + setDiagInProgress(true); + setSteps(items); + await postDeviceDiagnostic(device?.uuid as string) + .then(() => { + message.loading({ content: 'Diagnostic in progress...', duration: 5 }); + }) + .catch(() => { + setDiagInProgress(false); + setSteps(items); + }); + }; + + return ( + + + } + /> + + + Advanced Diagnostic + + + } + style={{ marginBottom: 10 }} + styles={{ + header: { height: 55, minHeight: 55, paddingLeft: 15 }, + body: { paddingBottom: 0 }, + }} + extra={ + + } + > + {diagInProgress && ( + + )} + + + ); +}; + +export default ExistingDeviceAdvancedDiagnostic; diff --git a/client/src/pages/Admin/Inventory/components/ConnectionTestTab.tsx b/client/src/components/DeviceConfiguration/diagnostic/ExistingDeviceConnectionTest.tsx similarity index 85% rename from client/src/pages/Admin/Inventory/components/ConnectionTestTab.tsx rename to client/src/components/DeviceConfiguration/diagnostic/ExistingDeviceConnectionTest.tsx index b73564f9..71d77aa2 100644 --- a/client/src/pages/Admin/Inventory/components/ConnectionTestTab.tsx +++ b/client/src/components/DeviceConfiguration/diagnostic/ExistingDeviceConnectionTest.tsx @@ -4,16 +4,17 @@ import { getCheckDeviceAnsibleConnection, getCheckDeviceDockerConnection, } from '@/services/rest/device'; -import { ProForm } from '@ant-design/pro-components'; import { Avatar, Button, Card, Col, Row } from 'antd'; import React, { useState } from 'react'; import { API } from 'ssm-shared-lib'; -export type ConnectionTestTabProps = { +type ConnectionTestTabProps = { device: Partial; }; -const ConnectionTestTab: React.FC = (props) => { +const ExistingDeviceConnectionTest: React.FC = ({ + device, +}) => { const [execId, setExecId] = useState(); const [dockerConnectionStatus, setDockerConnectionStatus] = useState< string | undefined @@ -22,17 +23,17 @@ const ConnectionTestTab: React.FC = (props) => { useState(); const [testStarted, setTestStarted] = useState(false); const asyncFetch = async () => { - if (!props.device.uuid) { + if (!device.uuid) { return; } setExecId(undefined); setDockerConnectionErrorMessage(undefined); setDockerConnectionStatus('running...'); setTestStarted(true); - await getCheckDeviceAnsibleConnection(props.device.uuid).then((e) => { + await getCheckDeviceAnsibleConnection(device.uuid).then((e) => { setExecId(e.data.taskId); }); - await getCheckDeviceDockerConnection(props.device.uuid).then((e) => { + await getCheckDeviceDockerConnection(device.uuid).then((e) => { setDockerConnectionStatus(e.data.connectionStatus); setDockerConnectionErrorMessage(e.data.errorMessage); }); @@ -80,4 +81,4 @@ const ConnectionTestTab: React.FC = (props) => { ); }; -export default ConnectionTestTab; +export default ExistingDeviceConnectionTest; diff --git a/client/src/components/Icons/CustomIcons.tsx b/client/src/components/Icons/CustomIcons.tsx index d42faba8..0f8395b8 100644 --- a/client/src/components/Icons/CustomIcons.tsx +++ b/client/src/components/Icons/CustomIcons.tsx @@ -2564,3 +2564,24 @@ const FileSystemSvg = React.memo((props) => ( export const FileSystem = (props: Partial) => ( ); + +const MedicalSearchDiagnosisSolidSvg = React.memo((props) => ( + + + +)); + +export const MedicalSearchDiagnosisSolid = ( + props: Partial, +) => ; diff --git a/client/src/pages/Admin/Inventory/components/ConfigurationModal.tsx b/client/src/pages/Admin/Inventory/components/ConfigurationModal.tsx index 45e6a264..dde1ed43 100644 --- a/client/src/pages/Admin/Inventory/components/ConfigurationModal.tsx +++ b/client/src/pages/Admin/Inventory/components/ConfigurationModal.tsx @@ -1,6 +1,6 @@ import { ServerEnvironmentSvg } from '@/components/Icons/CustomIcons'; import AgentConfigurationTab from '@/pages/Admin/Inventory/components/AgentConfigurationTab'; -import ConnectionTestTab from '@/pages/Admin/Inventory/components/ConnectionTestTab'; +import DiagnosticTab from '@/pages/Admin/Inventory/components/DiagnosticTab'; import DockerConfigurationForm from '@/pages/Admin/Inventory/components/DockerConfigurationForm'; import SSHConfigurationForm from '@/pages/Admin/Inventory/components/SSHConfigurationForm'; import { DockerOutlined } from '@ant-design/icons'; @@ -28,8 +28,8 @@ const ConfigurationModal: React.FC = (props) => { }, { key: '3', - label: 'Connection test', - children: , + label: 'Diagnostic', + children: , }, { key: '4', diff --git a/client/src/pages/Admin/Inventory/components/DiagnosticTab.tsx b/client/src/pages/Admin/Inventory/components/DiagnosticTab.tsx new file mode 100644 index 00000000..6cb004a6 --- /dev/null +++ b/client/src/pages/Admin/Inventory/components/DiagnosticTab.tsx @@ -0,0 +1,17 @@ +import ExistingDeviceAdvancedDiagnostic from '@/components/DeviceConfiguration/diagnostic/ExistingDeviceAdvancedDiagnostic'; +import ExistingDeviceConnectionTest from '@/components/DeviceConfiguration/diagnostic/ExistingDeviceConnectionTest'; +import React from 'react'; +import { API } from 'ssm-shared-lib'; + +export type ConnectionTestTabProps = { + device: Partial; +}; + +const DiagnosticTab: React.FC = (props) => ( + <> + + + +); + +export default DiagnosticTab; diff --git a/client/src/pages/Admin/Logs/ServerLogsColums.tsx b/client/src/pages/Admin/Logs/ServerLogsColums.tsx index fed28a0d..a7f75c44 100644 --- a/client/src/pages/Admin/Logs/ServerLogsColums.tsx +++ b/client/src/pages/Admin/Logs/ServerLogsColums.tsx @@ -64,8 +64,7 @@ const ServerLogsColumns: ProColumns[] = [ title: 'Message', dataIndex: 'msg', key: 'msg', - filters: true, - onFilter: true, + hideInSearch: true, responsive: ['xs'], }, { diff --git a/client/src/pages/Admin/Logs/index.tsx b/client/src/pages/Admin/Logs/index.tsx index ab54b399..686d6d42 100644 --- a/client/src/pages/Admin/Logs/index.tsx +++ b/client/src/pages/Admin/Logs/index.tsx @@ -37,6 +37,9 @@ const Index: React.FC = () => { if (searchParams.get('moduleId')) { form.setFieldsValue({ moduleId: searchParams.get('moduleId') }); } + if (searchParams.get('msg')) { + form.setFieldsValue({ msg: searchParams.get('msg') }); + } const logsTabItems: TabsProps['items'] = [ { diff --git a/client/src/services/rest/device.ts b/client/src/services/rest/device.ts index 21d7fdc7..8cb03d52 100644 --- a/client/src/services/rest/device.ts +++ b/client/src/services/rest/device.ts @@ -147,3 +147,16 @@ export async function updateAgentInstallMethod( }, ); } + +export async function postDeviceDiagnostic( + uuid: string, + options?: { [key: string]: any }, +) { + return request( + `/api/devices/${uuid}/check-connection/diagnostic`, + { + method: 'POST', + ...(options || {}), + }, + ); +} diff --git a/server/src/controllers/rest/devices/check-connection.ts b/server/src/controllers/rest/devices/check-connection.ts index 67cee6b3..b080a626 100644 --- a/server/src/controllers/rest/devices/check-connection.ts +++ b/server/src/controllers/rest/devices/check-connection.ts @@ -3,6 +3,7 @@ import DeviceAuthRepo from '../../../data/database/repository/DeviceAuthRepo'; import DeviceRepo from '../../../data/database/repository/DeviceRepo'; import { ForbiddenError, InternalError, NotFoundError } from '../../../middlewares/api/ApiError'; import { SuccessResponse } from '../../../middlewares/api/ApiResponse'; +import Diagnostic from '../../../modules/diagnostic/Diagnostic'; import DeviceUseCases from '../../../services/DeviceUseCases'; export const postCheckAnsibleConnection = async (req, res) => { @@ -115,3 +116,21 @@ export const getCheckDeviceAnsibleConnection = async (req, res) => { throw new InternalError(error.message); } }; + +export const postDiagnostic = async (req, res) => { + const { uuid } = req.params; + const device = await DeviceRepo.findOneByUuid(uuid); + if (!device) { + throw new NotFoundError('Device ID not found'); + } + const deviceAuth = await DeviceAuthRepo.findOneByDevice(device); + if (!deviceAuth) { + throw new NotFoundError('Device Auth not found'); + } + try { + void Diagnostic.run(device, deviceAuth); + new SuccessResponse('Get Device Diagnostic').send(res); + } catch (error: any) { + throw new InternalError(error.message); + } +}; diff --git a/server/src/controllers/rest/devices/check-connection.validator.ts b/server/src/controllers/rest/devices/check-connection.validator.ts index 149888e1..942af19e 100644 --- a/server/src/controllers/rest/devices/check-connection.validator.ts +++ b/server/src/controllers/rest/devices/check-connection.validator.ts @@ -105,3 +105,13 @@ export const getCheckDeviceAnsibleConnectionValidator = [ .withMessage('Uuid is not valid'), validator, ]; + +export const postDiagnosticValidator = [ + param('uuid') + .exists() + .notEmpty() + .withMessage('Uuid is required') + .isUUID() + .withMessage('Uuid is not valid'), + validator, +]; diff --git a/server/src/controllers/rest/logs/server.ts b/server/src/controllers/rest/logs/server.ts index 93028854..1933a027 100644 --- a/server/src/controllers/rest/logs/server.ts +++ b/server/src/controllers/rest/logs/server.ts @@ -24,7 +24,7 @@ export const getServerLogs = async (req, res) => { 'time', 'pid', 'level', - 'message', + 'msg', 'module', 'moduleId', 'moduleName', diff --git a/server/src/core/events/EventManager.ts b/server/src/core/events/EventManager.ts index ba38e4c8..fda5bce3 100644 --- a/server/src/core/events/EventManager.ts +++ b/server/src/core/events/EventManager.ts @@ -1,6 +1,7 @@ import { EventEmitter } from 'events'; import { Logger } from 'pino'; import log from '../../logger'; +import Events from './events'; type Listener = (...args: any[]) => void; @@ -10,6 +11,7 @@ export type Payload = { module: string; moduleId?: string; success?: boolean; + data?: any; }; interface EventListeners { @@ -20,7 +22,7 @@ const eventEmitter = new EventEmitter(); abstract class EventManager { protected readonly eventListeners: EventListeners; - protected currentEvent?: string; + protected currentEvent?: Events; private readonly logger: Logger; protected constructor() { @@ -28,20 +30,20 @@ abstract class EventManager { this.logger = log.child({ module: 'EventManager' }, { msgPrefix: '[EVENT_MANAGER] - ' }); } - private initializeEventListener(event: string) { + private initializeEventListener(event: Events) { if (!this.eventListeners[event]) { this.eventListeners[event] = []; } } - private addUniqueListener(event: string, listener: Listener) { + private addUniqueListener(event: Events, listener: Listener) { if (!this.eventListeners[event].includes(listener)) { this.eventListeners[event].push(listener); eventEmitter.on(event, listener); } } - on(event: string, listener: Listener) { + on(event: Events, listener: Listener) { this.logger.debug(`on: ${event}`); this.currentEvent = event; @@ -49,7 +51,7 @@ abstract class EventManager { this.addUniqueListener(event, listener); } - emit(event: string, payload?: Payload | string) { + emit(event: Events, payload?: Payload | string) { this.logger.debug(`emit: ${event}`, payload); eventEmitter.emit(event, payload); } diff --git a/server/src/core/events/events.ts b/server/src/core/events/events.ts index 8d1a7c7f..6d129d47 100644 --- a/server/src/core/events/events.ts +++ b/server/src/core/events/events.ts @@ -7,6 +7,7 @@ enum Events { UPDATED_NOTIFICATIONS = 'UPDATED_NOTIFICATIONS', ALERT = 'ALERT', VOLUME_BACKUP = 'VOLUME_BACKUP', + DIAGNOSTIC_CHECK = 'DIAGNOSTIC_CHECK', } export default Events; diff --git a/server/src/logger.ts b/server/src/logger.ts index bf519089..461b5f77 100644 --- a/server/src/logger.ts +++ b/server/src/logger.ts @@ -34,7 +34,7 @@ export const httpLoggerOptions = { }, // Define a custom success message customSuccessMessage: function (req, res) { - return `Request completed: ${req.method} - ${(req as typeof req & { originalUrl: string }).originalUrl}`; + return `Request completed (${res.statusCode}): ${req.method} - ${(req as typeof req & { originalUrl: string }).originalUrl}`; }, // Define a custom receive message diff --git a/server/src/modules/diagnostic/Diagnostic.ts b/server/src/modules/diagnostic/Diagnostic.ts new file mode 100644 index 00000000..5764b26d --- /dev/null +++ b/server/src/modules/diagnostic/Diagnostic.ts @@ -0,0 +1,248 @@ +import DockerModem from 'docker-modem'; +import Dockerode from 'dockerode'; +import { Client, ConnectConfig } from 'ssh2'; +import { SsmDeviceDiagnostic } from 'ssm-shared-lib'; +import EventManager from '../../core/events/EventManager'; +import Events from '../../core/events/events'; +import Device from '../../data/database/model/Device'; +import DeviceAuth from '../../data/database/model/DeviceAuth'; +import SSHCredentialsHelper from '../../helpers/ssh/SSHCredentialsHelper'; +import PinoLogger from '../../logger'; +import { getCustomAgent } from '../docker/core/CustomAgent'; + +const DIAGNOSTIC_SEQUENCE = Object.values(SsmDeviceDiagnostic.Checks); +const DISK_INFO_CMD = 'df -h'; +const CPU_MEM_INFO_CMD = `top -bn1 | grep "Cpu(s)" && free -m`; +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +class Diagnostic extends EventManager { + private childLogger = PinoLogger.child( + { module: 'DeviceDiagnostic' }, + { msgPrefix: '[DEVICE_DIAGNOSTIC] - ' }, + ); + + constructor() { + super(); + } + + private checkSSHConnectivity = (options: ConnectConfig) => { + return new Promise((resolve, reject) => { + const conn = new Client(); + conn + .on('ready', () => { + this.childLogger.info( + `checkSSHConnectivity - SSH connection established to ${options.host}`, + ); + resolve(true); + conn.end(); + }) + .on('error', (err) => { + this.childLogger.error( + `checkSSHConnectivity - SSH connection error to ${options.host}: ${err.message}`, + ); + reject(err); + }) + .connect(options); + }); + }; + + private checkDockerSocket = ( + options: Dockerode.DockerOptions & { modem?: any; _deviceUuid?: string }, + ) => { + return new Promise((resolve, reject) => { + const agent = getCustomAgent(this.childLogger, { + ...options.sshOptions, + timeout: 60000, + }); + + options.modem = new DockerModem({ agent }); + const dockerApi = new Dockerode({ ...options, timeout: 60000 }); + + dockerApi.ping((err) => { + if (err) { + this.childLogger.error('checkDockerSocket - Docker API ping error:', err.message); + reject(err); + } else { + this.childLogger.info('checkDockerSocket - Docker API ping successful'); + dockerApi.info((err, info) => { + if (err) { + this.childLogger.error('checkDockerSocket - Docker API info error:', err.message); + reject(err); + } else { + this.childLogger.info('checkDockerSocket - Docker API info retrieved:', info); + resolve(true); + } + }); + } + }); + }); + }; + + private checkDiskSpace = (options: ConnectConfig, path: string) => { + return new Promise((resolve, reject) => { + const conn = new Client(); + + conn.on('ready', () => { + this.childLogger.info(`checkDiskSpace - SSH connection established to ${options.host}`); + + conn.exec(`${DISK_INFO_CMD} ${path}`, (err, stream) => { + if (err) { + conn.end(); + return reject(err); + } + + let data = ''; + + stream + .on('data', (chunk: any) => { + data += chunk; + }) + .on('close', () => { + this.childLogger.info(`checkDiskSpace - Disk space info for ${path}: ${data}`); + resolve(data); + conn.end(); + }) + .stderr.on('data', (chunk) => { + this.childLogger.error(`checkDiskSpace - Disk space stderr: ${chunk}`); + }); + }); + }); + + conn.on('error', (err) => { + this.childLogger.error(`checkDiskSpace - SSH connection error: ${err.message}`); + reject(err); + }); + + conn.on('end', () => { + this.childLogger.info('checkDiskSpace - Connection ended'); + }); + + conn.connect(options); + }); + }; + + private checkCPUAndMemory = (options: ConnectConfig) => { + return new Promise((resolve, reject) => { + const conn = new Client(); + + conn.on('ready', () => { + this.childLogger.info(`checkCPUAndMemory - SSH connection established to ${options.host}`); + + conn.exec(`${CPU_MEM_INFO_CMD}`, (err, stream) => { + if (err) { + reject(err); + } else { + let data = ''; + stream + .on('data', (chunk: any) => { + data += chunk; + }) + .on('close', () => { + this.childLogger.info('checkCPUAndMemory - CPU and Memory usage:', data); + resolve(data); + conn.end(); + }) + .stderr.on('data', (chunk) => { + this.childLogger.error(`checkCPUAndMemory - CPU and Memory stderr: ${chunk}`); + }); + } + }); + }); + + conn.on('error', (err) => { + this.childLogger.error(`checkDiskSpace - SSH connection error: ${err.message}`); + reject(err); + }); + + conn.on('end', () => { + this.childLogger.info('checkDiskSpace - Connection ended'); + }); + + conn.connect(options); + }); + }; + + public run = async (device: Device, deviceAuth: DeviceAuth) => { + const defaultSshOptionsDocker = await SSHCredentialsHelper.getDockerSshConnectionOptions( + device, + deviceAuth, + ); + const defaultSshOptionsAnsible = await SSHCredentialsHelper.getSShConnection( + device, + deviceAuth, + ); + const sshOptionsDocker = { ...defaultSshOptionsDocker, timeout: 60000 }; + const sshOptionsAnsible = { ...defaultSshOptionsAnsible, timeout: 60000 }; + + let data; + for (const check of DIAGNOSTIC_SEQUENCE) { + this.childLogger.info(`Running diagnostic check: ${check}...`); + await sleep(2000); + data = undefined; + try { + switch (check) { + case SsmDeviceDiagnostic.Checks.SSH_CONNECT: + await this.checkSSHConnectivity(sshOptionsAnsible as ConnectConfig); + this.emit(Events.DIAGNOSTIC_CHECK, { + success: true, + severity: 'success', + module: 'DeviceDiagnostic', + data: { check }, + message: `✅ Ssh connect check passed on ${sshOptionsAnsible.host}:${sshOptionsAnsible.port}`, + }); + break; + case SsmDeviceDiagnostic.Checks.SSH_DOCKER_CONNECT: + await this.checkSSHConnectivity(sshOptionsDocker.sshOptions as ConnectConfig); + this.emit(Events.DIAGNOSTIC_CHECK, { + success: true, + severity: 'success', + module: 'DeviceDiagnostic', + data: { check }, + message: `✅ Ssh Docker connect check passed on ${sshOptionsDocker.host}:${sshOptionsDocker.port}`, + }); + break; + case SsmDeviceDiagnostic.Checks.DOCKER_SOCKET: + await this.checkDockerSocket(sshOptionsDocker); + this.emit(Events.DIAGNOSTIC_CHECK, { + success: true, + severity: 'success', + module: 'DeviceDiagnostic', + data: { check }, + message: `✅ Docker Socket check passed on ${sshOptionsDocker.host}:${sshOptionsDocker.port} - ${sshOptionsDocker.socketPath || '/var/run/docker.sock'}`, + }); + break; + case SsmDeviceDiagnostic.Checks.DISK_SPACE: + data = await this.checkDiskSpace(sshOptionsAnsible as ConnectConfig, '/var/lib/docker'); + this.emit(Events.DIAGNOSTIC_CHECK, { + success: true, + severity: 'success', + module: 'DeviceDiagnostic', + data: { check, diskSpaceInfo: data }, + message: `✅ Disk Space check passed on ${sshOptionsAnsible.host}:${sshOptionsAnsible.port} - ${DISK_INFO_CMD} /var/lib/docker => ${data}`, + }); + break; + case SsmDeviceDiagnostic.Checks.CPU_MEMORY_INFO: + data = await this.checkCPUAndMemory(sshOptionsAnsible as ConnectConfig); + this.emit(Events.DIAGNOSTIC_CHECK, { + success: true, + severity: 'success', + module: 'DeviceDiagnostic', + data: { check, cpuMemInfo: data }, + message: `✅ CPU & Memory check passed on ${sshOptionsAnsible.host}:${sshOptionsAnsible.port} - ${DISK_INFO_CMD} => ${data}`, + }); + break; + } + } catch (error: any) { + this.emit(Events.DIAGNOSTIC_CHECK, { + success: false, + severity: 'error', + module: 'DeviceDiagnostic', + data: { check }, + message: `❌ ${error.message}`, + }); + } + } + }; +} + +export default new Diagnostic(); diff --git a/server/src/modules/real-time/RealTime.ts b/server/src/modules/real-time/RealTime.ts index 38c74992..83abcaf1 100644 --- a/server/src/modules/real-time/RealTime.ts +++ b/server/src/modules/real-time/RealTime.ts @@ -17,21 +17,22 @@ const eventsToHandle = [ event: Events.UPDATED_NOTIFICATIONS, ssmEvent: SsmEvents.Update.NOTIFICATION_CHANGE, logMessage: 'Notifications updated', - debounceTime: 5000, }, { event: Events.ALERT, ssmEvent: SsmEvents.Alert.NEW_ALERT, logMessage: 'Alert sent', - debounceTime: 5000, }, { event: Events.VOLUME_BACKUP, ssmEvent: SsmEvents.VolumeBackup.PROGRESS, logMessage: 'Volume backup progress', - debounceTime: 5000, }, - // Add any additional events here + { + event: Events.DIAGNOSTIC_CHECK, + ssmEvent: SsmEvents.Diagnostic.PROGRESS, + logMessage: 'Device Diagnostic progress', + }, ]; class RealTimeEngine extends EventManager { @@ -57,6 +58,14 @@ class RealTimeEngine extends EventManager { }, debounceTime); } + private createEmitter(eventName: string, logMessage: string) { + return (payload: any) => { + const io = App.getSocket().getIo(); + this.childLogger.debug(`${logMessage}`); + io.emit(eventName, payload); + }; + } + public init() { try { this.childLogger.info('Init...'); @@ -65,8 +74,12 @@ class RealTimeEngine extends EventManager { this.childLogger.debug( `Registering event ${event} with ssmEvent ${ssmEvent} and debounceTime ${debounceTime}`, ); - const debouncedEmitter = this.createDebouncedEmitter(ssmEvent, logMessage, debounceTime); - this.on(event, (payload: any) => debouncedEmitter(payload)); + const emitter = + debounceTime !== undefined + ? this.createDebouncedEmitter(ssmEvent, logMessage, debounceTime) + : this.createEmitter(ssmEvent, logMessage); + + this.on(event, (payload: any) => emitter(payload)); }); } catch (error: any) { this.childLogger.error(error); diff --git a/server/src/routes/devices.ts b/server/src/routes/devices.ts index b05b9aa8..46ab18b3 100644 --- a/server/src/routes/devices.ts +++ b/server/src/routes/devices.ts @@ -5,12 +5,14 @@ import { getCheckDeviceDockerConnection, postCheckAnsibleConnection, postCheckDockerConnection, + postDiagnostic, } from '../controllers/rest/devices/check-connection'; import { getCheckDeviceAnsibleConnectionValidator, getCheckDeviceDockerConnectionValidator, postCheckAnsibleConnectionValidator, postCheckDockerConnectionValidator, + postDiagnosticValidator, } from '../controllers/rest/devices/check-connection.validator'; import { addDevice, @@ -67,7 +69,6 @@ router.post(`/:uuid`, updateDeviceAndAddDeviceStatValidator, updateDeviceAndAddD router.post('/', addDeviceAutoValidator, addDeviceAuto); router.use(passport.authenticate('jwt', { session: false })); - router.post( '/check-connection/ansible', postCheckAnsibleConnectionValidator, @@ -108,6 +109,7 @@ router router .route(`/:uuid/check-connection/docker`) .get(getCheckDeviceDockerConnectionValidator, getCheckDeviceDockerConnection); +router.post('/:uuid/check-connection/diagnostic', postDiagnosticValidator, postDiagnostic); router.route('/').put(addDeviceValidator, addDevice).get(getDevices); router.route('/all').get(getAllDevices); diff --git a/server/src/services/DeviceUseCases.ts b/server/src/services/DeviceUseCases.ts index 999ffc3c..c7563395 100644 --- a/server/src/services/DeviceUseCases.ts +++ b/server/src/services/DeviceUseCases.ts @@ -259,17 +259,19 @@ async function checkDeviceDockerConnection(device: Device, deviceAuth: DeviceAut const options = await SSHCredentialsHelper.getDockerSshConnectionOptions(device, deviceAuth); const agent = getCustomAgent(logger, { ...options.sshOptions, + timeout: 60000, }); options.modem = new DockerModem({ agent: agent, }); - const dockerApi = new Dockerode(options); + const dockerApi = new Dockerode({ ...options, timeout: 60000 }); await dockerApi.ping(); await dockerApi.info(); return { status: 'successful', }; } catch (error: any) { + logger.error(error); return { status: 'failed', message: error.message, diff --git a/shared-lib/src/enums/diagnostic.ts b/shared-lib/src/enums/diagnostic.ts new file mode 100644 index 00000000..22702693 --- /dev/null +++ b/shared-lib/src/enums/diagnostic.ts @@ -0,0 +1,7 @@ +export enum Checks { + SSH_CONNECT = 'SSH_CONNECT', + SSH_DOCKER_CONNECT = 'SSH_DOCKER_CONNECT', + DOCKER_SOCKET = 'DOCKER_SOCKET', + DISK_SPACE = 'DISK_SPACE', + CPU_MEMORY_INFO= 'CPU_MEMORY_INFO' +} diff --git a/shared-lib/src/index.ts b/shared-lib/src/index.ts index 37403941..29d49a33 100644 --- a/shared-lib/src/index.ts +++ b/shared-lib/src/index.ts @@ -13,3 +13,4 @@ export * as SsmEvents from './types/events'; export * as SsmAgent from './enums/agent'; export * as SsmAlert from './enums/alert'; export * as SsmGit from './enums/git'; +export * as SsmDeviceDiagnostic from './enums/diagnostic' diff --git a/shared-lib/src/types/events.ts b/shared-lib/src/types/events.ts index 48e17aec..ff0b627c 100644 --- a/shared-lib/src/types/events.ts +++ b/shared-lib/src/types/events.ts @@ -30,3 +30,7 @@ export enum Alert { export enum VolumeBackup { PROGRESS = 'volume:backup:progress', } + +export enum Diagnostic { + PROGRESS = 'diagnostic:progress' +}