From bd758fd4f30143df9ac378fa676d696806a3d620 Mon Sep 17 00:00:00 2001 From: SquirrelDevelopper Date: Fri, 15 Nov 2024 13:21:10 +0100 Subject: [PATCH 1/2] Add advanced diagnostic checks for device connections This commit introduces advanced diagnostic checks for SSH and Docker connections in the server and client. It includes updates to the EventManager, new components for diagnostics in the client, and necessary validation in the server. --- .../ExistingDeviceAdvancedDiagnostic.tsx | 144 ++++++++++ .../ExistingDeviceConnectionTest.tsx} | 15 +- client/src/components/Icons/CustomIcons.tsx | 21 ++ .../components/ConfigurationModal.tsx | 6 +- .../Inventory/components/DiagnosticTab.tsx | 17 ++ .../src/pages/Admin/Logs/ServerLogsColums.tsx | 3 +- client/src/pages/Admin/Logs/index.tsx | 3 + client/src/services/rest/device.ts | 13 + .../rest/devices/check-connection.ts | 19 ++ .../devices/check-connection.validator.ts | 10 + server/src/controllers/rest/logs/server.ts | 2 +- server/src/core/events/EventManager.ts | 12 +- server/src/core/events/events.ts | 1 + server/src/logger.ts | 2 +- server/src/modules/diagnostic/Diagnostic.ts | 248 ++++++++++++++++++ server/src/modules/real-time/RealTime.ts | 25 +- server/src/routes/devices.ts | 4 +- server/src/services/DeviceUseCases.ts | 4 +- shared-lib/src/enums/diagnostic.ts | 7 + shared-lib/src/index.ts | 1 + shared-lib/src/types/events.ts | 4 + 21 files changed, 534 insertions(+), 27 deletions(-) create mode 100644 client/src/components/DeviceConfiguration/diagnostic/ExistingDeviceAdvancedDiagnostic.tsx rename client/src/{pages/Admin/Inventory/components/ConnectionTestTab.tsx => components/DeviceConfiguration/diagnostic/ExistingDeviceConnectionTest.tsx} (85%) create mode 100644 client/src/pages/Admin/Inventory/components/DiagnosticTab.tsx create mode 100644 server/src/modules/diagnostic/Diagnostic.ts create mode 100644 shared-lib/src/enums/diagnostic.ts diff --git a/client/src/components/DeviceConfiguration/diagnostic/ExistingDeviceAdvancedDiagnostic.tsx b/client/src/components/DeviceConfiguration/diagnostic/ExistingDeviceAdvancedDiagnostic.tsx new file mode 100644 index 00000000..b7913d71 --- /dev/null +++ b/client/src/components/DeviceConfiguration/diagnostic/ExistingDeviceAdvancedDiagnostic.tsx @@ -0,0 +1,144 @@ +import { MedicalSearchDiagnosisSolid } from '@/components/Icons/CustomIcons'; +import { postDeviceDiagnostic } from '@/services/rest/device'; +import { socket } from '@/socket'; +import { history } from '@umijs/max'; +import { + Avatar, + Button, + Card, + Col, + message, + Row, + StepProps, + Steps, +} from 'antd'; +import React, { useEffect } from 'react'; +import { API, SsmDeviceDiagnostic, SsmEvents } from 'ssm-shared-lib'; + +type ExistingDeviceAdvancedDiagnosticProps = { + device: Partial; +}; + +const items: (StepProps & any)[] = [ + { + title: 'Ssh Connect', + description: 'Waiting...', + key: SsmDeviceDiagnostic.Checks.SSH_CONNECT, + }, + { + title: 'Ssh Docker Connect', + description: 'Waiting...', + key: SsmDeviceDiagnostic.Checks.SSH_DOCKER_CONNECT, + }, + { + title: 'Ssh Docker Socket Connectivity', + description: 'Waiting...', + key: SsmDeviceDiagnostic.Checks.DOCKER_SOCKET, + }, + { + title: 'Disk Space', + description: 'Waiting...', + key: SsmDeviceDiagnostic.Checks.DISK_SPACE, + }, + { + title: 'Memory & CPU', + description: 'Waiting...', + key: SsmDeviceDiagnostic.Checks.CPU_MEMORY_INFO, + }, +]; + +const ExistingDeviceAdvancedDiagnostic: React.FC< + ExistingDeviceAdvancedDiagnosticProps +> = ({ device }) => { + const [diagInProgress, setDiagInProgress] = React.useState(false); + const [steps, setSteps] = React.useState(items); + const onDiagnosticProgress = (payload: any) => { + setSteps((prevSteps) => + prevSteps.map((step) => + step.key === payload?.data?.check + ? { + ...step, + description: payload.message || step.description, // Update description if provided + status: !payload.success ? 'error' : 'success', + } + : step, + ), + ); + }; + + useEffect(() => { + socket.connect(); + socket.on(SsmEvents.Diagnostic.PROGRESS, onDiagnosticProgress); + + return () => { + socket.off(SsmEvents.Diagnostic.PROGRESS, onDiagnosticProgress); + socket.disconnect(); + }; + }, []); + + const onStartDiag = async () => { + setDiagInProgress(true); + setSteps(items); + await postDeviceDiagnostic(device?.uuid as string) + .then(() => { + message.loading({ content: 'Diagnostic in progress...', duration: 5 }); + }) + .catch(() => { + setDiagInProgress(false); + setSteps(items); + }); + }; + + return ( + + + } + /> + + + Advanced Diagnostic + + + } + style={{ marginBottom: 10 }} + styles={{ + header: { height: 55, minHeight: 55, paddingLeft: 15 }, + body: { paddingBottom: 0 }, + }} + extra={ + + } + > + {diagInProgress && ( + + )} + + + ); +}; + +export default ExistingDeviceAdvancedDiagnostic; diff --git a/client/src/pages/Admin/Inventory/components/ConnectionTestTab.tsx b/client/src/components/DeviceConfiguration/diagnostic/ExistingDeviceConnectionTest.tsx similarity index 85% rename from client/src/pages/Admin/Inventory/components/ConnectionTestTab.tsx rename to client/src/components/DeviceConfiguration/diagnostic/ExistingDeviceConnectionTest.tsx index b73564f9..71d77aa2 100644 --- a/client/src/pages/Admin/Inventory/components/ConnectionTestTab.tsx +++ b/client/src/components/DeviceConfiguration/diagnostic/ExistingDeviceConnectionTest.tsx @@ -4,16 +4,17 @@ import { getCheckDeviceAnsibleConnection, getCheckDeviceDockerConnection, } from '@/services/rest/device'; -import { ProForm } from '@ant-design/pro-components'; import { Avatar, Button, Card, Col, Row } from 'antd'; import React, { useState } from 'react'; import { API } from 'ssm-shared-lib'; -export type ConnectionTestTabProps = { +type ConnectionTestTabProps = { device: Partial; }; -const ConnectionTestTab: React.FC = (props) => { +const ExistingDeviceConnectionTest: React.FC = ({ + device, +}) => { const [execId, setExecId] = useState(); const [dockerConnectionStatus, setDockerConnectionStatus] = useState< string | undefined @@ -22,17 +23,17 @@ const ConnectionTestTab: React.FC = (props) => { useState(); const [testStarted, setTestStarted] = useState(false); const asyncFetch = async () => { - if (!props.device.uuid) { + if (!device.uuid) { return; } setExecId(undefined); setDockerConnectionErrorMessage(undefined); setDockerConnectionStatus('running...'); setTestStarted(true); - await getCheckDeviceAnsibleConnection(props.device.uuid).then((e) => { + await getCheckDeviceAnsibleConnection(device.uuid).then((e) => { setExecId(e.data.taskId); }); - await getCheckDeviceDockerConnection(props.device.uuid).then((e) => { + await getCheckDeviceDockerConnection(device.uuid).then((e) => { setDockerConnectionStatus(e.data.connectionStatus); setDockerConnectionErrorMessage(e.data.errorMessage); }); @@ -80,4 +81,4 @@ const ConnectionTestTab: React.FC = (props) => { ); }; -export default ConnectionTestTab; +export default ExistingDeviceConnectionTest; diff --git a/client/src/components/Icons/CustomIcons.tsx b/client/src/components/Icons/CustomIcons.tsx index d42faba8..0f8395b8 100644 --- a/client/src/components/Icons/CustomIcons.tsx +++ b/client/src/components/Icons/CustomIcons.tsx @@ -2564,3 +2564,24 @@ const FileSystemSvg = React.memo((props) => ( export const FileSystem = (props: Partial) => ( ); + +const MedicalSearchDiagnosisSolidSvg = React.memo((props) => ( + + + +)); + +export const MedicalSearchDiagnosisSolid = ( + props: Partial, +) => ; diff --git a/client/src/pages/Admin/Inventory/components/ConfigurationModal.tsx b/client/src/pages/Admin/Inventory/components/ConfigurationModal.tsx index 45e6a264..dde1ed43 100644 --- a/client/src/pages/Admin/Inventory/components/ConfigurationModal.tsx +++ b/client/src/pages/Admin/Inventory/components/ConfigurationModal.tsx @@ -1,6 +1,6 @@ import { ServerEnvironmentSvg } from '@/components/Icons/CustomIcons'; import AgentConfigurationTab from '@/pages/Admin/Inventory/components/AgentConfigurationTab'; -import ConnectionTestTab from '@/pages/Admin/Inventory/components/ConnectionTestTab'; +import DiagnosticTab from '@/pages/Admin/Inventory/components/DiagnosticTab'; import DockerConfigurationForm from '@/pages/Admin/Inventory/components/DockerConfigurationForm'; import SSHConfigurationForm from '@/pages/Admin/Inventory/components/SSHConfigurationForm'; import { DockerOutlined } from '@ant-design/icons'; @@ -28,8 +28,8 @@ const ConfigurationModal: React.FC = (props) => { }, { key: '3', - label: 'Connection test', - children: , + label: 'Diagnostic', + children: , }, { key: '4', diff --git a/client/src/pages/Admin/Inventory/components/DiagnosticTab.tsx b/client/src/pages/Admin/Inventory/components/DiagnosticTab.tsx new file mode 100644 index 00000000..6cb004a6 --- /dev/null +++ b/client/src/pages/Admin/Inventory/components/DiagnosticTab.tsx @@ -0,0 +1,17 @@ +import ExistingDeviceAdvancedDiagnostic from '@/components/DeviceConfiguration/diagnostic/ExistingDeviceAdvancedDiagnostic'; +import ExistingDeviceConnectionTest from '@/components/DeviceConfiguration/diagnostic/ExistingDeviceConnectionTest'; +import React from 'react'; +import { API } from 'ssm-shared-lib'; + +export type ConnectionTestTabProps = { + device: Partial; +}; + +const DiagnosticTab: React.FC = (props) => ( + <> + + + +); + +export default DiagnosticTab; diff --git a/client/src/pages/Admin/Logs/ServerLogsColums.tsx b/client/src/pages/Admin/Logs/ServerLogsColums.tsx index fed28a0d..a7f75c44 100644 --- a/client/src/pages/Admin/Logs/ServerLogsColums.tsx +++ b/client/src/pages/Admin/Logs/ServerLogsColums.tsx @@ -64,8 +64,7 @@ const ServerLogsColumns: ProColumns[] = [ title: 'Message', dataIndex: 'msg', key: 'msg', - filters: true, - onFilter: true, + hideInSearch: true, responsive: ['xs'], }, { diff --git a/client/src/pages/Admin/Logs/index.tsx b/client/src/pages/Admin/Logs/index.tsx index ab54b399..686d6d42 100644 --- a/client/src/pages/Admin/Logs/index.tsx +++ b/client/src/pages/Admin/Logs/index.tsx @@ -37,6 +37,9 @@ const Index: React.FC = () => { if (searchParams.get('moduleId')) { form.setFieldsValue({ moduleId: searchParams.get('moduleId') }); } + if (searchParams.get('msg')) { + form.setFieldsValue({ msg: searchParams.get('msg') }); + } const logsTabItems: TabsProps['items'] = [ { diff --git a/client/src/services/rest/device.ts b/client/src/services/rest/device.ts index 21d7fdc7..8cb03d52 100644 --- a/client/src/services/rest/device.ts +++ b/client/src/services/rest/device.ts @@ -147,3 +147,16 @@ export async function updateAgentInstallMethod( }, ); } + +export async function postDeviceDiagnostic( + uuid: string, + options?: { [key: string]: any }, +) { + return request( + `/api/devices/${uuid}/check-connection/diagnostic`, + { + method: 'POST', + ...(options || {}), + }, + ); +} diff --git a/server/src/controllers/rest/devices/check-connection.ts b/server/src/controllers/rest/devices/check-connection.ts index 67cee6b3..b080a626 100644 --- a/server/src/controllers/rest/devices/check-connection.ts +++ b/server/src/controllers/rest/devices/check-connection.ts @@ -3,6 +3,7 @@ import DeviceAuthRepo from '../../../data/database/repository/DeviceAuthRepo'; import DeviceRepo from '../../../data/database/repository/DeviceRepo'; import { ForbiddenError, InternalError, NotFoundError } from '../../../middlewares/api/ApiError'; import { SuccessResponse } from '../../../middlewares/api/ApiResponse'; +import Diagnostic from '../../../modules/diagnostic/Diagnostic'; import DeviceUseCases from '../../../services/DeviceUseCases'; export const postCheckAnsibleConnection = async (req, res) => { @@ -115,3 +116,21 @@ export const getCheckDeviceAnsibleConnection = async (req, res) => { throw new InternalError(error.message); } }; + +export const postDiagnostic = async (req, res) => { + const { uuid } = req.params; + const device = await DeviceRepo.findOneByUuid(uuid); + if (!device) { + throw new NotFoundError('Device ID not found'); + } + const deviceAuth = await DeviceAuthRepo.findOneByDevice(device); + if (!deviceAuth) { + throw new NotFoundError('Device Auth not found'); + } + try { + void Diagnostic.run(device, deviceAuth); + new SuccessResponse('Get Device Diagnostic').send(res); + } catch (error: any) { + throw new InternalError(error.message); + } +}; diff --git a/server/src/controllers/rest/devices/check-connection.validator.ts b/server/src/controllers/rest/devices/check-connection.validator.ts index 149888e1..942af19e 100644 --- a/server/src/controllers/rest/devices/check-connection.validator.ts +++ b/server/src/controllers/rest/devices/check-connection.validator.ts @@ -105,3 +105,13 @@ export const getCheckDeviceAnsibleConnectionValidator = [ .withMessage('Uuid is not valid'), validator, ]; + +export const postDiagnosticValidator = [ + param('uuid') + .exists() + .notEmpty() + .withMessage('Uuid is required') + .isUUID() + .withMessage('Uuid is not valid'), + validator, +]; diff --git a/server/src/controllers/rest/logs/server.ts b/server/src/controllers/rest/logs/server.ts index 93028854..1933a027 100644 --- a/server/src/controllers/rest/logs/server.ts +++ b/server/src/controllers/rest/logs/server.ts @@ -24,7 +24,7 @@ export const getServerLogs = async (req, res) => { 'time', 'pid', 'level', - 'message', + 'msg', 'module', 'moduleId', 'moduleName', diff --git a/server/src/core/events/EventManager.ts b/server/src/core/events/EventManager.ts index ba38e4c8..fda5bce3 100644 --- a/server/src/core/events/EventManager.ts +++ b/server/src/core/events/EventManager.ts @@ -1,6 +1,7 @@ import { EventEmitter } from 'events'; import { Logger } from 'pino'; import log from '../../logger'; +import Events from './events'; type Listener = (...args: any[]) => void; @@ -10,6 +11,7 @@ export type Payload = { module: string; moduleId?: string; success?: boolean; + data?: any; }; interface EventListeners { @@ -20,7 +22,7 @@ const eventEmitter = new EventEmitter(); abstract class EventManager { protected readonly eventListeners: EventListeners; - protected currentEvent?: string; + protected currentEvent?: Events; private readonly logger: Logger; protected constructor() { @@ -28,20 +30,20 @@ abstract class EventManager { this.logger = log.child({ module: 'EventManager' }, { msgPrefix: '[EVENT_MANAGER] - ' }); } - private initializeEventListener(event: string) { + private initializeEventListener(event: Events) { if (!this.eventListeners[event]) { this.eventListeners[event] = []; } } - private addUniqueListener(event: string, listener: Listener) { + private addUniqueListener(event: Events, listener: Listener) { if (!this.eventListeners[event].includes(listener)) { this.eventListeners[event].push(listener); eventEmitter.on(event, listener); } } - on(event: string, listener: Listener) { + on(event: Events, listener: Listener) { this.logger.debug(`on: ${event}`); this.currentEvent = event; @@ -49,7 +51,7 @@ abstract class EventManager { this.addUniqueListener(event, listener); } - emit(event: string, payload?: Payload | string) { + emit(event: Events, payload?: Payload | string) { this.logger.debug(`emit: ${event}`, payload); eventEmitter.emit(event, payload); } diff --git a/server/src/core/events/events.ts b/server/src/core/events/events.ts index 8d1a7c7f..6d129d47 100644 --- a/server/src/core/events/events.ts +++ b/server/src/core/events/events.ts @@ -7,6 +7,7 @@ enum Events { UPDATED_NOTIFICATIONS = 'UPDATED_NOTIFICATIONS', ALERT = 'ALERT', VOLUME_BACKUP = 'VOLUME_BACKUP', + DIAGNOSTIC_CHECK = 'DIAGNOSTIC_CHECK', } export default Events; diff --git a/server/src/logger.ts b/server/src/logger.ts index bf519089..461b5f77 100644 --- a/server/src/logger.ts +++ b/server/src/logger.ts @@ -34,7 +34,7 @@ export const httpLoggerOptions = { }, // Define a custom success message customSuccessMessage: function (req, res) { - return `Request completed: ${req.method} - ${(req as typeof req & { originalUrl: string }).originalUrl}`; + return `Request completed (${res.statusCode}): ${req.method} - ${(req as typeof req & { originalUrl: string }).originalUrl}`; }, // Define a custom receive message diff --git a/server/src/modules/diagnostic/Diagnostic.ts b/server/src/modules/diagnostic/Diagnostic.ts new file mode 100644 index 00000000..5764b26d --- /dev/null +++ b/server/src/modules/diagnostic/Diagnostic.ts @@ -0,0 +1,248 @@ +import DockerModem from 'docker-modem'; +import Dockerode from 'dockerode'; +import { Client, ConnectConfig } from 'ssh2'; +import { SsmDeviceDiagnostic } from 'ssm-shared-lib'; +import EventManager from '../../core/events/EventManager'; +import Events from '../../core/events/events'; +import Device from '../../data/database/model/Device'; +import DeviceAuth from '../../data/database/model/DeviceAuth'; +import SSHCredentialsHelper from '../../helpers/ssh/SSHCredentialsHelper'; +import PinoLogger from '../../logger'; +import { getCustomAgent } from '../docker/core/CustomAgent'; + +const DIAGNOSTIC_SEQUENCE = Object.values(SsmDeviceDiagnostic.Checks); +const DISK_INFO_CMD = 'df -h'; +const CPU_MEM_INFO_CMD = `top -bn1 | grep "Cpu(s)" && free -m`; +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +class Diagnostic extends EventManager { + private childLogger = PinoLogger.child( + { module: 'DeviceDiagnostic' }, + { msgPrefix: '[DEVICE_DIAGNOSTIC] - ' }, + ); + + constructor() { + super(); + } + + private checkSSHConnectivity = (options: ConnectConfig) => { + return new Promise((resolve, reject) => { + const conn = new Client(); + conn + .on('ready', () => { + this.childLogger.info( + `checkSSHConnectivity - SSH connection established to ${options.host}`, + ); + resolve(true); + conn.end(); + }) + .on('error', (err) => { + this.childLogger.error( + `checkSSHConnectivity - SSH connection error to ${options.host}: ${err.message}`, + ); + reject(err); + }) + .connect(options); + }); + }; + + private checkDockerSocket = ( + options: Dockerode.DockerOptions & { modem?: any; _deviceUuid?: string }, + ) => { + return new Promise((resolve, reject) => { + const agent = getCustomAgent(this.childLogger, { + ...options.sshOptions, + timeout: 60000, + }); + + options.modem = new DockerModem({ agent }); + const dockerApi = new Dockerode({ ...options, timeout: 60000 }); + + dockerApi.ping((err) => { + if (err) { + this.childLogger.error('checkDockerSocket - Docker API ping error:', err.message); + reject(err); + } else { + this.childLogger.info('checkDockerSocket - Docker API ping successful'); + dockerApi.info((err, info) => { + if (err) { + this.childLogger.error('checkDockerSocket - Docker API info error:', err.message); + reject(err); + } else { + this.childLogger.info('checkDockerSocket - Docker API info retrieved:', info); + resolve(true); + } + }); + } + }); + }); + }; + + private checkDiskSpace = (options: ConnectConfig, path: string) => { + return new Promise((resolve, reject) => { + const conn = new Client(); + + conn.on('ready', () => { + this.childLogger.info(`checkDiskSpace - SSH connection established to ${options.host}`); + + conn.exec(`${DISK_INFO_CMD} ${path}`, (err, stream) => { + if (err) { + conn.end(); + return reject(err); + } + + let data = ''; + + stream + .on('data', (chunk: any) => { + data += chunk; + }) + .on('close', () => { + this.childLogger.info(`checkDiskSpace - Disk space info for ${path}: ${data}`); + resolve(data); + conn.end(); + }) + .stderr.on('data', (chunk) => { + this.childLogger.error(`checkDiskSpace - Disk space stderr: ${chunk}`); + }); + }); + }); + + conn.on('error', (err) => { + this.childLogger.error(`checkDiskSpace - SSH connection error: ${err.message}`); + reject(err); + }); + + conn.on('end', () => { + this.childLogger.info('checkDiskSpace - Connection ended'); + }); + + conn.connect(options); + }); + }; + + private checkCPUAndMemory = (options: ConnectConfig) => { + return new Promise((resolve, reject) => { + const conn = new Client(); + + conn.on('ready', () => { + this.childLogger.info(`checkCPUAndMemory - SSH connection established to ${options.host}`); + + conn.exec(`${CPU_MEM_INFO_CMD}`, (err, stream) => { + if (err) { + reject(err); + } else { + let data = ''; + stream + .on('data', (chunk: any) => { + data += chunk; + }) + .on('close', () => { + this.childLogger.info('checkCPUAndMemory - CPU and Memory usage:', data); + resolve(data); + conn.end(); + }) + .stderr.on('data', (chunk) => { + this.childLogger.error(`checkCPUAndMemory - CPU and Memory stderr: ${chunk}`); + }); + } + }); + }); + + conn.on('error', (err) => { + this.childLogger.error(`checkDiskSpace - SSH connection error: ${err.message}`); + reject(err); + }); + + conn.on('end', () => { + this.childLogger.info('checkDiskSpace - Connection ended'); + }); + + conn.connect(options); + }); + }; + + public run = async (device: Device, deviceAuth: DeviceAuth) => { + const defaultSshOptionsDocker = await SSHCredentialsHelper.getDockerSshConnectionOptions( + device, + deviceAuth, + ); + const defaultSshOptionsAnsible = await SSHCredentialsHelper.getSShConnection( + device, + deviceAuth, + ); + const sshOptionsDocker = { ...defaultSshOptionsDocker, timeout: 60000 }; + const sshOptionsAnsible = { ...defaultSshOptionsAnsible, timeout: 60000 }; + + let data; + for (const check of DIAGNOSTIC_SEQUENCE) { + this.childLogger.info(`Running diagnostic check: ${check}...`); + await sleep(2000); + data = undefined; + try { + switch (check) { + case SsmDeviceDiagnostic.Checks.SSH_CONNECT: + await this.checkSSHConnectivity(sshOptionsAnsible as ConnectConfig); + this.emit(Events.DIAGNOSTIC_CHECK, { + success: true, + severity: 'success', + module: 'DeviceDiagnostic', + data: { check }, + message: `✅ Ssh connect check passed on ${sshOptionsAnsible.host}:${sshOptionsAnsible.port}`, + }); + break; + case SsmDeviceDiagnostic.Checks.SSH_DOCKER_CONNECT: + await this.checkSSHConnectivity(sshOptionsDocker.sshOptions as ConnectConfig); + this.emit(Events.DIAGNOSTIC_CHECK, { + success: true, + severity: 'success', + module: 'DeviceDiagnostic', + data: { check }, + message: `✅ Ssh Docker connect check passed on ${sshOptionsDocker.host}:${sshOptionsDocker.port}`, + }); + break; + case SsmDeviceDiagnostic.Checks.DOCKER_SOCKET: + await this.checkDockerSocket(sshOptionsDocker); + this.emit(Events.DIAGNOSTIC_CHECK, { + success: true, + severity: 'success', + module: 'DeviceDiagnostic', + data: { check }, + message: `✅ Docker Socket check passed on ${sshOptionsDocker.host}:${sshOptionsDocker.port} - ${sshOptionsDocker.socketPath || '/var/run/docker.sock'}`, + }); + break; + case SsmDeviceDiagnostic.Checks.DISK_SPACE: + data = await this.checkDiskSpace(sshOptionsAnsible as ConnectConfig, '/var/lib/docker'); + this.emit(Events.DIAGNOSTIC_CHECK, { + success: true, + severity: 'success', + module: 'DeviceDiagnostic', + data: { check, diskSpaceInfo: data }, + message: `✅ Disk Space check passed on ${sshOptionsAnsible.host}:${sshOptionsAnsible.port} - ${DISK_INFO_CMD} /var/lib/docker => ${data}`, + }); + break; + case SsmDeviceDiagnostic.Checks.CPU_MEMORY_INFO: + data = await this.checkCPUAndMemory(sshOptionsAnsible as ConnectConfig); + this.emit(Events.DIAGNOSTIC_CHECK, { + success: true, + severity: 'success', + module: 'DeviceDiagnostic', + data: { check, cpuMemInfo: data }, + message: `✅ CPU & Memory check passed on ${sshOptionsAnsible.host}:${sshOptionsAnsible.port} - ${DISK_INFO_CMD} => ${data}`, + }); + break; + } + } catch (error: any) { + this.emit(Events.DIAGNOSTIC_CHECK, { + success: false, + severity: 'error', + module: 'DeviceDiagnostic', + data: { check }, + message: `❌ ${error.message}`, + }); + } + } + }; +} + +export default new Diagnostic(); diff --git a/server/src/modules/real-time/RealTime.ts b/server/src/modules/real-time/RealTime.ts index 38c74992..83abcaf1 100644 --- a/server/src/modules/real-time/RealTime.ts +++ b/server/src/modules/real-time/RealTime.ts @@ -17,21 +17,22 @@ const eventsToHandle = [ event: Events.UPDATED_NOTIFICATIONS, ssmEvent: SsmEvents.Update.NOTIFICATION_CHANGE, logMessage: 'Notifications updated', - debounceTime: 5000, }, { event: Events.ALERT, ssmEvent: SsmEvents.Alert.NEW_ALERT, logMessage: 'Alert sent', - debounceTime: 5000, }, { event: Events.VOLUME_BACKUP, ssmEvent: SsmEvents.VolumeBackup.PROGRESS, logMessage: 'Volume backup progress', - debounceTime: 5000, }, - // Add any additional events here + { + event: Events.DIAGNOSTIC_CHECK, + ssmEvent: SsmEvents.Diagnostic.PROGRESS, + logMessage: 'Device Diagnostic progress', + }, ]; class RealTimeEngine extends EventManager { @@ -57,6 +58,14 @@ class RealTimeEngine extends EventManager { }, debounceTime); } + private createEmitter(eventName: string, logMessage: string) { + return (payload: any) => { + const io = App.getSocket().getIo(); + this.childLogger.debug(`${logMessage}`); + io.emit(eventName, payload); + }; + } + public init() { try { this.childLogger.info('Init...'); @@ -65,8 +74,12 @@ class RealTimeEngine extends EventManager { this.childLogger.debug( `Registering event ${event} with ssmEvent ${ssmEvent} and debounceTime ${debounceTime}`, ); - const debouncedEmitter = this.createDebouncedEmitter(ssmEvent, logMessage, debounceTime); - this.on(event, (payload: any) => debouncedEmitter(payload)); + const emitter = + debounceTime !== undefined + ? this.createDebouncedEmitter(ssmEvent, logMessage, debounceTime) + : this.createEmitter(ssmEvent, logMessage); + + this.on(event, (payload: any) => emitter(payload)); }); } catch (error: any) { this.childLogger.error(error); diff --git a/server/src/routes/devices.ts b/server/src/routes/devices.ts index b05b9aa8..46ab18b3 100644 --- a/server/src/routes/devices.ts +++ b/server/src/routes/devices.ts @@ -5,12 +5,14 @@ import { getCheckDeviceDockerConnection, postCheckAnsibleConnection, postCheckDockerConnection, + postDiagnostic, } from '../controllers/rest/devices/check-connection'; import { getCheckDeviceAnsibleConnectionValidator, getCheckDeviceDockerConnectionValidator, postCheckAnsibleConnectionValidator, postCheckDockerConnectionValidator, + postDiagnosticValidator, } from '../controllers/rest/devices/check-connection.validator'; import { addDevice, @@ -67,7 +69,6 @@ router.post(`/:uuid`, updateDeviceAndAddDeviceStatValidator, updateDeviceAndAddD router.post('/', addDeviceAutoValidator, addDeviceAuto); router.use(passport.authenticate('jwt', { session: false })); - router.post( '/check-connection/ansible', postCheckAnsibleConnectionValidator, @@ -108,6 +109,7 @@ router router .route(`/:uuid/check-connection/docker`) .get(getCheckDeviceDockerConnectionValidator, getCheckDeviceDockerConnection); +router.post('/:uuid/check-connection/diagnostic', postDiagnosticValidator, postDiagnostic); router.route('/').put(addDeviceValidator, addDevice).get(getDevices); router.route('/all').get(getAllDevices); diff --git a/server/src/services/DeviceUseCases.ts b/server/src/services/DeviceUseCases.ts index 999ffc3c..c7563395 100644 --- a/server/src/services/DeviceUseCases.ts +++ b/server/src/services/DeviceUseCases.ts @@ -259,17 +259,19 @@ async function checkDeviceDockerConnection(device: Device, deviceAuth: DeviceAut const options = await SSHCredentialsHelper.getDockerSshConnectionOptions(device, deviceAuth); const agent = getCustomAgent(logger, { ...options.sshOptions, + timeout: 60000, }); options.modem = new DockerModem({ agent: agent, }); - const dockerApi = new Dockerode(options); + const dockerApi = new Dockerode({ ...options, timeout: 60000 }); await dockerApi.ping(); await dockerApi.info(); return { status: 'successful', }; } catch (error: any) { + logger.error(error); return { status: 'failed', message: error.message, diff --git a/shared-lib/src/enums/diagnostic.ts b/shared-lib/src/enums/diagnostic.ts new file mode 100644 index 00000000..22702693 --- /dev/null +++ b/shared-lib/src/enums/diagnostic.ts @@ -0,0 +1,7 @@ +export enum Checks { + SSH_CONNECT = 'SSH_CONNECT', + SSH_DOCKER_CONNECT = 'SSH_DOCKER_CONNECT', + DOCKER_SOCKET = 'DOCKER_SOCKET', + DISK_SPACE = 'DISK_SPACE', + CPU_MEMORY_INFO= 'CPU_MEMORY_INFO' +} diff --git a/shared-lib/src/index.ts b/shared-lib/src/index.ts index 6cd6a171..6a9ff135 100644 --- a/shared-lib/src/index.ts +++ b/shared-lib/src/index.ts @@ -12,3 +12,4 @@ export * as Automations from './form/automation'; export * as SsmEvents from './types/events'; export * as SsmAgent from './enums/agent'; export * as SsmAlert from './enums/alert'; +export * as SsmDeviceDiagnostic from './enums/diagnostic' diff --git a/shared-lib/src/types/events.ts b/shared-lib/src/types/events.ts index 48e17aec..ff0b627c 100644 --- a/shared-lib/src/types/events.ts +++ b/shared-lib/src/types/events.ts @@ -30,3 +30,7 @@ export enum Alert { export enum VolumeBackup { PROGRESS = 'volume:backup:progress', } + +export enum Diagnostic { + PROGRESS = 'diagnostic:progress' +} From 88c778d9c11a9dfddc1eaed7919df0f08dda0720 Mon Sep 17 00:00:00 2001 From: SquirrelDeveloper Date: Fri, 15 Nov 2024 12:28:41 +0000 Subject: [PATCH 2/2] Updated CHANGELOG.md --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 948b9e63..ea2a8c0d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ [Full Changelog](https://github.com/SquirrelCorporation/SquirrelServersManager/compare/v0.1.22...HEAD) +**Implemented enhancements:** + +- \[FEAT\] Add advanced diagnostic checks for device connections [\#482](https://github.com/SquirrelCorporation/SquirrelServersManager/pull/482) ([SquirrelDeveloper](https://github.com/SquirrelDeveloper)) + **Merged pull requests:** - \[CHORE\] Increase JSON request size limit to 50mb [\#480](https://github.com/SquirrelCorporation/SquirrelServersManager/pull/480) ([SquirrelDeveloper](https://github.com/SquirrelDeveloper))