feat: Trace api calls

.gitignore

@@ -58,3 +58,4 @@ data/
 registration.yaml
 
 storybook-static
+development/tempo-data/

apps/meteor/app/api/server/api.ts

@@ -3,6 +3,7 @@ import { Logger } from '@rocket.chat/logger';
 import { Users } from '@rocket.chat/models';
 import { Random } from '@rocket.chat/random';
 import type { JoinPathPattern, Method } from '@rocket.chat/rest-typings';
+import { tracerSpan } from '@rocket.chat/tracing';
 import { Accounts } from 'meteor/accounts-base';
 import { DDP } from 'meteor/ddp';
 import { DDPCommon } from 'meteor/ddp-common';
@@ -645,8 +646,29 @@ export class APIClass<TBasePath extends string = ''> extends Restivus {
 							this.queryFields = options.queryFields;
 							this.parseJsonQuery = api.parseJsonQuery.bind(this as PartialThis);
 
-							result =
-								(await DDP._CurrentInvocation.withValue(invocation as any, async () => originalAction.apply(this))) || API.v1.success();
+							result = await tracerSpan(
+								`${this.request.method} ${this.request.url}`,
+								{
+									attributes: {
+										url: this.request.url,
+										route: this.request.route,
+										method: this.request.method,
+										userId: this.userId,
+									},
+								},
+								async (span) => {
+									if (span) {
+										this.response.setHeader('X-Trace-Id', span.spanContext().traceId);
+									}
+
+									const result =
+										(await DDP._CurrentInvocation.withValue(invocation as any, async () => originalAction.apply(this))) || API.v1.success();
+
+									span?.setAttribute('status', result.statusCode);
+
+									return result;
+								},
+							);
 
 							log.http({
 								status: result.statusCode,

apps/meteor/app/lib/server/lib/debug.js

@@ -1,5 +1,6 @@
 import { InstanceStatus } from '@rocket.chat/instance-status';
 import { Logger } from '@rocket.chat/logger';
+import { tracerActiveSpan } from '@rocket.chat/tracing';
 import { Meteor } from 'meteor/meteor';
 import { WebApp } from 'meteor/webapp';
 import _ from 'underscore';
@@ -72,9 +73,20 @@ const wrapMethods = function (name, originalHandler, methodsMap) {
 			...getMethodArgs(name, originalArgs),
 		});
 
-		const result = originalHandler.apply(this, originalArgs);
-		end();
-		return result;
+		return tracerActiveSpan(
+			`Method ${name}`,
+			{
+				attributes: {
+					method: name,
+					userId: this.userId,
+				},
+			},
+			async () => {
+				const result = await originalHandler.apply(this, originalArgs);
+				end();
+				return result;
+			},
+		);
 	};
 };
 

apps/meteor/app/metrics/server/lib/collectMetrics.ts

@@ -1,6 +1,7 @@
 import http from 'http';
 
 import { Statistics } from '@rocket.chat/models';
+import { tracerSpan } from '@rocket.chat/tracing';
 import connect from 'connect';
 import { Facts } from 'meteor/facts-base';
 import { Meteor } from 'meteor/meteor';
@@ -169,7 +170,20 @@ const updatePrometheusConfig = async (): Promise<void> => {
 			host: process.env.BIND_IP || '0.0.0.0',
 		});
 
-		timer = setInterval(setPrometheusData, 5000);
+		timer = setInterval(async () => {
+			void tracerSpan(
+				'setPrometheusData',
+				{
+					attributes: {
+						port: is.port,
+						host: process.env.BIND_IP || '0.0.0.0',
+					},
+				},
+				() => {
+					void setPrometheusData();
+				},
+			);
+		}, 5000);
 	}
 
 	clearInterval(resetTimer);

apps/meteor/app/notification-queue/server/NotificationQueue.ts

@@ -1,5 +1,6 @@
 import type { INotification, INotificationItemPush, INotificationItemEmail, NotificationItem, IUser } from '@rocket.chat/core-typings';
 import { NotificationQueue, Users } from '@rocket.chat/models';
+import { tracerSpan } from '@rocket.chat/tracing';
 import { Meteor } from 'meteor/meteor';
 
 import { SystemLogger } from '../../../server/lib/logger/system';
@@ -43,25 +44,37 @@ class NotificationClass {
 
 		setTimeout(async () => {
 			try {
-				await this.worker();
+				const continueLater = await tracerSpan(
+					'NotificationWorker',
+					{
+						attributes: {
+							workerTime: new Date().toISOString(),
+						},
+					},
+					() => this.worker(),
+				);
+
+				if (continueLater) {
+					this.executeWorkerLater();
+				}
 			} catch (err) {
 				SystemLogger.error({ msg: 'Error sending notification', err });
 				this.executeWorkerLater();
 			}
 		}, this.cyclePause);
 	}
 
-	async worker(counter = 0): Promise<void> {
+	async worker(counter = 0): Promise<boolean> {
 		const notification = await this.getNextNotification();
 
 		if (!notification) {
-			return this.executeWorkerLater();
+			return true;
 		}
 
 		// Once we start notifying the user we anticipate all the schedules
 		const flush = await NotificationQueue.clearScheduleByUserId(notification.uid);
 
-		// start worker again it queue flushed
+		// start worker again if queue flushed
 		if (flush.modifiedCount) {
 			await NotificationQueue.unsetSendingById(notification._id);
 			return this.worker(counter);
@@ -86,9 +99,10 @@ class NotificationClass {
 		}
 
 		if (counter >= this.maxBatchSize) {
-			return this.executeWorkerLater();
+			return true;
 		}
-		await this.worker(counter++);
+
+		return this.worker(counter++);
 	}
 
 	getNextNotification(): Promise<INotification | null> {

apps/meteor/package.json

@@ -225,6 +225,9 @@
 		"@nivo/heatmap": "0.84.0",
 		"@nivo/line": "0.84.0",
 		"@nivo/pie": "0.84.0",
+		"@opentelemetry/api": "^1.9.0",
+		"@opentelemetry/exporter-trace-otlp-grpc": "^0.53.0",
+		"@opentelemetry/sdk-node": "^0.53.0",
 		"@react-aria/color": "^3.0.0-beta.15",
 		"@react-aria/toolbar": "^3.0.0-beta.1",
 		"@react-pdf/renderer": "^3.4.5",
@@ -278,6 +281,7 @@
 		"@rocket.chat/sha256": "workspace:^",
 		"@rocket.chat/string-helpers": "~0.31.25",
 		"@rocket.chat/tools": "workspace:^",
+		"@rocket.chat/tracing": "workspace:^",
 		"@rocket.chat/ui-avatar": "workspace:^",
 		"@rocket.chat/ui-client": "workspace:^",
 		"@rocket.chat/ui-composer": "workspace:^",

apps/meteor/packages/rocketchat-mongo-config/server/index.js

@@ -20,6 +20,9 @@ const mongoConnectionOptions = {
 	// add retryWrites=false if not present in MONGO_URL
 	...(!process.env.MONGO_URL.includes('retryWrites') && { retryWrites: false }),
 	// ignoreUndefined: false, // TODO evaluate adding this config
+
+	// TODO ideally we should call isTracingEnabled(), but since this is a Meteor package we can't :/
+	monitorCommands: ['yes', 'true'].includes(String(process.env.TRACING_ENABLED).toLowerCase()),
 };
 
 const mongoOptionStr = process.env.MONGO_OPTIONS;

apps/meteor/server/cron/statistics.ts

@@ -2,36 +2,39 @@ import { cronJobs } from '@rocket.chat/cron';
 import type { Logger } from '@rocket.chat/logger';
 import { Statistics } from '@rocket.chat/models';
 import { serverFetch as fetch } from '@rocket.chat/server-fetch';
+import { tracerSpan } from '@rocket.chat/tracing';
 import { Meteor } from 'meteor/meteor';
 
 import { getWorkspaceAccessToken } from '../../app/cloud/server';
 import { statistics } from '../../app/statistics/server';
 
 async function generateStatistics(logger: Logger): Promise<void> {
-	const cronStatistics = await statistics.save();
-
-	try {
-		const token = await getWorkspaceAccessToken();
-		const headers = { ...(token && { Authorization: `Bearer ${token}` }) };
-
-		const response = await fetch('https://collector.rocket.chat/', {
-			method: 'POST',
-			body: {
-				...cronStatistics,
-				host: Meteor.absoluteUrl(),
-			},
-			headers,
-		});
-
-		const { statsToken } = await response.json();
-
-		if (statsToken != null) {
-			await Statistics.updateOne({ _id: cronStatistics._id }, { $set: { statsToken } });
+	await tracerSpan('generateStatistics', {}, async () => {
+		const cronStatistics = await statistics.save();
+
+		try {
+			const token = await getWorkspaceAccessToken();
+			const headers = { ...(token && { Authorization: `Bearer ${token}` }) };
+
+			const response = await fetch('https://collector.rocket.chat/', {
+				method: 'POST',
+				body: {
+					...cronStatistics,
+					host: Meteor.absoluteUrl(),
+				},
+				headers,
+			});
+
+			const { statsToken } = await response.json();
+
+			if (statsToken != null) {
+				await Statistics.updateOne({ _id: cronStatistics._id }, { $set: { statsToken } });
+			}
+		} catch (error) {
+			/* error*/
+			logger.warn('Failed to send usage report');
 		}
-	} catch (error) {
-		/* error*/
-		logger.warn('Failed to send usage report');
-	}
+	});
 }
 
 export async function statsCron(logger: Logger): Promise<void> {

apps/meteor/server/database/utils.ts

@@ -1,3 +1,6 @@
+import { initDatabaseTracing } from '@rocket.chat/tracing';
 import { MongoInternals } from 'meteor/mongo';
 
 export const { db, client } = MongoInternals.defaultRemoteCollectionDriver().mongo;
+
+initDatabaseTracing(client);

apps/meteor/server/main.ts

@@ -1,3 +1,4 @@
+import './tracing';
 import './models/startup';
 /**
  * ./settings uses top level await, in theory the settings creation

apps/meteor/server/models/raw/BaseRaw.ts

@@ -1,3 +1,4 @@
+import { traceInstanceMethods } from '@rocket.chat/core-services';
 import type { RocketChatRecordDeleted } from '@rocket.chat/core-typings';
 import type { IBaseModel, DefaultFields, ResultFields, FindPaginated, InsertionModel } from '@rocket.chat/model-typings';
 import type { Updater } from '@rocket.chat/models';
@@ -76,6 +77,8 @@ export abstract class BaseRaw<
 		void this.createIndexes();
 
 		this.preventSetUpdatedAt = options?.preventSetUpdatedAt ?? false;
+
+		return traceInstanceMethods(this);
 	}
 
 	private pendingIndexes: Promise<void> | undefined;

apps/meteor/server/tracing.ts

@@ -0,0 +1,3 @@
+import { startTracing } from '@rocket.chat/tracing';
+
+startTracing({ service: 'core' });

development/agent.yml

@@ -0,0 +1,16 @@
+server:
+  log_level: debug
+
+traces:
+  configs:
+  - name: default
+    receivers:
+      otlp:
+        protocols:
+          grpc:
+    remote_write:
+      - endpoint: tempo:4317
+        insecure: true
+    batch:
+      timeout: 5s
+      send_batch_size: 100

development/collector.config.yml

@@ -0,0 +1,24 @@
+receivers:
+  otlp:
+    protocols:
+      grpc:
+      http:
+
+processors:
+  batch:
+    timeout: 100ms
+
+exporters:
+  logging:
+    loglevel: debug
+  otlp/1:
+    endpoint: tempo:4317
+    tls:
+      insecure: true
+
+service:
+  pipelines:
+    traces:
+      receivers: [otlp]
+      processors: [batch]
+      exporters: [otlp/1]