diff --git a/experimental/CHANGELOG.md b/experimental/CHANGELOG.md index 885a4ec2925..786b78e9786 100644 --- a/experimental/CHANGELOG.md +++ b/experimental/CHANGELOG.md @@ -13,6 +13,8 @@ All notable changes to experimental packages in this project will be documented ### :rocket: (Enhancement) +* feat(otlp-proto): pre-compile proto files [#3098](https://github.com/open-telemetry/opentelemetry-js/pull/3098) @legendecas + ### :bug: (Bug Fix) * fix(histogram): fix maximum when only values < -1 are provided [#3086](https://github.com/open-telemetry/opentelemetry-js/pull/3086) @pichlermarc diff --git a/experimental/packages/exporter-trace-otlp-proto/test/OTLPTraceExporter.test.ts b/experimental/packages/exporter-trace-otlp-proto/test/OTLPTraceExporter.test.ts index 486a4fad6ba..f8f8ec87d04 100644 --- a/experimental/packages/exporter-trace-otlp-proto/test/OTLPTraceExporter.test.ts +++ b/experimental/packages/exporter-trace-otlp-proto/test/OTLPTraceExporter.test.ts @@ -30,7 +30,7 @@ import { MockedResponse, } from './traceHelper'; import { CompressionAlgorithm, OTLPExporterNodeConfigBase, OTLPExporterError } from '@opentelemetry/otlp-exporter-base'; -import { getExportRequestProto } from '@opentelemetry/otlp-proto-exporter-base'; +import { getExportRequestProto, ServiceClientType } from '@opentelemetry/otlp-proto-exporter-base'; import { IExportTraceServiceRequest } from '@opentelemetry/otlp-transformer'; let fakeRequest: PassThrough; @@ -208,8 +208,8 @@ describe('OTLPTraceExporter - node with proto over http', () => { let buff = Buffer.from(''); fakeRequest.on('end', () => { - const ExportTraceServiceRequestProto = getExportRequestProto(); - const data = ExportTraceServiceRequestProto?.decode(buff); + const ExportTraceServiceRequestProto = getExportRequestProto(ServiceClientType.SPANS); + const data = ExportTraceServiceRequestProto.decode(buff); const json = data?.toJSON() as IExportTraceServiceRequest; const span1 = json.resourceSpans?.[0].scopeSpans?.[0].spans?.[0]; assert.ok(typeof span1 !== 'undefined', "span doesn't exist"); @@ -294,8 +294,8 @@ describe('OTLPTraceExporter - node with proto over http', () => { let buff = Buffer.from(''); fakeRequest.on('end', () => { const unzippedBuff = zlib.gunzipSync(buff); - const ExportTraceServiceRequestProto = getExportRequestProto(); - const data = ExportTraceServiceRequestProto?.decode(unzippedBuff); + const ExportTraceServiceRequestProto = getExportRequestProto(ServiceClientType.SPANS); + const data = ExportTraceServiceRequestProto.decode(unzippedBuff); const json = data?.toJSON() as IExportTraceServiceRequest; const span1 = json.resourceSpans?.[0].scopeSpans?.[0].spans?.[0]; assert.ok(typeof span1 !== 'undefined', "span doesn't exist"); diff --git a/experimental/packages/opentelemetry-exporter-metrics-otlp-proto/test/OTLPMetricExporter.test.ts b/experimental/packages/opentelemetry-exporter-metrics-otlp-proto/test/OTLPMetricExporter.test.ts index c513289d46d..ad81bb5e6a7 100644 --- a/experimental/packages/opentelemetry-exporter-metrics-otlp-proto/test/OTLPMetricExporter.test.ts +++ b/experimental/packages/opentelemetry-exporter-metrics-otlp-proto/test/OTLPMetricExporter.test.ts @@ -16,7 +16,7 @@ import { diag } from '@opentelemetry/api'; import { ExportResultCode } from '@opentelemetry/core'; -import { getExportRequestProto } from '@opentelemetry/otlp-proto-exporter-base'; +import { getExportRequestProto, ServiceClientType } from '@opentelemetry/otlp-proto-exporter-base'; import * as assert from 'assert'; import * as http from 'http'; import * as sinon from 'sinon'; @@ -236,8 +236,8 @@ describe('OTLPMetricExporter - node with proto over http', () => { let buff = Buffer.from(''); fakeRequest.on('end', () => { - const ExportTraceServiceRequestProto = getExportRequestProto(); - const data = ExportTraceServiceRequestProto?.decode(buff); + const ExportTraceServiceRequestProto = getExportRequestProto(ServiceClientType.METRICS); + const data = ExportTraceServiceRequestProto.decode(buff); const json = data?.toJSON() as IExportMetricsServiceRequest; const metric1 = json.resourceMetrics[0].scopeMetrics[0].metrics[0]; diff --git a/experimental/packages/otlp-proto-exporter-base/.gitignore b/experimental/packages/otlp-proto-exporter-base/.gitignore new file mode 100644 index 00000000000..c82683cbd73 --- /dev/null +++ b/experimental/packages/otlp-proto-exporter-base/.gitignore @@ -0,0 +1,2 @@ +src/generated/* +!src/generated/.gitkeep diff --git a/experimental/packages/otlp-proto-exporter-base/package.json b/experimental/packages/otlp-proto-exporter-base/package.json index 97cbb4dcdd0..6a2379ff3e8 100644 --- a/experimental/packages/otlp-proto-exporter-base/package.json +++ b/experimental/packages/otlp-proto-exporter-base/package.json @@ -7,15 +7,14 @@ "repository": "open-telemetry/opentelemetry-js", "scripts": { "prepublishOnly": "npm run compile", - "compile": "tsc --build", + "compile": "npm run protos && tsc --build", "clean": "tsc --build --clean", "lint": "eslint . --ext .ts", "lint:fix": "eslint . --ext .ts --fix", - "postcompile": "npm run submodule && npm run protos:copy", - "protos:copy": "cpx protos/opentelemetry/**/*.* build/protos/opentelemetry", + "protos": "npm run submodule && node scripts/protos.js", "submodule": "git submodule sync --recursive && git submodule update --init --recursive", "version": "node ../../../scripts/version-update.js", - "watch": "npm run protos:copy && tsc -w", + "watch": "npm run protos && tsc -w", "precompile": "lerna run version --scope $(npm pkg get name) --include-dependencies", "prewatch": "npm run precompile" }, @@ -37,7 +36,6 @@ "build/src/**/*.js", "build/src/**/*.js.map", "build/src/**/*.d.ts", - "build/protos/**/*.proto", "doc", "LICENSE", "README.md" @@ -52,9 +50,9 @@ "@types/node": "14.17.33", "@types/sinon": "10.0.6", "codecov": "3.8.3", - "cpx": "1.5.0", "mocha": "7.2.0", "nyc": "15.1.0", + "protobufjs": "^6.9.0", "rimraf": "3.0.2", "sinon": "12.0.1", "ts-loader": "8.3.0", @@ -67,7 +65,6 @@ "dependencies": { "@grpc/proto-loader": "^0.6.9", "@opentelemetry/core": "1.4.0", - "@opentelemetry/otlp-exporter-base": "0.30.0", - "protobufjs": "^6.9.0" + "@opentelemetry/otlp-exporter-base": "0.30.0" } } diff --git a/experimental/packages/otlp-proto-exporter-base/scripts/protos.js b/experimental/packages/otlp-proto-exporter-base/scripts/protos.js new file mode 100644 index 00000000000..c5ab118f181 --- /dev/null +++ b/experimental/packages/otlp-proto-exporter-base/scripts/protos.js @@ -0,0 +1,58 @@ +'use strict'; + +const cp = require('child_process'); +const path = require('path'); + +const generatedPath = path.resolve(__dirname, '../src/generated'); +const protosPath = path.resolve(__dirname, '../protos'); +const protos = [ + 'opentelemetry/proto/common/v1/common.proto', + 'opentelemetry/proto/resource/v1/resource.proto', + 'opentelemetry/proto/trace/v1/trace.proto', + 'opentelemetry/proto/collector/trace/v1/trace_service.proto', + 'opentelemetry/proto/metrics/v1/metrics.proto', + 'opentelemetry/proto/collector/metrics/v1/metrics_service.proto', +].map(it => { + return path.join(protosPath, it); +}); + +function exec(command, argv) { + return new Promise((resolve, reject) => { + const child = cp.spawn(process.execPath, [command, ...argv], { + stdio: ['ignore', 'inherit', 'inherit'], + }); + child.on('exit', (code, signal) => { + if (code !== 0) { + reject(new Error(`${command} exited with non-zero code(${code}, ${signal})`)); + return; + } + resolve(); + }) + }); +} + +function pbts(pbjsOutFile) { + const pbtsPath = path.resolve(__dirname, '../node_modules/.bin/pbts'); + const pbtsOptions = [ + '-o', path.join(generatedPath, 'root.d.ts'), + ] + return exec(pbtsPath, [...pbtsOptions, pbjsOutFile]); +} + +async function pbjs(files) { + const pbjsPath = path.resolve(__dirname, '../node_modules/.bin/pbjs'); + const outFile = path.join(generatedPath, 'root.js'); + const pbjsOptions = [ + '-t', 'static-module', + '-w', 'commonjs', + '--null-defaults', + '-o', outFile, + ]; + await exec(pbjsPath, [...pbjsOptions, ...files]); + return outFile; +} + +(async function main() { + const pbjsOut = await pbjs(protos); + await pbts(pbjsOut); +})(); diff --git a/experimental/packages/otlp-proto-exporter-base/src/OTLPProtoExporterNodeBase.ts b/experimental/packages/otlp-proto-exporter-base/src/OTLPProtoExporterNodeBase.ts index 6ff136ed941..c08e2b5ef82 100644 --- a/experimental/packages/otlp-proto-exporter-base/src/OTLPProtoExporterNodeBase.ts +++ b/experimental/packages/otlp-proto-exporter-base/src/OTLPProtoExporterNodeBase.ts @@ -60,16 +60,6 @@ export abstract class OTLPProtoExporterNodeBase< promise.then(popPromise, popPromise); } - override onInit(config: OTLPExporterNodeConfigBase): void { - // defer to next tick and lazy load to avoid loading protobufjs too early - // and making this impossible to be instrumented - setImmediate(() => { - // eslint-disable-next-line @typescript-eslint/no-var-requires - const { onInit } = require('./util'); - onInit(this, config); - }); - } - override send( objects: ExportItem[], onSuccess: () => void, diff --git a/experimental/packages/otlp-proto-exporter-base/src/generated/.gitkeep b/experimental/packages/otlp-proto-exporter-base/src/generated/.gitkeep new file mode 100644 index 00000000000..e69de29bb2d diff --git a/experimental/packages/otlp-proto-exporter-base/src/util.ts b/experimental/packages/otlp-proto-exporter-base/src/util.ts index 0ab12f2e3e2..dcb4fa1c735 100644 --- a/experimental/packages/otlp-proto-exporter-base/src/util.ts +++ b/experimental/packages/otlp-proto-exporter-base/src/util.ts @@ -14,50 +14,29 @@ * limitations under the License. */ -import * as path from 'path'; - import { ServiceClientType } from './types'; import { OTLPProtoExporterNodeBase } from './OTLPProtoExporterNodeBase'; -import type { Type } from 'protobufjs'; -import * as protobufjs from 'protobufjs'; import { CompressionAlgorithm, OTLPExporterError, - OTLPExporterNodeConfigBase, sendWithHttp } from '@opentelemetry/otlp-exporter-base'; +import type * as protobuf from 'protobufjs'; +import * as root from './generated/root'; -let ExportRequestProto: Type | undefined; - -export function getExportRequestProto(): Type | undefined { - return ExportRequestProto; +export interface ExportRequestType unknown }> { + create(properties?: T): R; + encode(message: T, writer?: protobuf.Writer): protobuf.Writer; + decode(reader: (protobuf.Reader | Uint8Array), length?: number): R; } -export function onInit( - collector: OTLPProtoExporterNodeBase, - _config: OTLPExporterNodeConfigBase -): void { - const dir = path.resolve(__dirname, '..', 'protos'); - const root = new protobufjs.Root(); - root.resolvePath = function (origin, target) { - return `${dir}/${target}`; - }; - if (collector.getServiceClientType() === ServiceClientType.SPANS) { - const proto = root.loadSync([ - 'opentelemetry/proto/common/v1/common.proto', - 'opentelemetry/proto/resource/v1/resource.proto', - 'opentelemetry/proto/trace/v1/trace.proto', - 'opentelemetry/proto/collector/trace/v1/trace_service.proto', - ]); - ExportRequestProto = proto?.lookupType('ExportTraceServiceRequest'); +export function getExportRequestProto( + clientType: ServiceClientType, +): ExportRequestType { + if (clientType === ServiceClientType.SPANS) { + return root.opentelemetry.proto.collector.trace.v1.ExportTraceServiceRequest as unknown as ExportRequestType; } else { - const proto = root.loadSync([ - 'opentelemetry/proto/common/v1/common.proto', - 'opentelemetry/proto/resource/v1/resource.proto', - 'opentelemetry/proto/metrics/v1/metrics.proto', - 'opentelemetry/proto/collector/metrics/v1/metrics_service.proto', - ]); - ExportRequestProto = proto?.lookupType('ExportMetricsServiceRequest'); + return root.opentelemetry.proto.collector.metrics.v1.ExportMetricsServiceRequest as unknown as ExportRequestType; } } @@ -70,9 +49,10 @@ export function send( ): void { const serviceRequest = collector.convert(objects); - const message = getExportRequestProto()?.create(serviceRequest); + const exportRequestType = getExportRequestProto(collector.getServiceClientType()); + const message = exportRequestType.create(serviceRequest); if (message) { - const body = getExportRequestProto()?.encode(message).finish(); + const body = exportRequestType.encode(message).finish(); if (body) { sendWithHttp( collector, diff --git a/experimental/packages/otlp-proto-exporter-base/tsconfig.json b/experimental/packages/otlp-proto-exporter-base/tsconfig.json index 0282e8dc835..3e05eef24c0 100644 --- a/experimental/packages/otlp-proto-exporter-base/tsconfig.json +++ b/experimental/packages/otlp-proto-exporter-base/tsconfig.json @@ -1,11 +1,13 @@ { "extends": "../../../tsconfig.base.json", "compilerOptions": { + "allowJs": true, "rootDir": ".", "outDir": "build" }, "include": [ "src/**/*.ts", + "src/generated/*.js", "test/**/*.ts" ], "references": [ diff --git a/package.json b/package.json index 5aee41c22e8..4222ca52189 100644 --- a/package.json +++ b/package.json @@ -4,7 +4,7 @@ "description": "OpenTelemetry is a distributed tracing and stats collection framework.", "scripts": { "precompile": "lerna run version", - "compile": "tsc --build", + "compile": "lerna run protos && tsc --build", "prewatch": "npm run precompile", "watch": "tsc --build --watch", "clean": "tsc --build --clean",