feat(proto): add @opentelemetry/otlp-transformer package with hand-ro…

…lled transformation (#2746)

Co-authored-by: Gerhard Stöbich <[email protected]>

.github/workflows/peer-api.yaml

@@ -27,4 +27,4 @@ jobs:
 
       - name: Check API dependency semantics (experimental)
         working-directory: experimental
-        run: lerna exec --ignore propagation-validation-server --ignore @opentelemetry/selenium-tests --ignore @opentelemetry/api-metrics-wip "node ../../../scripts/peer-api-check.js"
+        run: lerna exec --ignore propagation-validation-server --ignore @opentelemetry/selenium-tests --ignore @opentelemetry/api-metrics-wip --ignore @opentelemetry/otlp-transformer "node ../../../scripts/peer-api-check.js"

.nycrc

@@ -12,7 +12,8 @@
     "test/**/*.*",
     ".eslintrc.js",
     "karma.conf.js",
-    "webpack/*.js"
+    "webpack/*.js",
+    "src/generated/**"
   ],
   "all": true
 }

experimental/packages/otlp-transformer/.eslintignore

@@ -0,0 +1,2 @@
+build
+src/generated

experimental/packages/otlp-transformer/.eslintrc.js

@@ -0,0 +1,8 @@
+module.exports = {
+    "env": {
+        "mocha": true,
+        "commonjs": true,
+        "shared-node-browser": true
+    },
+    ...require('../../../eslint.config.js')
+}

experimental/packages/otlp-transformer/.gitignore

@@ -0,0 +1 @@
+src/generated

experimental/packages/otlp-transformer/README.md

@@ -0,0 +1,115 @@
+# OpenTelemetry Protocol
+
+[![NPM Published Version][npm-img]][npm-url]
+[![Apache License][license-image]][license-image]
+
+This package provides everything needed to serialize [OpenTelemetry SDK][sdk] traces and metrics into the [OpenTelemetry Protocol][otlp] format using [protocol buffers][protobuf] or JSON.
+It also contains service clients for exporting traces and metrics to the OpenTelemetry Collector or a compatible receiver using using OTLP over [gRPC][grpc].
+This module uses [`protobufjs`][protobufjs] for serialization and is compatible with [`@grpc/grpc-js`][grpc-js].
+
+## Quick Start
+
+To get started you will need to install a compatible OpenTelemetry API.
+
+### Install Peer Dependencies
+
+```sh
+npm install \
+    @opentelemetry/api \
+    @grpc/grpc-js # only required if you are using gRPC
+```
+
+### Serialize Traces and Metrics
+
+This module exports functions to serialize traces and metrics from the OpenTelemetry SDK into protocol buffers which can be sent over HTTP to the OpenTelemetry collector or a compatible receiver.
+
+```typescript
+import { createExportTraceServiceRequest, createExportMetricsServiceRequest } from "@opentelemetry/otlp-transformer";
+
+const serializedSpans = createExportTraceServiceRequest(readableSpans);
+const serializedMetrics = createExportMetricsServiceRequest(readableMetrics);
+```
+
+### Create gRPC Service Clients
+
+This module also contains gRPC service clients for exporting traces and metrics to an OpenTelemetry collector or compatible receiver over gRPC.
+In order to avoid bundling a gRPC module with this module, it is required to construct an RPC implementation to pass to the constructor of the service clients.
+Any RPC implementation compatible with `grpc` or `@grpc/grpc-js` may be used, but `@grpc/grpc-js` is recommended.
+
+```typescript
+import type { RPCImpl } from 'protobufjs';
+import { makeGenericClientConstructor, credentials } from '@gprc/grpc-js';
+import { MetricServiceClient, TraceServiceClient } from "@opentelemetry/otlp-transformer";
+
+// Construct a RPC Implementation according to protobufjs docs
+const GrpcClientConstructor = makeGenericClientConstructor({});
+
+const metricGRPCClient = new GrpcClientConstructor(
+    "http://localhost:4317/v1/metrics", // default collector metrics endpoint
+    credentials.createInsecure(),
+);
+
+const traceGRPCClient = new GrpcClientConstructor(
+    "http://localhost:4317/v1/traces", // default collector traces endpoint
+    credentials.createInsecure(),
+);
+
+const metricRpc: RPCImpl = function(method, requestData, callback) {
+  metricGRPCClient.makeUnaryRequest(
+    method.name,
+    arg => arg,
+    arg => arg,
+    requestData,
+    callback
+  );
+}
+
+const traceRpc: RPCImpl = function(method, requestData, callback) {
+  traceGRPCClient.makeUnaryRequest(
+    method.name,
+    arg => arg,
+    arg => arg,
+    requestData,
+    callback
+  );
+}
+
+// Construct service clients to use RPC Implementations
+const metricServiceClient = new MetricServiceClient({
+    rpcImpl: metricRpc,
+    startTime: Date.now(), // exporter start time in milliseconds
+});
+
+const traceServiceClient = new TraceServiceClient({
+    rpcImpl: traceRpc,
+});
+
+// Export ReadableSpan[] and ReadableMetric[] over gRPC
+await metricServiceClient.export(readableMetrics);
+await traceServiceClient.export(readableSpans);
+```
+
+## Useful links
+
+- For more information on OpenTelemetry, visit: <https://opentelemetry.io/>
+- For more about OpenTelemetry JavaScript: <https://github.com/open-telemetry/opentelemetry-js>
+- For help or feedback on this project, join us in [GitHub Discussions][discussions-url]
+
+## License
+
+Apache 2.0 - See [LICENSE][license-url] for more information.
+
+[discussions-url]: https://github.com/open-telemetry/opentelemetry-js/discussions
+[license-url]: https://github.com/open-telemetry/opentelemetry-js/blob/main/LICENSE
+[license-image]: https://img.shields.io/badge/license-Apache_2.0-green.svg?style=flat
+[npm-url]: https://www.npmjs.com/package/@opentelemetry/otlp-transformer
+[npm-img]: https://badge.fury.io/js/%40opentelemetry%otlp-transformer.svg
+
+[sdk]: https://github.com/open-telemetry/opentelemetry-js
+[otlp]: https://github.com/open-telemetry/opentelemetry-proto
+
+[protobuf]: https://developers.google.com/protocol-buffers
+[grpc]: https://grpc.io/
+
+[protobufjs]: https://www.npmjs.com/package/protobufjs
+[grpc-js]: https://www.npmjs.com/package/@grpc/grpc-js

experimental/packages/otlp-transformer/karma.conf.js

@@ -0,0 +1,24 @@
+/*!
+ * Copyright The OpenTelemetry Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+const karmaWebpackConfig = require('../../../karma.webpack');
+const karmaBaseConfig = require('../../../karma.base');
+
+module.exports = (config) => {
+  config.set(Object.assign({}, karmaBaseConfig, {
+    webpack: karmaWebpackConfig
+  }))
+};

experimental/packages/otlp-transformer/package.json

@@ -0,0 +1,77 @@
+{
+  "name": "@opentelemetry/otlp-transformer",
+  "private": true,
+  "publishConfig": {
+    "access": "restricted"
+  },
+  "version": "0.27.0",
+  "description": "Transform OpenTelemetry SDK data into OTLP",
+  "module": "build/esm/index.js",
+  "esnext": "build/esnext/index.js",
+  "types": "build/src/index.d.ts",
+  "repository": "open-telemetry/opentelemetry-js",
+  "scripts": {
+    "compile": "tsc --build tsconfig.all.json",
+    "clean": "tsc --build --clean tsconfig.all.json",
+    "lint": "eslint . --ext .ts",
+    "lint:fix": "eslint . --ext .ts --fix",
+    "tdd": "npm run test -- --watch-extensions ts --watch",
+    "test": "nyc ts-mocha -p tsconfig.json 'test/**/*.test.ts'",
+    "test:browser": "nyc karma start --single-run",
+    "watch": "tsc --build -w tsconfig.all.json"
+  },
+  "keywords": [
+    "opentelemetry",
+    "nodejs",
+    "grpc",
+    "protobuf",
+    "otlp",
+    "tracing",
+    "metrics"
+  ],
+  "author": "OpenTelemetry Authors",
+  "license": "Apache-2.0",
+  "engines": {
+    "node": ">=8.0.0"
+  },
+  "files": [
+    "build/src/**/*.js",
+    "build/src/**/*.js.map",
+    "build/src/**/*.d.ts",
+    "LICENSE",
+    "README.md"
+  ],
+  "peerDependencies": {
+    "@opentelemetry/api": ">=1.0.0 <1.1.0",
+    "@opentelemetry/api-metrics": "~0.27.0"
+  },
+  "devDependencies": {
+    "@opentelemetry/api": "1.0.4",
+    "@opentelemetry/api-metrics": "0.27.0",
+    "@types/mocha": "8.2.3",
+    "@types/webpack-env": "1.16.3",
+    "codecov": "3.8.3",
+    "istanbul-instrumenter-loader": "3.0.1",
+    "karma": "6.3.16",
+    "karma-chrome-launcher": "3.1.0",
+    "karma-coverage-istanbul-reporter": "3.0.3",
+    "karma-mocha": "2.0.1",
+    "karma-spec-reporter": "0.0.32",
+    "karma-webpack": "4.0.2",
+    "mkdirp": "1.0.4",
+    "mocha": "7.2.0",
+    "nyc": "15.1.0",
+    "protobufjs": "6.11.2",
+    "rimraf": "3.0.2",
+    "ts-loader": "8.3.0",
+    "ts-mocha": "8.0.0",
+    "typescript": "4.4.4",
+    "webpack": "4.46.0"
+  },
+  "dependencies": {
+    "@opentelemetry/core": "1.0.1",
+    "@opentelemetry/resources": "1.0.1",
+    "@opentelemetry/sdk-metrics-base": "0.27.0",
+    "@opentelemetry/sdk-trace-base": "1.0.1"
+  }
+}

experimental/packages/otlp-transformer/src/common/internal.ts

@@ -0,0 +1,62 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import type { SpanAttributes } from '@opentelemetry/api';
+import { IAnyValue, IKeyValue } from './types';
+
+export function toAttributes(
+  attributes: SpanAttributes
+): IKeyValue[] {
+  return Object.keys(attributes).map(key => toKeyValue(key, attributes[key]));
+}
+
+export function toKeyValue(
+  key: string,
+  value: unknown
+): IKeyValue {
+  return {
+    key: key,
+    value: toAnyValue(value),
+  };
+}
+
+export function toAnyValue(value: unknown): IAnyValue {
+  const t = typeof value;
+  if (t === 'string') return { stringValue: value as string };
+  if (t === 'number') {
+    if (!Number.isInteger(value)) return { doubleValue: value as number };
+    return { intValue: value as number };
+  }
+  if (t === 'boolean') return { boolValue: value as boolean };
+  if (value instanceof Uint8Array) return { bytesValue: value };
+  if (Array.isArray(value)) return { arrayValue: { values: value.map(toAnyValue) } };
+  if (t === 'object' && value != null) return { kvlistValue: { values: Object.entries(value as object).map(([k, v]) => toKeyValue(k, v)) } };
+
+  return {};
+}
+
+export function hexToBuf(hex: string): Uint8Array | undefined {
+  const ints = hex.match(/[\da-f]{2}/gi)?.map(h => parseInt(h, 16));
+  return ints && new Uint8Array(ints);
+}
+
+function i2hex(i: number): string {
+  return ('0' + i.toString(16)).slice(-2);
+}
+
+export function bufToHex(buf?: Uint8Array | null): string | undefined {
+  if (buf == null || buf.length === 0) return undefined;
+  return Array.from(buf).map(i2hex).join('');
+}

experimental/packages/otlp-transformer/src/common/types.ts

@@ -0,0 +1,69 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/** Properties of an InstrumentationLibrary. */
+export interface IInstrumentationLibrary {
+  /** InstrumentationLibrary name */
+  name: string
+
+  /** InstrumentationLibrary version */
+  version?: string;
+}
+
+/** Properties of a KeyValue. */
+export interface IKeyValue {
+  /** KeyValue key */
+  key: string;
+
+  /** KeyValue value */
+  value: IAnyValue;
+}
+
+/** Properties of an AnyValue. */
+export interface IAnyValue {
+  /** AnyValue stringValue */
+  stringValue?: (string | null);
+
+  /** AnyValue boolValue */
+  boolValue?: (boolean | null);
+
+  /** AnyValue intValue */
+  intValue?: (number | Long | null);
+
+  /** AnyValue doubleValue */
+  doubleValue?: (number | null);
+
+  /** AnyValue arrayValue */
+  arrayValue?: IArrayValue;
+
+  /** AnyValue kvlistValue */
+  kvlistValue?: IKeyValueList;
+
+  /** AnyValue bytesValue */
+  bytesValue?: Uint8Array;
+}
+
+/** Properties of an ArrayValue. */
+export interface IArrayValue {
+  /** ArrayValue values */
+  values: IAnyValue[];
+}
+
+/** Properties of a KeyValueList. */
+export interface IKeyValueList {
+  /** KeyValueList values */
+  values: IKeyValue[];
+}