diff --git a/.goreleaser.yml b/.goreleaser.yml index 8433bbc985..c629185b02 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -15,7 +15,7 @@ builds: asmflags: - all=-trimpath={{.Env.GOPATH}} ldflags: - - -s -w -X main.version={{.Version}} -X main.commit={{.Commit}} -X main.date={{.Date}} -X main.telemetryReportPeriod=24h + - -s -w -X main.version={{.Version}} -X main.commit={{.Commit}} -X main.date={{.Date}} -X main.telemetryReportPeriod=24h -X main.telemetryEndpointInsecure=false main: ./cmd/gateway/ binary: gateway diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e6f7d7ab6f..19224e2f11 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,7 +5,7 @@ repos: rev: v4.5.0 hooks: - id: trailing-whitespace - exclude: (^tests/results/) + exclude: (^tests/results/|\.avdl$|_generated.go$) - id: end-of-file-fixer - id: check-yaml args: [--allow-multiple-documents] diff --git a/Makefile b/Makefile index 2e30909276..2b625d4d81 100644 --- a/Makefile +++ b/Makefile @@ -9,12 +9,19 @@ NJS_DIR = internal/mode/static/nginx/modules/src NGINX_DOCKER_BUILD_PLUS_ARGS = --secret id=nginx-repo.crt,src=nginx-repo.crt --secret id=nginx-repo.key,src=nginx-repo.key BUILD_AGENT=local TELEMETRY_REPORT_PERIOD = 24h # also configured in goreleaser.yml + +# FIXME(pleshakov) - TELEMETRY_ENDPOINT will have the default value of F5 telemetry service once we're ready +# to report. https://github.com/nginxinc/nginx-gateway-fabric/issues/1563 +# Also, we will need to set it in goreleaser.yml +TELEMETRY_ENDPOINT =# if empty, NGF will report telemetry in its logs at debug level. + +TELEMETRY_ENDPOINT_INSECURE = false # also configured in goreleaser.yml GW_API_VERSION = 1.0.0 INSTALL_WEBHOOK = false NODE_VERSION = $(shell cat .nvmrc) # go build flags - should not be overridden by the user -GO_LINKER_FlAGS_VARS = -X main.version=${VERSION} -X main.commit=${GIT_COMMIT} -X main.date=${DATE} -X main.telemetryReportPeriod=${TELEMETRY_REPORT_PERIOD} +GO_LINKER_FlAGS_VARS = -X main.version=${VERSION} -X main.commit=${GIT_COMMIT} -X main.date=${DATE} -X main.telemetryReportPeriod=${TELEMETRY_REPORT_PERIOD} -X main.telemetryEndpoint=${TELEMETRY_ENDPOINT} -X main.telemetryEndpointInsecure=${TELEMETRY_ENDPOINT_INSECURE} GO_LINKER_FLAGS_OPTIMIZATIONS = -s -w GO_LINKER_FLAGS = $(GO_LINKER_FLAGS_OPTIMIZATIONS) $(GO_LINKER_FlAGS_VARS) diff --git a/build/Dockerfile b/build/Dockerfile index 4411500348..2de7f5d980 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -26,6 +26,9 @@ COPY dist/gateway_linux_$TARGETARCH*/gateway /usr/bin/ RUN setcap 'cap_kill=+ep' /usr/bin/gateway FROM scratch as common +# CA certs are needed for telemetry report and NGINX Plus usage report features, so that +# NGF can verify the server's certificate. +COPY --from=builder --link /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ USER 102:1001 ARG BUILD_AGENT ENV BUILD_AGENT=${BUILD_AGENT} diff --git a/cmd/gateway/commands.go b/cmd/gateway/commands.go index 0a5d54d43f..222088f795 100644 --- a/cmd/gateway/commands.go +++ b/cmd/gateway/commands.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" "os" + "strconv" "time" "github.com/spf13/cobra" @@ -160,6 +161,17 @@ func createStaticModeCommand() *cobra.Command { return fmt.Errorf("error parsing telemetry report period: %w", err) } + if telemetryEndpoint != "" { + if err := validateEndpoint(telemetryEndpoint); err != nil { + return fmt.Errorf("error validating telemetry endpoint: %w", err) + } + } + + telemetryEndpointInsecure, err := strconv.ParseBool(telemetryEndpointInsecure) + if err != nil { + return fmt.Errorf("error parsing telemetry endpoint insecure: %w", err) + } + var gwNsName *types.NamespacedName if cmd.Flags().Changed(gatewayFlag) { gwNsName = &gateway.value @@ -211,8 +223,10 @@ func createStaticModeCommand() *cobra.Command { }, UsageReportConfig: usageReportConfig, ProductTelemetryConfig: config.ProductTelemetryConfig{ - TelemetryReportPeriod: period, - Enabled: !disableProductTelemetry, + ReportPeriod: period, + Enabled: !disableProductTelemetry, + Endpoint: telemetryEndpoint, + EndpointInsecure: telemetryEndpointInsecure, }, Plus: plus, Version: version, diff --git a/cmd/gateway/main.go b/cmd/gateway/main.go index 31ffdf61d5..8761e3f1cd 100644 --- a/cmd/gateway/main.go +++ b/cmd/gateway/main.go @@ -13,6 +13,10 @@ var ( // telemetryReportPeriod is the period at which telemetry reports are sent. telemetryReportPeriod string + // telemetryEndpoint is the endpoint to which telemetry reports are sent. + telemetryEndpoint string + // telemetryEndpointInsecure controls whether TLS should be used when sending telemetry reports. + telemetryEndpointInsecure string ) func main() { diff --git a/cmd/gateway/validation.go b/cmd/gateway/validation.go index b31f8c6a2d..8fbd4b4c4a 100644 --- a/cmd/gateway/validation.go +++ b/cmd/gateway/validation.go @@ -6,6 +6,7 @@ import ( "net" "net/url" "regexp" + "strconv" "strings" "k8s.io/apimachinery/pkg/types" @@ -133,6 +134,35 @@ func validateIP(ip string) error { return nil } +// validateEndpoint validates an endpoint, which is : where host is either a hostname or an IP address. +func validateEndpoint(endpoint string) error { + host, port, err := net.SplitHostPort(endpoint) + if err != nil { + return fmt.Errorf("%q must be in the format :: %w", endpoint, err) + } + + portVal, err := strconv.ParseInt(port, 10, 16) + if err != nil { + return fmt.Errorf("port must be a valid number: %w", err) + } + + if portVal < 1 || portVal > 65535 { + return fmt.Errorf("port outside of valid port range [1 - 65535]: %v", port) + } + + if err := validateIP(host); err == nil { + return nil + } + + if errs := validation.IsDNS1123Subdomain(host); len(errs) == 0 { + return nil + } + + // we don't know if the user intended to use a hostname or an IP address, + // so we return a generic error message + return fmt.Errorf("%q must be in the format :", endpoint) +} + // validatePort makes sure a given port is inside the valid port range for its usage func validatePort(port int) error { if port < 1024 || port > 65535 { diff --git a/cmd/gateway/validation_test.go b/cmd/gateway/validation_test.go index 2e85a9d128..f19e3ac84f 100644 --- a/cmd/gateway/validation_test.go +++ b/cmd/gateway/validation_test.go @@ -419,6 +419,73 @@ func TestValidateIP(t *testing.T) { } } +func TestValidateEndpoint(t *testing.T) { + tests := []struct { + name string + endp string + expErr bool + }{ + { + name: "valid endpoint with hostname", + endp: "localhost:8080", + expErr: false, + }, + { + name: "valid endpoint with IPv4", + endp: "1.2.3.4:8080", + expErr: false, + }, + { + name: "valid endpoint with IPv6", + endp: "[::1]:8080", + expErr: false, + }, + { + name: "invalid port - 1", + endp: "localhost:0", + expErr: true, + }, + { + name: "invalid port - 2", + endp: "localhost:65536", + expErr: true, + }, + { + name: "missing port with hostname", + endp: "localhost", + expErr: true, + }, + { + name: "missing port with IPv4", + endp: "1.2.3.4", + expErr: true, + }, + { + name: "missing port with IPv6", + endp: "[::1]", + expErr: true, + }, + { + name: "invalid hostname or IP", + endp: "loc@lhost:8080", + expErr: true, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + g := NewWithT(t) + + err := validateEndpoint(tc.endp) + if !tc.expErr { + g.Expect(err).ToNot(HaveOccurred()) + } else { + g.Expect(err).To(HaveOccurred()) + } + }) + } +} + func TestValidatePort(t *testing.T) { tests := []struct { name string diff --git a/docs/developer/implementing-a-feature.md b/docs/developer/implementing-a-feature.md index c56d5dd7ee..e7759767ca 100644 --- a/docs/developer/implementing-a-feature.md +++ b/docs/developer/implementing-a-feature.md @@ -59,7 +59,11 @@ practices to ensure a successful feature development process. different reviewer in mind, you can request them as well. Refer to the [pull request](/docs/developer/pull-request.md) documentation for expectations and guidelines. 14. **Obtain the necessary approvals**: Work with code reviewers to maintain the required number of approvals. -15. **Squash and merge**: Squash your commits locally, or use the GitHub UI to squash and merge. Only one commit per +15. **Ensure the product telemetry works**. If you made any changes to the product telemetry data points, it is + necessary to push the generated scheme (`.avdl`, generated in Step 12) to the scheme registry. After that, manually + verify that the product telemetry data is successfully pushed to the telemetry service by confirming that the data + has been received. +16. **Squash and merge**: Squash your commits locally, or use the GitHub UI to squash and merge. Only one commit per pull request should be merged. Make sure the first line of the final commit message includes the pull request number. For example, Fix supported gateway conditions in compatibility doc (#674). > **Note**: diff --git a/docs/developer/quickstart.md b/docs/developer/quickstart.md index 41de3a4b6d..c8846cb8b2 100644 --- a/docs/developer/quickstart.md +++ b/docs/developer/quickstart.md @@ -214,7 +214,7 @@ Run the following make command from the project's root directory to lint the Hel make lint-helm ``` -## Run go generate +## Run Code Generation To ensure all the generated code is up to date, run the following make command from the project's root directory: @@ -222,6 +222,8 @@ To ensure all the generated code is up to date, run the following make command f make generate ``` +That command also will generate the avro scheme (`.avdl`) for product telemetry data points. + ## Update Generated Manifests To update the generated manifests, run the following make command from the project's root directory: diff --git a/go.mod b/go.mod index 91c1a8304e..5160512a55 100644 --- a/go.mod +++ b/go.mod @@ -12,6 +12,7 @@ require ( github.com/maxbrunsfeld/counterfeiter/v6 v6.8.1 github.com/nginxinc/nginx-plus-go-client v1.2.0 github.com/nginxinc/nginx-prometheus-exporter v1.1.0 + github.com/nginxinc/telemetry-exporter v0.0.0-20240307135433-a5ecce59bddf github.com/onsi/ginkgo/v2 v2.16.0 github.com/onsi/gomega v1.31.1 github.com/prometheus/client_golang v1.19.0 @@ -19,6 +20,8 @@ require ( github.com/spf13/cobra v1.8.0 github.com/spf13/pflag v1.0.5 github.com/tsenart/vegeta/v12 v12.11.1 + go.opentelemetry.io/otel v1.24.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.24.0 go.uber.org/zap v1.27.0 k8s.io/api v0.29.2 k8s.io/apiextensions-apiserver v0.29.2 @@ -32,6 +35,7 @@ require ( require ( github.com/beorn7/perks v1.0.1 // indirect + github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/emicklei/go-restful/v3 v3.11.0 // indirect @@ -40,6 +44,7 @@ require ( github.com/fatih/color v1.16.0 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/go-logfmt/logfmt v0.5.1 // indirect + github.com/go-logr/stdr v1.2.2 // indirect github.com/go-logr/zapr v1.3.0 // indirect github.com/go-openapi/jsonpointer v0.20.0 // indirect github.com/go-openapi/jsonreference v0.20.2 // indirect @@ -52,8 +57,9 @@ require ( github.com/google/gnostic-models v0.6.8 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect - github.com/google/uuid v1.3.1 // indirect + github.com/google/uuid v1.6.0 // indirect github.com/gorilla/websocket v1.5.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.0 // indirect github.com/imdario/mergo v0.3.16 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/influxdata/tdigest v0.0.1 // indirect @@ -73,9 +79,14 @@ require ( github.com/prometheus/procfs v0.12.0 // indirect github.com/rs/dnscache v0.0.0-20211102005908-e0241e321417 // indirect github.com/stretchr/testify v1.8.4 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.24.0 // indirect + go.opentelemetry.io/otel/metric v1.24.0 // indirect + go.opentelemetry.io/otel/sdk v1.24.0 // indirect + go.opentelemetry.io/otel/trace v1.24.0 // indirect + go.opentelemetry.io/proto/otlp v1.1.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect - golang.org/x/mod v0.14.0 // indirect + golang.org/x/mod v0.16.0 // indirect golang.org/x/net v0.22.0 // indirect golang.org/x/oauth2 v0.18.0 // indirect golang.org/x/sync v0.6.0 // indirect @@ -83,9 +94,12 @@ require ( golang.org/x/term v0.18.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.3.0 // indirect - golang.org/x/tools v0.17.0 // indirect + golang.org/x/tools v0.19.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect google.golang.org/appengine v1.6.8 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240102182953-50ed04b92917 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240102182953-50ed04b92917 // indirect + google.golang.org/grpc v1.61.1 // indirect google.golang.org/protobuf v1.33.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/go.sum b/go.sum index 6136827dcb..2d58966401 100644 --- a/go.sum +++ b/go.sum @@ -4,6 +4,8 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bmizerany/perks v0.0.0-20230307044200-03f9df79da1e h1:mWOqoK5jV13ChKf/aF3plwQ96laasTJgZi4f1aSOu+M= github.com/bmizerany/perks v0.0.0-20230307044200-03f9df79da1e/go.mod h1:ac9efd0D1fsDb3EJvhqgXRbFx7bs2wqZ10HQPeU8U/Q= +github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= +github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.2.0/go.mod h1:9+9sk7u7pGNWYMkh0hdiL++6OeibzJccyQU4p4MedaY= @@ -30,9 +32,12 @@ github.com/go-kit/log v0.2.1 h1:MRVx0/zhvdseW+Gza6N9rVzU/IVzaeE1SFI4raAhmBU= github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= github.com/go-logfmt/logfmt v0.5.1 h1:otpy5pqBCBZ1ng9RQ0dPu4PN7ba75Y/aA+UpowDyNVA= github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= @@ -67,11 +72,13 @@ github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= -github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.0 h1:Wqo399gCIufwto+VfwCSvsnfGpF/w5E9CNxSwbpD6No= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.0/go.mod h1:qmOFXW2epJhM0qSnUUYpldc7gVz2KMQwJ/QYCDIa7XU= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= @@ -116,6 +123,8 @@ github.com/nginxinc/nginx-plus-go-client v1.2.0 h1:NVfRsHbMJ7lOhkqMG52uvODiDBhQZ github.com/nginxinc/nginx-plus-go-client v1.2.0/go.mod h1:n8OFLzrJulJ2fur28Cwa1Qp5DZNS2VicLV+Adt30LQ4= github.com/nginxinc/nginx-prometheus-exporter v1.1.0 h1:Uj+eWKGvUionZc8gWFDnrb3jpdkuZAlPKo4ck96cOmE= github.com/nginxinc/nginx-prometheus-exporter v1.1.0/go.mod h1:A1Fy5uLQonVGmwLC5xNxBX+vPFgYzBOvPjNRs8msT0k= +github.com/nginxinc/telemetry-exporter v0.0.0-20240307135433-a5ecce59bddf h1:PM0o/J1QyRpNCn8C9SI17b5ePuAnLdI1D5B/TV2hneY= +github.com/nginxinc/telemetry-exporter v0.0.0-20240307135433-a5ecce59bddf/go.mod h1:rZ+Ohzwv9LJMzxRDPS/dEwXOUPlNrzjoGkICaG9fv0k= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= @@ -164,6 +173,20 @@ github.com/tsenart/vegeta/v12 v12.11.1/go.mod h1:swiFmrgpqj2llHURgHYFRFN0tfrIrln github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo= +go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.24.0 h1:t6wl9SPayj+c7lEIFgm4ooDBZVb01IhLB4InpomhRw8= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.24.0/go.mod h1:iSDOcsnSA5INXzZtwaBPrKp/lWu/V14Dd+llD0oI2EA= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.24.0 h1:Mw5xcxMwlqoJd97vwPxA8isEaIoxsta9/Q51+TTJLGE= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.24.0/go.mod h1:CQNu9bj7o7mC6U7+CA/schKEYakYXWr79ucDHTMGhCM= +go.opentelemetry.io/otel/metric v1.24.0 h1:6EhoGWWK28x1fbpA4tYTOWBkPefTDQnb8WSGXlc88kI= +go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco= +go.opentelemetry.io/otel/sdk v1.24.0 h1:YMPPDNymmQN3ZgczicBY3B6sf9n62Dlj9pWD3ucgoDw= +go.opentelemetry.io/otel/sdk v1.24.0/go.mod h1:KVrIYw6tEubO9E96HQpcmpTKDVn9gdv35HoYiQWGDFg= +go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI= +go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU= +go.opentelemetry.io/proto/otlp v1.1.0 h1:2Di21piLrCqJ3U3eXGCTPHE9R8Nh+0uglSnOyxikMeI= +go.opentelemetry.io/proto/otlp v1.1.0/go.mod h1:GpBHCBWiqvVLDqmHZsoMM3C5ySeKTC7ej/RNTae6MdY= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= @@ -180,8 +203,8 @@ golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQz golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0= -golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.16.0 h1:QX4fJ0Rr5cPQCF7O9lh9Se4pmwfwskqZfq5moyldzic= +golang.org/x/mod v0.16.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -228,8 +251,8 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.17.0 h1:FvmRgNOcs3kOa+T20R1uhfP9F6HgG2mfxDv1vrx1Htc= -golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps= +golang.org/x/tools v0.19.0 h1:tfGCXNR1OsFG+sVdLAitlpjAvD/I6dHDKnYrpEZUHkw= +golang.org/x/tools v0.19.0/go.mod h1:qoJWxmGSIBmAeriMx19ogtrEPrGtDbPK634QFIcLAhc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -241,6 +264,14 @@ gonum.org/v1/gonum v0.0.0-20181121035319-3f7ecaa7e8ca/go.mod h1:Y+Yx5eoAFn32cQvJ gonum.org/v1/netlib v0.0.0-20181029234149-ec6d1f5cefe6/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= +google.golang.org/genproto v0.0.0-20231212172506-995d672761c0 h1:YJ5pD9rF8o9Qtta0Cmy9rdBwkSjrTCT6XTiUQVOtIos= +google.golang.org/genproto v0.0.0-20231212172506-995d672761c0/go.mod h1:l/k7rMz0vFTBPy+tFSGvXEd3z+BcoG1k7EHbqm+YBsY= +google.golang.org/genproto/googleapis/api v0.0.0-20240102182953-50ed04b92917 h1:rcS6EyEaoCO52hQDupoSfrxI3R6C2Tq741is7X8OvnM= +google.golang.org/genproto/googleapis/api v0.0.0-20240102182953-50ed04b92917/go.mod h1:CmlNWB9lSezaYELKS5Ym1r44VrrbPUa7JTvw+6MbpJ0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240102182953-50ed04b92917 h1:6G8oQ016D88m1xAKljMlBOOGWDZkes4kMhgGFlf8WcQ= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240102182953-50ed04b92917/go.mod h1:xtjpI3tXFPP051KaWnhvxkiubL/6dJ18vLVf7q2pTOU= +google.golang.org/grpc v1.61.1 h1:kLAiWrZs7YeDM6MumDe7m3y4aM6wacLzM1Y/wiLP9XY= +google.golang.org/grpc v1.61.1/go.mod h1:VUbo7IFqmF1QtCAstipjG0GIoq49KvMe9+h1jFLBNJs= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= diff --git a/internal/mode/static/config/config.go b/internal/mode/static/config/config.go index ecc60cb668..1a26cf5f03 100644 --- a/internal/mode/static/config/config.go +++ b/internal/mode/static/config/config.go @@ -9,12 +9,14 @@ import ( ) type Config struct { + // AtomicLevel is an atomically changeable, dynamic logging level. + AtomicLevel zap.AtomicLevel + // UsageReportConfig specifies the NGINX Plus usage reporting config. + UsageReportConfig *UsageReportConfig // Version is the running NGF version. Version string // ImageSource is the source of the NGINX Gateway image. ImageSource string - // AtomicLevel is an atomically changeable, dynamic logging level. - AtomicLevel zap.AtomicLevel // Flags contains the NGF command-line flag names and values. Flags Flags // GatewayNsName is the namespaced name of a Gateway resource that the Gateway will use. @@ -24,8 +26,6 @@ type Config struct { GatewayPodConfig GatewayPodConfig // Logger is the Zap Logger used by all components. Logger logr.Logger - // UsageReportConfig specifies the NGINX Plus usage reporting config. - UsageReportConfig *UsageReportConfig // GatewayCtlrName is the name of this controller. GatewayCtlrName string // ConfigName is the name of the NginxGateway resource for this controller. @@ -34,12 +34,12 @@ type Config struct { GatewayClassName string // LeaderElection contains the configuration for leader election. LeaderElection LeaderElectionConfig + // ProductTelemetryConfig contains the configuration for collecting product telemetry. + ProductTelemetryConfig ProductTelemetryConfig // MetricsConfig specifies the metrics config. MetricsConfig MetricsConfig // HealthConfig specifies the health probe config. HealthConfig HealthConfig - // ProductTelemetryConfig contains the configuration for collecting product telemetry. - ProductTelemetryConfig ProductTelemetryConfig // UpdateGatewayClassStatus enables updating the status of the GatewayClass resource. UpdateGatewayClassStatus bool // Plus indicates whether NGINX Plus is being used. @@ -90,8 +90,12 @@ type LeaderElectionConfig struct { // ProductTelemetryConfig contains the configuration for collecting product telemetry. type ProductTelemetryConfig struct { - // TelemetryReportPeriod is the period at which telemetry reports are sent. - TelemetryReportPeriod time.Duration + // Endpoint is the : of the telemetry service. + Endpoint string + // ReportPeriod is the period at which telemetry reports are sent. + ReportPeriod time.Duration + // EndpointInsecure controls if TLS should be used for the telemetry service. + EndpointInsecure bool // Enabled is the flag for toggling the collection of product telemetry. Enabled bool } diff --git a/internal/mode/static/manager.go b/internal/mode/static/manager.go index c4dcba32ab..21c123ba99 100644 --- a/internal/mode/static/manager.go +++ b/internal/mode/static/manager.go @@ -8,7 +8,9 @@ import ( "github.com/go-logr/logr" ngxclient "github.com/nginxinc/nginx-plus-go-client/client" + tel "github.com/nginxinc/telemetry-exporter/pkg/telemetry" "github.com/prometheus/client_golang/prometheus" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc" appsv1 "k8s.io/api/apps/v1" apiv1 "k8s.io/api/core/v1" discoveryV1 "k8s.io/api/discovery/v1" @@ -255,7 +257,13 @@ func StartManager(cfg config.Config) error { ImageSource: cfg.ImageSource, Flags: cfg.Flags, }) - if err = mgr.Add(createTelemetryJob(cfg, dataCollector, nginxChecker.getReadyCh())); err != nil { + + job, err := createTelemetryJob(cfg, dataCollector, nginxChecker.getReadyCh()) + if err != nil { + return fmt.Errorf("cannot create telemetry job: %w", err) + } + + if err = mgr.Add(job); err != nil { return fmt.Errorf("cannot register telemetry job: %w", err) } } @@ -467,21 +475,51 @@ func createTelemetryJob( cfg config.Config, dataCollector telemetry.DataCollector, readyCh <-chan struct{}, -) *runnables.Leader { +) (*runnables.Leader, error) { logger := cfg.Logger.WithName("telemetryJob") - exporter := telemetry.NewLoggingExporter(cfg.Logger.WithName("telemetryExporter").V(1 /* debug */)) + + var exporter telemetry.Exporter + + if cfg.ProductTelemetryConfig.Endpoint != "" { + errorHandler := tel.NewErrorHandler() + + options := []otlptracegrpc.Option{ + otlptracegrpc.WithEndpoint(cfg.ProductTelemetryConfig.Endpoint), + otlptracegrpc.WithHeaders(map[string]string{ + "X-F5-OTEL": "GRPC", + }), + } + if cfg.ProductTelemetryConfig.EndpointInsecure { + options = append(options, otlptracegrpc.WithInsecure()) + } + + var err error + exporter, err = tel.NewExporter( + tel.ExporterConfig{ + SpanProvider: tel.CreateOTLPSpanProvider(options...), + }, + tel.WithGlobalOTelLogger(logger.WithName("otel")), + tel.WithGlobalOTelErrorHandler(errorHandler), + ) + if err != nil { + return nil, fmt.Errorf("cannot create telemetry exporter: %w", err) + } + + } else { + exporter = telemetry.NewLoggingExporter(cfg.Logger.WithName("telemetryExporter").V(1 /* debug */)) + } return &runnables.Leader{ Runnable: runnables.NewCronJob( runnables.CronJobConfig{ Worker: telemetry.CreateTelemetryJobWorker(logger, exporter, dataCollector), Logger: logger, - Period: cfg.ProductTelemetryConfig.TelemetryReportPeriod, + Period: cfg.ProductTelemetryConfig.ReportPeriod, JitterFactor: telemetryJitterFactor, ReadyCh: readyCh, }, ), - } + }, nil } func createUsageReporterJob( @@ -504,7 +542,7 @@ func createUsageReporterJob( Runnable: runnables.NewCronJob(runnables.CronJobConfig{ Worker: usage.CreateUsageJobWorker(logger, k8sClient, reporter, cfg), Logger: logger, - Period: cfg.ProductTelemetryConfig.TelemetryReportPeriod, + Period: cfg.ProductTelemetryConfig.ReportPeriod, JitterFactor: telemetryJitterFactor, ReadyCh: readyCh, }), diff --git a/internal/mode/static/telemetry/collector.go b/internal/mode/static/telemetry/collector.go index 42c5749f0c..b809df6ea2 100644 --- a/internal/mode/static/telemetry/collector.go +++ b/internal/mode/static/telemetry/collector.go @@ -6,6 +6,7 @@ import ( "fmt" "runtime" + tel "github.com/nginxinc/telemetry-exporter/pkg/telemetry" appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -31,35 +32,40 @@ type ConfigurationGetter interface { GetLatestConfiguration() *dataplane.Configuration } -// NGFResourceCounts stores the counts of all relevant resources that NGF processes and generates configuration from. -type NGFResourceCounts struct { - Gateways int - GatewayClasses int - HTTPRoutes int - Secrets int - Services int - // Endpoints include the total count of Endpoints(IP:port) across all referenced services. - Endpoints int -} - -// ProjectMetadata stores the name of the project and the current version. -type ProjectMetadata struct { - Name string - Version string -} - // Data is telemetry data. -// Note: this type might change once https://github.com/nginxinc/nginx-gateway-fabric/issues/1318 is implemented. +// +//go:generate go run -tags generator github.com/nginxinc/telemetry-exporter/cmd/generator -type=Data -scheme -scheme-protocol=NGFProductTelemetry -scheme-df-datatype=ngf-product-telemetry type Data struct { - ProjectMetadata ProjectMetadata - ClusterID string - Arch string - DeploymentID string - ImageSource string - Flags config.Flags - NGFResourceCounts NGFResourceCounts - NodeCount int - NGFReplicaCount int + // ImageSource tells whether the image was built by GitHub or locally (values are 'gha', 'local', or 'unknown') + ImageSource string + tel.Data // embedding is required by the generator. + // FlagNames contains the command-line flag names. + FlagNames []string + // FlagValues contains the values of the command-line flags, where each value corresponds to the flag from FlagNames + // at the same index. + // Each value is either 'true' or 'false' for boolean flags and 'default' or 'user-defined' for non-boolean flags. + FlagValues []string + NGFResourceCounts // embedding is required by the generator. + // NGFReplicaCount is the number of replicas of the NGF Pod. + NGFReplicaCount int64 +} + +// NGFResourceCounts stores the counts of all relevant resources that NGF processes and generates configuration from. +// +//go:generate go run -tags generator github.com/nginxinc/telemetry-exporter/cmd/generator -type=NGFResourceCounts +type NGFResourceCounts struct { + // GatewayCount is the number of relevant Gateways. + GatewayCount int64 + // GatewayClassCount is the number of relevant GatewayClasses. + GatewayClassCount int64 + // HTTPRouteCount is the number of relevant HTTPRoutes. + HTTPRouteCount int64 + // SecretCount is the number of relevant Secrets. + SecretCount int64 + // ServiceCount is the number of relevant Services. + ServiceCount int64 + // EndpointCount include the total count of Endpoints(IP:port) across all referenced services. + EndpointCount int64 } // DataCollectorConfig holds configuration parameters for DataCollectorImpl. @@ -94,6 +100,9 @@ func NewDataCollectorImpl( } } +// notImplemented is a value for string field, for which collection is not implemented yet. +const notImplemented = "not-implemented" + // Collect collects and returns telemetry Data. func (c DataCollectorImpl) Collect(ctx context.Context) (Data, error) { nodeCount, err := CollectNodeCount(ctx, c.cfg.K8sClientReader) @@ -127,18 +136,21 @@ func (c DataCollectorImpl) Collect(ctx context.Context) (Data, error) { } data := Data{ - NodeCount: nodeCount, - NGFResourceCounts: graphResourceCount, - ProjectMetadata: ProjectMetadata{ - Name: "NGF", - Version: c.cfg.Version, + Data: tel.Data{ + ProjectName: "NGF", + ProjectVersion: c.cfg.Version, + ProjectArchitecture: runtime.GOARCH, + ClusterID: clusterID, + ClusterVersion: notImplemented, + ClusterPlatform: notImplemented, + InstallationID: deploymentID, + ClusterNodeCount: int64(nodeCount), }, - NGFReplicaCount: replicaCount, - ClusterID: clusterID, - ImageSource: c.cfg.ImageSource, - Arch: runtime.GOARCH, - DeploymentID: deploymentID, - Flags: c.cfg.Flags, + NGFResourceCounts: graphResourceCount, + ImageSource: c.cfg.ImageSource, + FlagNames: c.cfg.Flags.Names, + FlagValues: c.cfg.Flags.Values, + NGFReplicaCount: int64(replicaCount), } return data, nil @@ -169,23 +181,23 @@ func collectGraphResourceCount( return ngfResourceCounts, errors.New("latest configuration cannot be nil") } - ngfResourceCounts.GatewayClasses = len(g.IgnoredGatewayClasses) + ngfResourceCounts.GatewayClassCount = int64(len(g.IgnoredGatewayClasses)) if g.GatewayClass != nil { - ngfResourceCounts.GatewayClasses++ + ngfResourceCounts.GatewayClassCount++ } - ngfResourceCounts.Gateways = len(g.IgnoredGateways) + ngfResourceCounts.GatewayCount = int64(len(g.IgnoredGateways)) if g.Gateway != nil { - ngfResourceCounts.Gateways++ + ngfResourceCounts.GatewayCount++ } - ngfResourceCounts.HTTPRoutes = len(g.Routes) - ngfResourceCounts.Secrets = len(g.ReferencedSecrets) - ngfResourceCounts.Services = len(g.ReferencedServices) + ngfResourceCounts.HTTPRouteCount = int64(len(g.Routes)) + ngfResourceCounts.SecretCount = int64(len(g.ReferencedSecrets)) + ngfResourceCounts.ServiceCount = int64(len(g.ReferencedServices)) for _, upstream := range cfg.Upstreams { if upstream.ErrorMsg == "" { - ngfResourceCounts.Endpoints += len(upstream.Endpoints) + ngfResourceCounts.EndpointCount += int64(len(upstream.Endpoints)) } } diff --git a/internal/mode/static/telemetry/collector_test.go b/internal/mode/static/telemetry/collector_test.go index 271bbc0ca8..65525d0e61 100644 --- a/internal/mode/static/telemetry/collector_test.go +++ b/internal/mode/static/telemetry/collector_test.go @@ -7,6 +7,7 @@ import ( "reflect" "runtime" + tel "github.com/nginxinc/telemetry-exporter/pkg/telemetry" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" appsv1 "k8s.io/api/apps/v1" @@ -135,15 +136,21 @@ var _ = Describe("Collector", Ordered, func() { BeforeEach(func() { expData = telemetry.Data{ - ProjectMetadata: telemetry.ProjectMetadata{Name: "NGF", Version: version}, - NodeCount: 0, + Data: tel.Data{ + ProjectName: "NGF", + ProjectVersion: version, + ProjectArchitecture: runtime.GOARCH, + ClusterID: string(kubeNamespace.GetUID()), + ClusterVersion: "not-implemented", + ClusterPlatform: "not-implemented", + InstallationID: string(ngfReplicaSet.ObjectMeta.OwnerReferences[0].UID), + ClusterNodeCount: 0, + }, NGFResourceCounts: telemetry.NGFResourceCounts{}, NGFReplicaCount: 1, - ClusterID: string(kubeNamespace.GetUID()), ImageSource: "local", - Arch: runtime.GOARCH, - DeploymentID: string(ngfReplicaSet.ObjectMeta.OwnerReferences[0].UID), - Flags: flags, + FlagNames: flags.Names, + FlagValues: flags.Values, } k8sClientReader = &eventsfakes.FakeReader{} @@ -278,14 +285,14 @@ var _ = Describe("Collector", Ordered, func() { fakeGraphGetter.GetLatestGraphReturns(graph) fakeConfigurationGetter.GetLatestConfigurationReturns(config) - expData.NodeCount = 3 + expData.ClusterNodeCount = 3 expData.NGFResourceCounts = telemetry.NGFResourceCounts{ - Gateways: 3, - GatewayClasses: 3, - HTTPRoutes: 3, - Secrets: 3, - Services: 3, - Endpoints: 4, + GatewayCount: 3, + GatewayClassCount: 3, + HTTPRouteCount: 3, + SecretCount: 3, + ServiceCount: 3, + EndpointCount: 4, } data, err := dataCollector.Collect(ctx) @@ -337,7 +344,7 @@ var _ = Describe("Collector", Ordered, func() { k8sClientReader.ListCalls(createListCallsFunc(nodes)) - expData.NodeCount = 1 + expData.ClusterNodeCount = 1 data, err := dataCollector.Collect(ctx) @@ -442,12 +449,12 @@ var _ = Describe("Collector", Ordered, func() { fakeConfigurationGetter.GetLatestConfigurationReturns(config1) expData.NGFResourceCounts = telemetry.NGFResourceCounts{ - Gateways: 1, - GatewayClasses: 1, - HTTPRoutes: 1, - Secrets: 1, - Services: 1, - Endpoints: 1, + GatewayCount: 1, + GatewayClassCount: 1, + HTTPRouteCount: 1, + SecretCount: 1, + ServiceCount: 1, + EndpointCount: 1, } data, err := dataCollector.Collect(ctx) @@ -460,12 +467,12 @@ var _ = Describe("Collector", Ordered, func() { fakeGraphGetter.GetLatestGraphReturns(&graph.Graph{}) fakeConfigurationGetter.GetLatestConfigurationReturns(invalidUpstreamsConfig) expData.NGFResourceCounts = telemetry.NGFResourceCounts{ - Gateways: 0, - GatewayClasses: 0, - HTTPRoutes: 0, - Secrets: 0, - Services: 0, - Endpoints: 0, + GatewayCount: 0, + GatewayClassCount: 0, + HTTPRouteCount: 0, + SecretCount: 0, + ServiceCount: 0, + EndpointCount: 0, } data, err := dataCollector.Collect(ctx) diff --git a/internal/mode/static/telemetry/data.avdl b/internal/mode/static/telemetry/data.avdl new file mode 100644 index 0000000000..2077d28779 --- /dev/null +++ b/internal/mode/static/telemetry/data.avdl @@ -0,0 +1,69 @@ +@namespace("gateway.nginx.org") protocol NGFProductTelemetry { + @df_datatype("ngf-product-telemetry") record Data { + /** The field that identifies what type of data this is. */ + string dataType; + /** The time the event occurred */ + long eventTime; + /** The time our edge ingested the event */ + long ingestTime; + + + /** ImageSource tells whether the image was built by GitHub or locally (values are 'gha', 'local', or 'unknown') */ + string? ImageSource = null; + + /** ProjectName is the name of the project. */ + string? ProjectName = null; + + /** ProjectVersion is the version of the project. */ + string? ProjectVersion = null; + + /** ProjectArchitecture is the architecture of the project. For example, "amd64". */ + string? ProjectArchitecture = null; + + /** ClusterID is the unique id of the Kubernetes cluster where the project is installed. +It is the UID of the `kube-system` Namespace. */ + string? ClusterID = null; + + /** ClusterVersion is the Kubernetes version of the cluster. */ + string? ClusterVersion = null; + + /** ClusterPlatform is the Kubernetes platform of the cluster. */ + string? ClusterPlatform = null; + + /** InstallationID is the unique id of the project installation in the cluster. */ + string? InstallationID = null; + + /** ClusterNodeCount is the number of nodes in the cluster. */ + long? ClusterNodeCount = null; + + /** FlagNames contains the command-line flag names. */ + union {null, array} FlagNames = null; + + /** FlagValues contains the values of the command-line flags, where each value corresponds to the flag from FlagNames +at the same index. +Each value is either 'true' or 'false' for boolean flags and 'default' or 'user-defined' for non-boolean flags. */ + union {null, array} FlagValues = null; + + /** GatewayCount is the number of relevant Gateways. */ + long? GatewayCount = null; + + /** GatewayClassCount is the number of relevant GatewayClasses. */ + long? GatewayClassCount = null; + + /** HTTPRouteCount is the number of relevant HTTPRoutes. */ + long? HTTPRouteCount = null; + + /** SecretCount is the number of relevant Secrets. */ + long? SecretCount = null; + + /** ServiceCount is the number of relevant Services. */ + long? ServiceCount = null; + + /** EndpointCount include the total count of Endpoints(IP:port) across all referenced services. */ + long? EndpointCount = null; + + /** NGFReplicaCount is the number of replicas of the NGF Pod. */ + long? NGFReplicaCount = null; + + } +} diff --git a/internal/mode/static/telemetry/data_attributes_generated.go b/internal/mode/static/telemetry/data_attributes_generated.go new file mode 100644 index 0000000000..ba7c7405fa --- /dev/null +++ b/internal/mode/static/telemetry/data_attributes_generated.go @@ -0,0 +1,31 @@ + +package telemetry +/* +This is a generated file. DO NOT EDIT. +*/ + +import ( + "go.opentelemetry.io/otel/attribute" + + + ngxTelemetry "github.com/nginxinc/telemetry-exporter/pkg/telemetry" + +) + +func (d *Data) Attributes() []attribute.KeyValue { + var attrs []attribute.KeyValue + attrs = append(attrs, attribute.String("dataType", "ngf-product-telemetry")) + + + attrs = append(attrs, attribute.String("ImageSource", d.ImageSource)) + attrs = append(attrs, d.Data.Attributes()...) + attrs = append(attrs, attribute.StringSlice("FlagNames", d.FlagNames)) + attrs = append(attrs, attribute.StringSlice("FlagValues", d.FlagValues)) + attrs = append(attrs, d.NGFResourceCounts.Attributes()...) + attrs = append(attrs, attribute.Int64("NGFReplicaCount", d.NGFReplicaCount)) + + + return attrs +} + +var _ ngxTelemetry.Exportable = (*Data)(nil) diff --git a/internal/mode/static/telemetry/data_test.go b/internal/mode/static/telemetry/data_test.go new file mode 100644 index 0000000000..b18ce17cb9 --- /dev/null +++ b/internal/mode/static/telemetry/data_test.go @@ -0,0 +1,96 @@ +package telemetry + +import ( + "testing" + + tel "github.com/nginxinc/telemetry-exporter/pkg/telemetry" + . "github.com/onsi/gomega" + "go.opentelemetry.io/otel/attribute" +) + +func TestDataAttributes(t *testing.T) { + data := Data{ + ImageSource: "local", + Data: tel.Data{ + ProjectName: "NGF", + ProjectVersion: "edge", + ProjectArchitecture: "arm64", + ClusterID: "1", + ClusterVersion: "1.23", + ClusterPlatform: "test", + InstallationID: "123", + ClusterNodeCount: 3, + }, + FlagNames: []string{"test-flag"}, + FlagValues: []string{"test-value"}, + NGFResourceCounts: NGFResourceCounts{ + GatewayCount: 1, + GatewayClassCount: 2, + HTTPRouteCount: 3, + SecretCount: 4, + ServiceCount: 5, + EndpointCount: 6, + }, + NGFReplicaCount: 3, + } + + expected := []attribute.KeyValue{ + attribute.String("dataType", "ngf-product-telemetry"), + attribute.String("ImageSource", "local"), + attribute.String("ProjectName", "NGF"), + attribute.String("ProjectVersion", "edge"), + attribute.String("ProjectArchitecture", "arm64"), + attribute.String("ClusterID", "1"), + attribute.String("ClusterVersion", "1.23"), + attribute.String("ClusterPlatform", "test"), + attribute.String("InstallationID", "123"), + attribute.Int64("ClusterNodeCount", 3), + attribute.StringSlice("FlagNames", []string{"test-flag"}), + attribute.StringSlice("FlagValues", []string{"test-value"}), + attribute.Int64("GatewayCount", 1), + attribute.Int64("GatewayClassCount", 2), + attribute.Int64("HTTPRouteCount", 3), + attribute.Int64("SecretCount", 4), + attribute.Int64("ServiceCount", 5), + attribute.Int64("EndpointCount", 6), + attribute.Int64("NGFReplicaCount", 3), + } + + result := data.Attributes() + + g := NewWithT(t) + + g.Expect(result).To(Equal(expected)) +} + +func TestDataAttributesWithEmptyData(t *testing.T) { + data := Data{} + + expected := []attribute.KeyValue{ + attribute.String("dataType", "ngf-product-telemetry"), + attribute.String("ImageSource", ""), + attribute.String("ProjectName", ""), + attribute.String("ProjectVersion", ""), + attribute.String("ProjectArchitecture", ""), + attribute.String("ClusterID", ""), + attribute.String("ClusterVersion", ""), + attribute.String("ClusterPlatform", ""), + attribute.String("InstallationID", ""), + attribute.Int64("ClusterNodeCount", 0), + attribute.StringSlice("FlagNames", nil), + attribute.StringSlice("FlagValues", nil), + attribute.Int64("GatewayCount", 0), + attribute.Int64("GatewayClassCount", 0), + attribute.Int64("HTTPRouteCount", 0), + attribute.Int64("SecretCount", 0), + attribute.Int64("ServiceCount", 0), + attribute.Int64("EndpointCount", 0), + attribute.Int64("NGFReplicaCount", 0), + } + + result := data.Attributes() + + g := NewWithT(t) + + g.Expect(result).To(Equal(expected)) +} diff --git a/internal/mode/static/telemetry/exporter.go b/internal/mode/static/telemetry/exporter.go index 55bee6f2be..e70b6d3f45 100644 --- a/internal/mode/static/telemetry/exporter.go +++ b/internal/mode/static/telemetry/exporter.go @@ -4,15 +4,14 @@ import ( "context" "github.com/go-logr/logr" + tel "github.com/nginxinc/telemetry-exporter/pkg/telemetry" ) -//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 . Exporter - // Exporter exports telemetry data to some destination. -// Note: this is a temporary interface. It will be finalized once the Exporter of the common telemetry library -// https://github.com/nginxinc/nginx-gateway-fabric/issues/1318 is implemented. +// +//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 . Exporter type Exporter interface { - Export(ctx context.Context, data Data) error + Export(ctx context.Context, data tel.Exportable) error } // LoggingExporter logs telemetry data. @@ -28,7 +27,7 @@ func NewLoggingExporter(logger logr.Logger) *LoggingExporter { } // Export logs the provided telemetry data. -func (e *LoggingExporter) Export(_ context.Context, data Data) error { +func (e *LoggingExporter) Export(_ context.Context, data tel.Exportable) error { e.logger.Info("Exporting telemetry", "data", data) return nil } diff --git a/internal/mode/static/telemetry/exporter_test.go b/internal/mode/static/telemetry/exporter_test.go index 66f80606f4..1a51c6f318 100644 --- a/internal/mode/static/telemetry/exporter_test.go +++ b/internal/mode/static/telemetry/exporter_test.go @@ -16,7 +16,7 @@ func TestLoggingExporter(t *testing.T) { logger := zap.New(zap.WriteTo(&buffer)) exporter := NewLoggingExporter(logger) - err := exporter.Export(context.Background(), Data{}) + err := exporter.Export(context.Background(), &Data{}) g.Expect(err).To(BeNil()) g.Expect(buffer.String()).To(ContainSubstring(`"level":"info"`)) diff --git a/internal/mode/static/telemetry/job_worker.go b/internal/mode/static/telemetry/job_worker.go index a4dc81932a..d77189761f 100644 --- a/internal/mode/static/telemetry/job_worker.go +++ b/internal/mode/static/telemetry/job_worker.go @@ -32,7 +32,7 @@ func CreateTelemetryJobWorker( // Export telemetry logger.V(1).Info("Exporting telemetry data") - if err := exporter.Export(ctx, data); err != nil { + if err := exporter.Export(ctx, &data); err != nil { logger.Error(err, "Failed to export telemetry data") } } diff --git a/internal/mode/static/telemetry/job_worker_test.go b/internal/mode/static/telemetry/job_worker_test.go index 4e804ae26d..64a1c289c6 100644 --- a/internal/mode/static/telemetry/job_worker_test.go +++ b/internal/mode/static/telemetry/job_worker_test.go @@ -5,6 +5,7 @@ import ( "testing" "time" + tel "github.com/nginxinc/telemetry-exporter/pkg/telemetry" . "github.com/onsi/gomega" "sigs.k8s.io/controller-runtime/pkg/log/zap" @@ -21,15 +22,8 @@ func TestCreateTelemetryJobWorker(t *testing.T) { worker := telemetry.CreateTelemetryJobWorker(zap.New(), exporter, dataCollector) expData := telemetry.Data{ - ProjectMetadata: telemetry.ProjectMetadata{Name: "NGF", Version: "1.1"}, - NodeCount: 3, - NGFResourceCounts: telemetry.NGFResourceCounts{ - Gateways: 1, - GatewayClasses: 1, - HTTPRoutes: 1, - Secrets: 1, - Services: 1, - Endpoints: 1, + Data: tel.Data{ + ProjectName: "NGF", }, } dataCollector.CollectReturns(expData, nil) @@ -40,5 +34,5 @@ func TestCreateTelemetryJobWorker(t *testing.T) { worker(ctx) _, data := exporter.ExportArgsForCall(0) - g.Expect(data).To(Equal(expData)) + g.Expect(data).To(Equal(&expData)) } diff --git a/internal/mode/static/telemetry/ngfresourcecounts_attributes_generated.go b/internal/mode/static/telemetry/ngfresourcecounts_attributes_generated.go new file mode 100644 index 0000000000..19fa4ae744 --- /dev/null +++ b/internal/mode/static/telemetry/ngfresourcecounts_attributes_generated.go @@ -0,0 +1,29 @@ + +package telemetry +/* +This is a generated file. DO NOT EDIT. +*/ + +import ( + "go.opentelemetry.io/otel/attribute" + + + ngxTelemetry "github.com/nginxinc/telemetry-exporter/pkg/telemetry" + +) + +func (d *NGFResourceCounts) Attributes() []attribute.KeyValue { + var attrs []attribute.KeyValue + + attrs = append(attrs, attribute.Int64("GatewayCount", d.GatewayCount)) + attrs = append(attrs, attribute.Int64("GatewayClassCount", d.GatewayClassCount)) + attrs = append(attrs, attribute.Int64("HTTPRouteCount", d.HTTPRouteCount)) + attrs = append(attrs, attribute.Int64("SecretCount", d.SecretCount)) + attrs = append(attrs, attribute.Int64("ServiceCount", d.ServiceCount)) + attrs = append(attrs, attribute.Int64("EndpointCount", d.EndpointCount)) + + + return attrs +} + +var _ ngxTelemetry.Exportable = (*NGFResourceCounts)(nil) diff --git a/internal/mode/static/telemetry/telemetryfakes/fake_exporter.go b/internal/mode/static/telemetry/telemetryfakes/fake_exporter.go index 741ab3cdec..15c100fc62 100644 --- a/internal/mode/static/telemetry/telemetryfakes/fake_exporter.go +++ b/internal/mode/static/telemetry/telemetryfakes/fake_exporter.go @@ -6,14 +6,15 @@ import ( "sync" "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/telemetry" + telemetrya "github.com/nginxinc/telemetry-exporter/pkg/telemetry" ) type FakeExporter struct { - ExportStub func(context.Context, telemetry.Data) error + ExportStub func(context.Context, telemetrya.Exportable) error exportMutex sync.RWMutex exportArgsForCall []struct { arg1 context.Context - arg2 telemetry.Data + arg2 telemetrya.Exportable } exportReturns struct { result1 error @@ -25,12 +26,12 @@ type FakeExporter struct { invocationsMutex sync.RWMutex } -func (fake *FakeExporter) Export(arg1 context.Context, arg2 telemetry.Data) error { +func (fake *FakeExporter) Export(arg1 context.Context, arg2 telemetrya.Exportable) error { fake.exportMutex.Lock() ret, specificReturn := fake.exportReturnsOnCall[len(fake.exportArgsForCall)] fake.exportArgsForCall = append(fake.exportArgsForCall, struct { arg1 context.Context - arg2 telemetry.Data + arg2 telemetrya.Exportable }{arg1, arg2}) stub := fake.ExportStub fakeReturns := fake.exportReturns @@ -51,13 +52,13 @@ func (fake *FakeExporter) ExportCallCount() int { return len(fake.exportArgsForCall) } -func (fake *FakeExporter) ExportCalls(stub func(context.Context, telemetry.Data) error) { +func (fake *FakeExporter) ExportCalls(stub func(context.Context, telemetrya.Exportable) error) { fake.exportMutex.Lock() defer fake.exportMutex.Unlock() fake.ExportStub = stub } -func (fake *FakeExporter) ExportArgsForCall(i int) (context.Context, telemetry.Data) { +func (fake *FakeExporter) ExportArgsForCall(i int) (context.Context, telemetrya.Exportable) { fake.exportMutex.RLock() defer fake.exportMutex.RUnlock() argsForCall := fake.exportArgsForCall[i]