appsec: Service Extension callout

.github/workflows/service-extensions-publish.yml

@@ -0,0 +1,130 @@
+name: Publish Service Extensions Callout images packages
+
+on:
+  push:
+    tags:
+      - 'v*.*'
+  workflow_dispatch:
+    inputs:
+      tag_name:
+        description: 'Docker image tag to use for the package'
+        required: true
+        default: 'dev'
+      commit_sha:
+        description: 'Commit SHA to checkout'
+        required: true
+      set_as_latest:
+        description: 'Set the tag as latest'
+        required: false
+        default: 'false'
+
+permissions:
+  contents: read
+  packages: write
+
+env:
+  TAG_NAME: ${{ github.ref_name || github.event.inputs.tag_name }}
+  REF_NAME: ${{ github.ref || github.event.inputs.commit_sha }}
+  COMMIT_SHA: ${{ github.sha || github.event.inputs.commit_sha }}
+  PUSH_LATEST: ${{ github.event.inputs.set_as_latest || 'true' }}
+  REGISTRY_IMAGE: ghcr.io/datadog/dd-trace-go/service-extensions-callout
+
+jobs:
+  build-service-extensions:
+    runs-on: ${{ matrix.platform == 'linux/amd64' && 'ubuntu-latest' || 'arm-4core-linux' }}
+    strategy:
+      matrix:
+        platform: [ linux/amd64, linux/arm64 ]
+
+    steps:
+      - name: Prepare
+        run: |
+          platform=${{ matrix.platform }}
+          echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV
+
+      - name: Checkout
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          ref: ${{ env.REF_NAME }}
+
+      - name: Install Docker (only arm64)
+        if: matrix.platform == 'linux/arm64'
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y docker.io
+          sudo systemctl start docker
+          sudo systemctl enable docker
+          sudo usermod -aG docker $USER
+          newgrp docker
+          sudo chmod 666 /var/run/docker.sock
+
+      - name: Set up Docker Buildx
+        uses: docker/[email protected]
+
+      - name: Login to Docker
+        shell: bash
+        run: docker login -u publisher -p ${{ secrets.GITHUB_TOKEN }} ghcr.io
+
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.REGISTRY_IMAGE }}
+
+      - name: Build and push by digest
+        id: build
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: ./contrib/envoyproxy/go-control-plane/cmd/serviceextensions/Dockerfile
+          platforms: ${{ matrix.platform }}
+          labels: ${{ steps.meta.outputs.labels }}
+          outputs: type=image,name=${{ env.REGISTRY_IMAGE }},push-by-digest=true,name-canonical=true,push=true
+
+      - name: Export digest
+        run: |
+          mkdir -p /tmp/digests
+          digest="${{ steps.build.outputs.digest }}"
+          touch "/tmp/digests/${digest#sha256:}"
+
+      - name: Upload digest
+        uses: actions/upload-artifact@v4
+        with:
+          name: digests-${{ env.PLATFORM_PAIR }}
+          path: /tmp/digests/*
+          if-no-files-found: error
+          retention-days: 1
+
+  publish-service-extensions:
+    runs-on: ubuntu-latest
+    needs:
+      - build-service-extensions
+
+    steps:
+      - name: Download digests
+        uses: actions/download-artifact@v4
+        with:
+          path: /tmp/digests
+          pattern: digests-*
+          merge-multiple: true
+
+      - name: Set up Docker Buildx
+        uses: docker/[email protected]
+
+      - name: Login to Docker
+        shell: bash
+        run: docker login -u publisher -p ${{ secrets.GITHUB_TOKEN }} ghcr.io
+
+      - name: Create tags
+        id: tags
+        run: |
+            tagname=${TAG_NAME//\//-} # remove slashes from tag name
+            echo "tags=-t ghcr.io/datadog/dd-trace-go/service-extensions-callout:${tagname} \
+            -t ghcr.io/datadog/dd-trace-go/service-extensions-callout:${{ env.COMMIT_SHA }} \
+            ${{ env.PUSH_LATEST == 'true' && '-t ghcr.io/datadog/dd-trace-go/service-extensions-callout:latest' }}" >> $GITHUB_OUTPUT
+
+      - name: Create manifest list and push
+        working-directory: /tmp/digests
+        run: |
+          docker buildx imagetools create ${{ steps.tags.outputs.tags }} \
+            $(printf '${{ env.REGISTRY_IMAGE }}@sha256:%s ' *)

contrib/envoyproxy/go-control-plane/cmd/serviceextensions/.gitignore

@@ -0,0 +1 @@
+serviceextensions

contrib/envoyproxy/go-control-plane/cmd/serviceextensions/Dockerfile

@@ -0,0 +1,35 @@
+# Build stage
+FROM golang:1.23-alpine AS builder
+ENV CGO_ENABLED=1
+
+WORKDIR /app
+COPY . .
+
+RUN apk add --no-cache --update git build-base openssl
+
+# Generate SSL self-signed localhost certificate
+RUN openssl genrsa -out localhost.key 3072
+RUN openssl req -new \
+  -key localhost.key \
+  -subj "/C=US/ST=New York/O=Datadog/OU=gRPC/CN=localhost" \
+  -out request.csr 
+RUN openssl x509 -req -days 3660 \
+  -in request.csr \
+  -signkey localhost.key \
+  -out localhost.crt
+
+# Build the serviceextensions binary
+RUN go build -tags=appsec -o ./contrib/envoyproxy/go-control-plane/cmd/serviceextensions/serviceextensions ./contrib/envoyproxy/go-control-plane/cmd/serviceextensions
+
+# Runtime stage
+FROM alpine:3.20.3
+RUN apk --no-cache add ca-certificates tzdata libc6-compat libgcc libstdc++
+WORKDIR /app
+COPY --from=builder /app/contrib/envoyproxy/go-control-plane/cmd/serviceextensions/serviceextensions /app/serviceextensions
+COPY --from=builder /app/localhost.crt /app/localhost.crt
+COPY --from=builder /app/localhost.key /app/localhost.key
+
+EXPOSE 80
+EXPOSE 443
+
+CMD ["./serviceextensions"]

contrib/envoyproxy/go-control-plane/cmd/serviceextensions/README.md

@@ -0,0 +1,42 @@
+# ASM Service Extension
+
+[GCP Services Extensions](https://cloud.google.com/service-extensions/docs/overview) enable Google Cloud users to provide programmability and extensibility on Cloud Load Balancing data paths and at the edge.
+
+## Installation
+
+### From Release
+
+This package provides a docker image to be used with Google Cloud Service Extensions.
+The images are published at each release of the tracer and can be found in [the repo registry](https://github.com/DataDog/dd-trace-go/pkgs/container/dd-trace-go%2Fservice-extensions-callout).
+
+### Build image
+
+The docker image can be build locally using docker. Start by cloning the `dd-trace-go` repo, `cd` inside it and run that command:
+```sh
+docker build --build-arg -f contrib/envoyproxy/go-control-plane/cmd/serviceextensions/Dockerfile -t datadog/dd-trace-go/service-extensions-callout:local .
+```
+
+## Configuration
+
+The ASM Service Extension expose some configuration. The configuration can be tweaked if the Service Extension is only used as an External Processor for Envoy that is not operated by GCP.
+
+>**GCP requires that the default configuration for the Service Extension should not change.**
+
+| Environment variable | Default value | Description |
+|---|---|---|
+| `DD_SERVICE_EXTENSION_HOST` | `0.0.0.0` | Host on where the gRPC and HTTP server should listen to. |
+| `DD_SERVICE_EXTENSION_PORT` | `443` | Port used by the gRPC Server.<br>Envoy Google backend’s is only using secure connection to Service Extension. |
+| `DD_SERVICE_EXTENSION_HEALTHCHECK_PORT` | `80` | Port used for the HTTP server for the health check. |
+
+> The Service Extension need to be connected to a deployed [Datadog agent](https://docs.datadoghq.com/agent).
+
+| Environment variable | Default value | Description |
+|---|---|---|
+| `DD_AGENT_HOST` | `N/A` | Host of a running Datadog Agent. |
+| `DD_TRACE_AGENT_PORT` | `8126` | Port of a running Datadog Agent. |
+
+### SSL Configuration
+
+The Envoy of GCP is configured to communicate to the Service Extension with TLS.
+
+`localhost` self signed certificates are generated and bundled into the ASM Service Extension docker image and loaded at the start of the gRPC server.

contrib/envoyproxy/go-control-plane/cmd/serviceextensions/main.go

@@ -0,0 +1,164 @@
+// Unless explicitly stated otherwise all files in this repository are licensed
+// under the Apache License Version 2.0.
+// This product includes software developed at Datadog (https://www.datadoghq.com/).
+// Copyright 2016 Datadog, Inc.
+
+package main
+
+import (
+	"context"
+	"crypto/tls"
+	"errors"
+	"fmt"
+	"net"
+	"net/http"
+	"os"
+	"os/signal"
+	"strconv"
+	"syscall"
+	"time"
+
+	"golang.org/x/sync/errgroup"
+
+	gocontrolplane "gopkg.in/DataDog/dd-trace-go.v1/contrib/envoyproxy/go-control-plane"
+	"gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer"
+	"gopkg.in/DataDog/dd-trace-go.v1/internal"
+	"gopkg.in/DataDog/dd-trace-go.v1/internal/log"
+	"gopkg.in/DataDog/dd-trace-go.v1/internal/version"
+
+	extproc "github.com/envoyproxy/go-control-plane/envoy/service/ext_proc/v3"
+	"github.com/gorilla/mux"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials"
+)
+
+// AppsecCalloutExtensionService defines the struct that follows the ExternalProcessorServer interface.
+type AppsecCalloutExtensionService struct {
+	extproc.ExternalProcessorServer
+}
+
+type serviceExtensionConfig struct {
+	extensionPort   string
+	extensionHost   string
+	healthcheckPort string
+}
+
+func loadConfig() serviceExtensionConfig {
+	extensionPortInt := internal.IntEnv("DD_SERVICE_EXTENSION_PORT", 443)
+	healthcheckPortInt := internal.IntEnv("DD_SERVICE_EXTENSION_HEALTHCHECK_PORT", 80)
+	extensionHostStr := internal.IpEnv("DD_SERVICE_EXTENSION_HOST", net.IP{0, 0, 0, 0}).String()
+
+	extensionPortStr := strconv.FormatInt(int64(extensionPortInt), 10)
+	healthcheckPortStr := strconv.FormatInt(int64(healthcheckPortInt), 10)
+
+	return serviceExtensionConfig{
+		extensionPort:   extensionPortStr,
+		extensionHost:   extensionHostStr,
+		healthcheckPort: healthcheckPortStr,
+	}
+}
+
+func main() {
+	// Set the DD_VERSION to the current tracer version if not set
+	if os.Getenv("DD_VERSION") == "" {
+		if err := os.Setenv("DD_VERSION", version.Tag); err != nil {
+			log.Error("service_extension: failed to set DD_VERSION environment variable: %v\n", err)
+		}
+	}
+
+	config := loadConfig()
+
+	if err := startService(config); err != nil {
+		log.Error("service_extension: %v\n", err)
+		log.Flush()
+		os.Exit(1)
+	}
+
+	log.Info("service_extension: shutting down\n")
+}
+
+func startService(config serviceExtensionConfig) error {
+	var extensionService AppsecCalloutExtensionService
+
+	tracer.Start(tracer.WithAppSecEnabled(true))
+	defer tracer.Stop()
+	// TODO: Enable ASM standalone mode when it is developed (should be done for Q4 2024)
+
+	ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
+	defer cancel()
+	g, ctx := errgroup.WithContext(ctx)
+
+	g.Go(func() error {
+		return startGPRCSsl(ctx, &extensionService, config)
+	})
+
+	g.Go(func() error {
+		return startHealthCheck(ctx, config)
+	})
+
+	if err := g.Wait(); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func startHealthCheck(ctx context.Context, config serviceExtensionConfig) error {
+	muxServer := mux.NewRouter()
+	muxServer.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/json")
+		w.WriteHeader(http.StatusOK)
+		w.Write([]byte(`{"status": "ok", "library": {"language": "golang", "version": "` + version.Tag + `"}}`))
+	})
+
+	server := &http.Server{
+		Addr:    config.extensionHost + ":" + config.healthcheckPort,
+		Handler: muxServer,
+	}
+
+	go func() {
+		<-ctx.Done()
+		shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+		defer cancel()
+		if err := server.Shutdown(shutdownCtx); err != nil {
+			log.Error("service_extension: health check server shutdown: %v\n", err)
+		}
+	}()
+
+	log.Info("service_extension: health check server started on %s:%s\n", config.extensionHost, config.healthcheckPort)
+	if err := server.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
+		return fmt.Errorf("health check http server: %v", err)
+	}
+
+	return nil
+}
+
+func startGPRCSsl(ctx context.Context, service extproc.ExternalProcessorServer, config serviceExtensionConfig) error {
+	cert, err := tls.LoadX509KeyPair("localhost.crt", "localhost.key")
+	if err != nil {
+		return fmt.Errorf("failed to load key pair: %v", err)
+	}
+
+	lis, err := net.Listen("tcp", config.extensionHost+":"+config.extensionPort)
+	if err != nil {
+		return fmt.Errorf("gRPC server: %v", err)
+	}
+
+	grpcCredentials := credentials.NewServerTLSFromCert(&cert)
+	grpcServer := grpc.NewServer(grpc.Creds(grpcCredentials))
+
+	appsecEnvoyExternalProcessorServer := gocontrolplane.AppsecEnvoyExternalProcessorServer(service)
+
+	go func() {
+		<-ctx.Done()
+		grpcServer.GracefulStop()
+	}()
+
+	extproc.RegisterExternalProcessorServer(grpcServer, appsecEnvoyExternalProcessorServer)
+	log.Info("service_extension: callout gRPC server started on %s:%s\n", config.extensionHost, config.extensionPort)
+	if err := grpcServer.Serve(lis); err != nil {
+		return fmt.Errorf("error starting gRPC server: %v", err)
+	}
+
+	return nil
+}

contrib/internal/telemetrytest/telemetry_test.go

@@ -83,7 +83,7 @@ func TestTelemetryEnabled(t *testing.T) {
 		packages = append(packages, out)
 	}
 	for _, pkg := range packages {
-		if strings.Contains(pkg.ImportPath, "/test") || strings.Contains(pkg.ImportPath, "/internal") {
+		if strings.Contains(pkg.ImportPath, "/test") || strings.Contains(pkg.ImportPath, "/internal") || strings.Contains(pkg.ImportPath, "/cmd") {
 			continue
 		}
 		if !pkg.hasTelemetryImport(t) {

ddtrace/tracer/option_test.go

@@ -343,7 +343,7 @@ func TestIntegrationEnabled(t *testing.T) {
 		packages = append(packages, out)
 	}
 	for _, pkg := range packages {
-		if strings.Contains(pkg.ImportPath, "/test") || strings.Contains(pkg.ImportPath, "/internal") {
+		if strings.Contains(pkg.ImportPath, "/test") || strings.Contains(pkg.ImportPath, "/internal") || strings.Contains(pkg.ImportPath, "/cmd") {
 			continue
 		}
 		p := strings.Replace(pkg.Dir, pkg.Root, "../..", 1)