Skip to content

Commit

Permalink
docs: gRPC Access Log Service (ALS) sink (envoyproxy#3768)
Browse files Browse the repository at this point in the history
* docs: gRPC Access Log Service (ALS) sink

Signed-off-by: zirain <[email protected]>

* ignore githubusercontent.com

Signed-off-by: zirain <[email protected]>

* update

Signed-off-by: zirain <[email protected]>

---------

Signed-off-by: zirain <[email protected]>
Co-authored-by: Guy Daich <[email protected]>
  • Loading branch information
zirain and guydc committed Jul 22, 2024
1 parent b9a9fff commit 6fb5349
Show file tree
Hide file tree
Showing 4 changed files with 291 additions and 2 deletions.
231 changes: 231 additions & 0 deletions examples/kubernetes/envoy-als.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,231 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: envoy-als
data:
go.mod: |
module envoy-als
go 1.22
require (
github.com/envoyproxy/go-control-plane v0.12.0
github.com/prometheus/client_golang v1.19.1
google.golang.org/grpc v1.64.0
)
require (
github.com/beorn7/perks v1.0.1 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/cncf/xds/go v0.0.0-20240318125728-8a4994d93e50 // indirect
github.com/envoyproxy/protoc-gen-validate v1.0.4 // indirect
github.com/golang/protobuf v1.5.4 // indirect
github.com/prometheus/client_model v0.5.0 // indirect
github.com/prometheus/common v0.48.0 // indirect
github.com/prometheus/procfs v0.12.0 // indirect
golang.org/x/net v0.22.0 // indirect
golang.org/x/sys v0.18.0 // indirect
golang.org/x/text v0.14.0 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 // indirect
google.golang.org/protobuf v1.33.0 // indirect
)
go.sum: |
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/cncf/xds/go v0.0.0-20240318125728-8a4994d93e50 h1:DBmgJDC9dTfkVyGgipamEh2BpGYxScCH1TOF1LL1cXc=
github.com/cncf/xds/go v0.0.0-20240318125728-8a4994d93e50/go.mod h1:5e1+Vvlzido69INQaVO6d87Qn543Xr6nooe9Kz7oBFM=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/envoyproxy/go-control-plane v0.12.0 h1:4X+VP1GHd1Mhj6IB5mMeGbLCleqxjletLK6K0rbxyZI=
github.com/envoyproxy/go-control-plane v0.12.0/go.mod h1:ZBTaoJ23lqITozF0M6G4/IragXCQKCnYbmlmtHvwRG0=
github.com/envoyproxy/protoc-gen-validate v1.0.4 h1:gVPz/FMfvh57HdSJQyvBtF00j8JU4zdyUgIUNhlgg0A=
github.com/envoyproxy/protoc-gen-validate v1.0.4/go.mod h1:qys6tmnRsYrQqIhm2bvKZH4Blx/1gTIZ2UKVY1M+Yew=
github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE=
github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho=
github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw=
github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI=
github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSzKKE=
github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc=
github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo=
github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo=
golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc=
golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 h1:NnYq6UN9ReLM9/Y01KWNOWyI5xQ9kbIms5GGJVwS/Yc=
google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY=
google.golang.org/grpc v1.64.0 h1:KH3VH9y/MgNQg1dE7b3XfVK0GsPSIzJwdF617gUSbvY=
google.golang.org/grpc v1.64.0/go.mod h1:oxjF8E3FBnjp+/gVFYdWacaLDx9na1aqy9oovLpxQYg=
google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI=
google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
main.go: |
package main
import (
"log"
"net"
"net/http"
alsv2 "github.com/envoyproxy/go-control-plane/envoy/service/accesslog/v2"
alsv3 "github.com/envoyproxy/go-control-plane/envoy/service/accesslog/v3"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promhttp"
"google.golang.org/grpc"
)
var (
LogCount = prometheus.NewCounterVec(prometheus.CounterOpts{
Name: "log_count",
Help: "The total number of logs received.",
}, []string{"api_version"})
)
func init() {
// Register the summary and the histogram with Prometheus's default registry.
prometheus.MustRegister(LogCount)
}
type ALSServer struct {
}
func (a *ALSServer) StreamAccessLogs(logStream alsv2.AccessLogService_StreamAccessLogsServer) error {
log.Println("Streaming als v2 logs")
for {
data, err := logStream.Recv()
if err != nil {
return err
}
httpLogs := data.GetHttpLogs()
if httpLogs != nil {
LogCount.WithLabelValues("v2").Add(float64(len(httpLogs.LogEntry)))
}
log.Printf("Received v2 log data: %s\n", data.String())
}
}
type ALSServerV3 struct {
}
func (a *ALSServerV3) StreamAccessLogs(logStream alsv3.AccessLogService_StreamAccessLogsServer) error {
log.Println("Streaming als v3 logs")
for {
data, err := logStream.Recv()
if err != nil {
return err
}
httpLogs := data.GetHttpLogs()
if httpLogs != nil {
LogCount.WithLabelValues("v3").Add(float64(len(httpLogs.LogEntry)))
}
log.Printf("Received v3 log data: %s\n", data.String())
}
}
func NewALSServer() *ALSServer {
return &ALSServer{}
}
func NewALSServerV3() *ALSServerV3 {
return &ALSServerV3{}
}
func main() {
mux := http.NewServeMux()
if err := addMonitor(mux); err != nil {
log.Printf("could not establish self-monitoring: %v\n", err)
}
s := &http.Server{
Addr: ":19001",
Handler: mux,
}
go func() {
s.ListenAndServe()
}()
listener, err := net.Listen("tcp", "0.0.0.0:8080")
if err != nil {
log.Fatalf("Failed to start listener on port 8080: %v", err)
}
var opts []grpc.ServerOption
grpcServer := grpc.NewServer(opts...)
alsv2.RegisterAccessLogServiceServer(grpcServer, NewALSServer())
alsv3.RegisterAccessLogServiceServer(grpcServer, NewALSServerV3())
log.Println("Starting ALS Server")
if err := grpcServer.Serve(listener); err != nil {
log.Fatalf("grpc serve err: %v", err)
}
}
func addMonitor(mux *http.ServeMux) error {
mux.Handle("/metrics", promhttp.HandlerFor(prometheus.DefaultGatherer, promhttp.HandlerOpts{EnableOpenMetrics: true}))
return nil
}
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: envoy-als
spec:
replicas: 1
selector:
matchLabels:
app: envoy-als
template:
metadata:
annotations:
prometheus.io/scrape: "true"
prometheus.io/port: "19001"
labels:
app: envoy-als
spec:
containers:
- name: envoy-als
command:
- sh
- "-c"
- "cp -a /app /app-live && cd /app-live && go run . "
image: golang:1.22.3-alpine
ports:
- containerPort: 8080
- containerPort: 19001
volumeMounts:
- name: envoy-als
mountPath: /app
volumes:
- name: envoy-als
configMap:
name: envoy-als
---
apiVersion: v1
kind: Service
metadata:
name: envoy-als
spec:
selector:
app: envoy-als
type: LoadBalancer
ports:
- name: grpc-als
protocol: TCP
appProtocol: grpc
port: 8080
targetPort: 8080
- name: http-monitoring
protocol: TCP
port: 19001
targetPort: 19001
53 changes: 52 additions & 1 deletion site/content/en/latest/tasks/observability/proxy-accesslog.md
Original file line number Diff line number Diff line change
Expand Up @@ -140,6 +140,57 @@ Verify logs from loki:
curl -s "http://$LOKI_IP:3100/loki/api/v1/query_range" --data-urlencode "query={exporter=\"OTLP\"}" | jq '.data.result[0].values'
```

## gGRPC Access Log Service(ALS) Sink

Envoy Gateway can send logs to a backend implemented [gRPC access log service proto](https://www.envoyproxy.io/docs/envoy/latest/api-v3/service/accesslog/v3/als.proto).
There's an example service [here](https://raw.githubusercontent.com/envoyproxy/gateway/main/examples/kubernetes/envoy-als.yaml), which simply count the log and export to prometheus endpoint.

```shell
kubectl apply -f https://raw.githubusercontent.com/envoyproxy/gateway/main/examples/kubernetes/envoy-als.yaml -n monitoring
```

The following configuration sends logs to the gRPC access log service:

```shell
kubectl apply -f - <<EOF
apiVersion: gateway.networking.k8s.io/v1
kind: GatewayClass
metadata:
name: eg
spec:
controllerName: gateway.envoyproxy.io/gatewayclass-controller
parametersRef:
group: gateway.envoyproxy.io
kind: EnvoyProxy
name: als
namespace: envoy-gateway-system
---
apiVersion: gateway.envoyproxy.io/v1alpha1
kind: EnvoyProxy
metadata:
name: als
namespace: envoy-gateway-system
spec:
telemetry:
accessLog:
settings:
- sinks:
- type: ALS
als:
backendRefs:
- name: envoy-als
namespace: monitoring
port: 8080
type: HTTP
EOF
```

Verify logs from envoy-als:

```shell
curl -s "http://$(kubectl get svc envoy-als -n monitoring -o jsonpath='{.status.loadBalancer.ingress[0].ip}'):19001/metrics" | grep log_count
```

## CEL Expressions

Envoy Gateway provides [CEL expressions](https://www.envoyproxy.io/docs/envoy/latest/xds/type/v3/cel.proto.html#common-expression-language-cel-proto) to filter access log .
Expand Down Expand Up @@ -197,4 +248,4 @@ curl -s "http://$LOKI_IP:3100/loki/api/v1/query_range" --data-urlencode "query={
Envoy Gateway provides additional metadata about the K8s resources that were translated to certain envoy resources.
For example, details about the `HTTPRoute` and `GRPCRoute` (kind, group, name, namespace and annotations) are available
for access log formatter using the `METADATA` operator. To enrich logs, users can add log operator such as:
`%METADATA(ROUTE:envoy-gateway:resources)%` to their access log format.
`%METADATA(ROUTE:envoy-gateway:resources)%` to their access log format.
6 changes: 6 additions & 0 deletions tools/hack/create-cluster.sh
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,10 @@ for _ in $(seq 1 "${NUM_WORKERS}"); do
done
fi

## Check if kind cluster already exists.
if tools/bin/kind get clusters | grep -q "${CLUSTER_NAME}"; then
echo "Cluster ${CLUSTER_NAME} already exists."
else
## Create kind cluster.
if [[ -z "${KIND_NODE_TAG}" ]]; then
cat << EOF | tools/bin/kind create cluster --name "${CLUSTER_NAME}" --config -
Expand All @@ -34,6 +38,8 @@ else
${KIND_CFG}
EOF
fi
fi


## Install MetalLB.
kubectl apply -f https://raw.githubusercontent.com/metallb/metallb/"${METALLB_VERSION}"/config/manifests/metallb-native.yaml
Expand Down
3 changes: 2 additions & 1 deletion tools/make/docs.mk
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
DOCS_OUTPUT_DIR := site/public
RELEASE_VERSIONS ?= $(foreach v,$(wildcard ${ROOT_DIR}/docs/*),$(notdir ${v}))
LINKINATOR_IGNORE := "github.com githubusercontent.com example.com github.io _print v0.6.0 v0.5.0 v0.4.0 v0.3.0 v0.2.0"
CLEAN_NODE_MODULES ?= true

##@ Docs
Expand Down Expand Up @@ -111,7 +112,7 @@ docs-check-links:
# github.com does not allow access too often, there are a lot of 429 errors
# TODO: find a way to remove github.com from ignore list
# TODO: example.com is not a valid domain, we should remove it from ignore list
linkinator site/public/ -r --concurrency 25 --skip "github.com example.com github.io _print v0.6.0 v0.5.0 v0.4.0 v0.3.0 v0.2.0"
linkinator site/public/ -r --concurrency 25 --skip $(LINKINATOR_IGNORE)

release-notes-docs: $(tools/release-notes-docs)
@$(LOG_TARGET)
Expand Down

0 comments on commit 6fb5349

Please sign in to comment.