Skip to content

Commit

Permalink
Merge branch 'main' into envoy-1.27/extproctap-fork
Browse files Browse the repository at this point in the history
  • Loading branch information
nfuden authored Nov 30, 2023
2 parents 3653e00 + 63bda62 commit 7f3ec49
Show file tree
Hide file tree
Showing 6 changed files with 259 additions and 0 deletions.
24 changes: 24 additions & 0 deletions .github/workflows/check-extensions-build-config.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
name: check-extensions-build-config

on:
push:
branches:
- main
pull_request:

jobs:
check-extensions-build-config:
name: check-extensions-build-config
runs-on: ubuntu-20.04-8core
steps:
- uses: actions/checkout@v3
- name: Add safe directory
run: git config --global --add safe.directory /__w/envoy-gloo-ee/envoy-gloo-ee
- name: Run check-extensions-build-config.sh
run: ci/check_extensions_build_config.sh
# - name: Archive check results
# if: ${{ !cancelled() }}
# uses: actions/upload-artifact@v3
# with:
# name: static-analysis-report
# path: linux/amd64/analysis/scan-build-*/
83 changes: 83 additions & 0 deletions bazel/extensions/extensions_build_config.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -62,16 +62,25 @@ EXTENSIONS = {
#

"envoy.health_checkers.redis": "//source/extensions/health_checkers/redis:config",
# "envoy.health_checkers.thrift": "//source/extensions/health_checkers/thrift:config",
"envoy.health_checkers.tcp": "//source/extensions/health_checkers/tcp:health_checker_lib",
"envoy.health_checkers.http": "//source/extensions/health_checkers/http:health_checker_lib",
"envoy.health_checkers.grpc": "//source/extensions/health_checkers/grpc:health_checker_lib",

#
# Health check event sinks
#

# "envoy.health_check.event_sinks.file": "//source/extensions/health_check/event_sinks/file:file_sink_lib",

#
# Input Matchers
#

"envoy.matching.matchers.consistent_hashing": "//source/extensions/matching/input_matchers/consistent_hashing:config",
"envoy.matching.matchers.ip": "//source/extensions/matching/input_matchers/ip:config",
"envoy.matching.matchers.runtime_fraction": "//source/extensions/matching/input_matchers/runtime_fraction:config",
"envoy.matching.matchers.cel_matcher": "//source/extensions/matching/input_matchers/cel_matcher:config",

#
# Network Matchers
Expand All @@ -94,6 +103,17 @@ EXTENSIONS = {

"envoy.matching.common_inputs.environment_variable": "//source/extensions/matching/common_inputs/environment_variable:config",

#
# CEL Matching Input
#
# "envoy.matching.inputs.cel_data_input": "//source/extensions/matching/http/cel_input:cel_input_lib",

#
# Matching actions
#

# "envoy.matching.actions.format_string": "//source/extensions/matching/actions/format_string:config",

#
# HTTP filters
#
Expand All @@ -110,14 +130,18 @@ EXTENSIONS = {
"envoy.filters.http.compressor": "//source/extensions/filters/http/compressor:config",
"envoy.filters.http.cors": "//source/extensions/filters/http/cors:config",
"envoy.filters.http.composite": "//source/extensions/filters/http/composite:config",
# "envoy.filters.http.connect_grpc_bridge": "//source/extensions/filters/http/connect_grpc_bridge:config",
"envoy.filters.http.csrf": "//source/extensions/filters/http/csrf:config",
# "envoy.filters.http.custom_response": "//source/extensions/filters/http/custom_response:factory",
"envoy.filters.http.decompressor": "//source/extensions/filters/http/decompressor:config",
"envoy.filters.http.dynamic_forward_proxy": "//source/extensions/filters/http/dynamic_forward_proxy:config",
"envoy.filters.http.ext_authz": "//source/extensions/filters/http/ext_authz:config",
"envoy.filters.http.ext_proc": "//source/extensions/filters/http/ext_proc:config",
"envoy.filters.http.fault": "//source/extensions/filters/http/fault:config",
"envoy.filters.http.file_system_buffer": "//source/extensions/filters/http/file_system_buffer:config",
"envoy.filters.http.gcp_authn": "//source/extensions/filters/http/gcp_authn:config",
# "envoy.filters.http.geoip": "//source/extensions/filters/http/geoip:config",
# "envoy.filters.http.grpc_field_extraction": "//source/extensions/filters/http/grpc_field_extraction:config",
"envoy.filters.http.grpc_http1_bridge": "//source/extensions/filters/http/grpc_http1_bridge:config",
"envoy.filters.http.grpc_http1_reverse_bridge": "//source/extensions/filters/http/grpc_http1_reverse_bridge:config",
"envoy.filters.http.grpc_json_transcoder": "//source/extensions/filters/http/grpc_json_transcoder:config",
Expand All @@ -127,6 +151,7 @@ EXTENSIONS = {
"envoy.filters.http.health_check": "//source/extensions/filters/http/health_check:config",
"envoy.filters.http.ip_tagging": "//source/extensions/filters/http/ip_tagging:config",
"envoy.filters.http.jwt_authn": "//source/extensions/filters/http/jwt_authn:config",
# "envoy.filters.http.rate_limit_quota": "//source/extensions/filters/http/rate_limit_quota:config",
# Disabled by default
"envoy.filters.http.kill_request": "//source/extensions/filters/http/kill_request:kill_request_config",
"envoy.filters.http.local_ratelimit": "//source/extensions/filters/http/local_ratelimit:config",
Expand All @@ -141,12 +166,14 @@ EXTENSIONS = {
"envoy.filters.http.tap": "//source/extensions/filters/http/tap:config",
"envoy.filters.http.wasm": "//source/extensions/filters/http/wasm:config",
"envoy.filters.http.stateful_session": "//source/extensions/filters/http/stateful_session:config",
# "envoy.filters.http.header_mutation": "//source/extensions/filters/http/header_mutation:config",

#
# Listener filters
#

"envoy.filters.listener.http_inspector": "//source/extensions/filters/listener/http_inspector:config",
# "envoy.filters.listener.local_ratelimit": "//source/extensions/filters/listener/local_ratelimit:config",
# NOTE: The original_dst filter is implicitly loaded if original_dst functionality is
# configured on the listener. Do not remove it in that case or configs will fail to load.
"envoy.filters.listener.original_dst": "//source/extensions/filters/listener/original_dst:config",
Expand Down Expand Up @@ -191,6 +218,7 @@ EXTENSIONS = {

"envoy.resource_monitors.fixed_heap": "//source/extensions/resource_monitors/fixed_heap:config",
"envoy.resource_monitors.injected_resource": "//source/extensions/resource_monitors/injected_resource:config",
# "envoy.resource_monitors.downstream_connections": "//source/extensions/resource_monitors/downstream_connections:config",

#
# Stat sinks
Expand All @@ -200,6 +228,7 @@ EXTENSIONS = {
"envoy.stat_sinks.graphite_statsd": "//source/extensions/stat_sinks/graphite_statsd:config",
"envoy.stat_sinks.hystrix": "//source/extensions/stat_sinks/hystrix:config",
"envoy.stat_sinks.metrics_service": "//source/extensions/stat_sinks/metrics_service:config",
# "envoy.stat_sinks.open_telemetry": "//source/extensions/stat_sinks/open_telemetry:config",
"envoy.stat_sinks.statsd": "//source/extensions/stat_sinks/statsd:config",
"envoy.stat_sinks.wasm": "//source/extensions/stat_sinks/wasm:config",

Expand All @@ -209,6 +238,7 @@ EXTENSIONS = {

"envoy.filters.thrift.router": "//source/extensions/filters/network/thrift_proxy/router:config",
"envoy.filters.thrift.header_to_metadata": "//source/extensions/filters/network/thrift_proxy/filters/header_to_metadata:config",
# "envoy.filters.thrift.payload_to_metadata": "//source/extensions/filters/network/thrift_proxy/filters/payload_to_metadata:config",
"envoy.filters.thrift.rate_limit": "//source/extensions/filters/network/thrift_proxy/filters/ratelimit:config",

#
Expand Down Expand Up @@ -253,6 +283,7 @@ EXTENSIONS = {
#
# CacheFilter plugins
#
# "envoy.extensions.http.cache.file_system_http_cache": "//source/extensions/http/cache/file_system_http_cache:config",
"envoy.extensions.http.cache.simple": "//source/extensions/http/cache/simple_http_cache:config",

#
Expand All @@ -269,6 +300,7 @@ EXTENSIONS = {

"envoy.upstreams.http.http": "//source/extensions/upstreams/http/http:config",
"envoy.upstreams.http.tcp": "//source/extensions/upstreams/http/tcp:config",
# "envoy.upstreams.http.udp": "//source/extensions/upstreams/http/udp:config",

#
# Watchdog actions
Expand Down Expand Up @@ -323,13 +355,24 @@ EXTENSIONS = {
#

"envoy.http.stateful_session.cookie": "//source/extensions/http/stateful_session/cookie:config",
# "envoy.http.stateful_session.header": "//source/extensions/http/stateful_session/header:config",

#
# Custom response policies
#

# "envoy.http.custom_response.redirect_policy": "//source/extensions/http/custom_response/redirect_policy:redirect_policy_lib",
# "envoy.http.custom_response.local_response_policy": "//source/extensions/http/custom_response/local_response_policy:local_response_policy_lib",

#
# QUIC extensions
#

#"envoy.quic.deterministic_connection_id_generator": "//source/extensions/quic/connection_id_generator:envoy_deterministic_connection_id_generator_config",
"envoy.quic.crypto_stream.server.quiche": "//source/extensions/quic/crypto_stream:envoy_quic_default_crypto_server_stream",
"envoy.quic.proof_source.filter_chain": "//source/extensions/quic/proof_source:envoy_quic_default_proof_source",
#"envoy.quic.server_preferred_address.fixed": "//source/extensions/quic/server_preferred_address:fixed_server_preferred_address_config_factory_config",


#
# UDP packet writers
Expand All @@ -341,6 +384,7 @@ EXTENSIONS = {
# Formatter
#

# "envoy.formatter.cel": "//source/extensions/formatter/cel:config",
"envoy.formatter.metadata": "//source/extensions/formatter/metadata:config",
"envoy.formatter.req_without_query": "//source/extensions/formatter/req_without_query:config",

Expand Down Expand Up @@ -379,11 +423,50 @@ EXTENSIONS = {

"envoy.http.header_validators.envoy_default": "//source/extensions/http/header_validators/envoy_default:config",

#
# Path Pattern Match and Path Pattern Rewrite
#
# "envoy.path.match.uri_template.uri_template_matcher": "//source/extensions/path/match/uri_template:config",
# "envoy.path.rewrite.uri_template.uri_template_rewriter": "//source/extensions/path/rewrite/uri_template:config",

#
# Early Data option
#

"envoy.route.early_data_policy.default": "//source/extensions/early_data:default_early_data_policy_lib",

#
# Load balancing policies for upstream
#
"envoy.load_balancing_policies.least_request": "//source/extensions/load_balancing_policies/least_request:config",
"envoy.load_balancing_policies.random": "//source/extensions/load_balancing_policies/random:config",
"envoy.load_balancing_policies.round_robin": "//source/extensions/load_balancing_policies/round_robin:config",
"envoy.load_balancing_policies.maglev": "//source/extensions/load_balancing_policies/maglev:config",
"envoy.load_balancing_policies.ring_hash": "//source/extensions/load_balancing_policies/ring_hash:config",
"envoy.load_balancing_policies.subset": "//source/extensions/load_balancing_policies/subset:config",
"envoy.load_balancing_policies.cluster_provided": "//source/extensions/load_balancing_policies/cluster_provided:config",

#
# HTTP Early Header Mutation
#
# "envoy.http.early_header_mutation.header_mutation": "//source/extensions/http/early_header_mutation/header_mutation:config",


#
# Config Subscription
#

"envoy.config_subscription.rest": "//source/extensions/config_subscription/rest:http_subscription_lib",
"envoy.config_subscription.filesystem": "//source/extensions/config_subscription/filesystem:filesystem_subscription_lib",
"envoy.config_subscription.filesystem_collection": "//source/extensions/config_subscription/filesystem:filesystem_subscription_lib",
"envoy.config_subscription.grpc": "//source/extensions/config_subscription/grpc:grpc_subscription_lib",
"envoy.config_subscription.delta_grpc": "//source/extensions/config_subscription/grpc:grpc_subscription_lib",
"envoy.config_subscription.ads": "//source/extensions/config_subscription/grpc:grpc_subscription_lib",
"envoy.config_subscription.aggregated_grpc_collection": "//source/extensions/config_subscription/grpc:grpc_collection_subscription_lib",
"envoy.config_subscription.aggregated_delta_grpc_collection": "//source/extensions/config_subscription/grpc:grpc_collection_subscription_lib",
"envoy.config_subscription.ads_collection": "//source/extensions/config_subscription/grpc:grpc_collection_subscription_lib",
"envoy.config_mux.delta_grpc_mux_factory": "//source/extensions/config_subscription/grpc/xds_mux:grpc_mux_lib",
"envoy.config_mux.sotw_grpc_mux_factory": "//source/extensions/config_subscription/grpc/xds_mux:grpc_mux_lib",
}

# These can be changed to ["//visibility:public"], for downstream builds which
Expand Down
7 changes: 7 additions & 0 deletions changelog/v1.27.1-patch2/make-envoy-binary-executable.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
changelog:
- type: NON_USER_FACING
description: >
Make envoy binary executable after extracting it from the tarball
in do_ci.sh. This is intended to resolve an issue where published
envoy-gloo docker images exit on startup with a permission denied
error when trying to execute the envoy binary.
10 changes: 10 additions & 0 deletions changelog/v1.27.1-patch3/update-extensions-build-config.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
changelog:
- type: NON_USER_FACING
description: >-
Upstream moved xDs code extensions. In order to use these extensions, we have to
modify our extensions_build_config. See here:
https://www.envoyproxy.io/docs/envoy/latest/version_history/v1.27/v1.27.0
- type: NON_USER_FACING
description: >-
Introduce check-extensions-build-config script and github action to ensure that
the extensions_build_config.bzl is up to date with the current envoy version.
134 changes: 134 additions & 0 deletions ci/check_extensions_build_config.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,134 @@
#!/bin/bash

########################
### Helper functions ###
########################

# Define the upstream repository and file path to the upstream extensions_build_config.bzl
UPSTREAM_REPO="envoyproxy/envoy"
UPSTREAM_FILE_PATH="source/extensions/extensions_build_config.bzl"
# Define the path to the local repository_locations.bzl, which contains the envoy commit hash
REPOSITORY_LOCATIONS_FILE="./bazel/repository_locations.bzl"
# Define the path to the local version
ENVOY_GLOO_FILE="./bazel/extensions/extensions_build_config.bzl"

# Function to extract the envoy commit hash from repository_locations.bzl
get_envoy_commit_hash() {
local file_path="$1"
local commit_hash
commit_hash=$(grep -A 2 "envoy =" "$file_path" | grep commit | cut -d '"' -f 2)
echo "$commit_hash"
}

# Function to trim leading and trailing whitespaces
trim() {
local var="$*"
var="${var#"${var%%[![:space:]]*}"}" # remove leading whitespace characters
var="${var%"${var##*[![:space:]]}"}" # remove trailing whitespace characters
echo -n "$var"
}

# Function to report an error
report_error() {
local line_number=$1
local line_content="$2"
local github_line_url="https://github.com/$UPSTREAM_REPO/blob/$ENVOY_COMMIT_HASH/$UPSTREAM_FILE_PATH#L$line_number"
echo "Error: Line not found in the envoy-gloo version of extensions_build_config.bzl at line $line_number\n\t$line_content\n\tSee: $github_line_url"
}

########################
#### Main execution ####
########################

# Extract the envoy commit hash
ENVOY_COMMIT_HASH=$(get_envoy_commit_hash "$REPOSITORY_LOCATIONS_FILE")
if [ -z "$ENVOY_COMMIT_HASH" ]; then
echo "Error: Failed to extract envoy commit hash from $REPOSITORY_LOCATIONS_FILE"
exit 1
fi

# Update the URL to point to the specific envoy commit
UPSTREAM_URL="https://raw.githubusercontent.com/$UPSTREAM_REPO/$ENVOY_COMMIT_HASH/$UPSTREAM_FILE_PATH"

curl -s "$UPSTREAM_URL" -o upstream_file.tmp
if [ $? -ne 0 ]; then
echo "Error: Failed to fetch the upstream file from $UPSTREAM_URL"
exit 1
fi

# Initialize arrays to store the upstream and envoy-gloo versions of the file
declare -a UPSTREAM_LINES ENVOY_GLOO_LINES

# Read the upstream version of the file into an array, including empty lines
UPSTREAM_LINES=()
while IFS= read -r line || [[ -n "$line" ]]; do
UPSTREAM_LINES+=("$line")
done < upstream_file.tmp

# Read the envoy-gloo version of the file into an array, including empty lines
ENVOY_GLOO_LINES=()
while IFS= read -r line || [[ -n "$line" ]]; do
ENVOY_GLOO_LINES+=("$line")
done < "$ENVOY_GLOO_FILE"

# Preprocess the envoy-gloo lines for faster lookup
declare -a PROCESSED_ENVOY_GLOO_LINES
for line in "${ENVOY_GLOO_LINES[@]}"; do
# Remove only the first leading '#' character (if present) and any spaces before it
trimmed_line=$(echo "$line" | sed 's/^[[:space:]]*#//')

# remove any remaining whitespace
trimmed_line=$(trim "$trimmed_line")

PROCESSED_ENVOY_GLOO_LINES+=("$trimmed_line")
done

# Flag to track if any issues are found
ISSUES_FOUND=false
ERRORS=()

# Loop through each line in the upstream version
LINE_NUMBER=0
for UPSTREAM_LINE in "${UPSTREAM_LINES[@]}"; do
((LINE_NUMBER++))
TRIMMED_UPSTREAM_LINE=$(trim "$UPSTREAM_LINE")

# Skip if the line is empty or a comment
if [[ -z "$TRIMMED_UPSTREAM_LINE" || "$TRIMMED_UPSTREAM_LINE" =~ ^# ]]; then
continue
fi

# Reject any lines that don't match the format "key": "value".
# Additional spaces around the colon and any content within the quotes are allowed.
if ! [[ "$TRIMMED_UPSTREAM_LINE" =~ ^[^\"]*\"[^\"]*\":\ *\"[^\"]*\" ]]; then
continue
fi

# Check if the trimmed line exists in the processed envoy-gloo lines
LINE_FOUND=false
for PROCESSED_LINE in "${PROCESSED_ENVOY_GLOO_LINES[@]}"; do
if [[ "$PROCESSED_LINE" == "$TRIMMED_UPSTREAM_LINE" ]]; then
LINE_FOUND=true
break
fi
done

if [ "$LINE_FOUND" = false ]; then
ERRORS+=("$(report_error "$LINE_NUMBER" "$UPSTREAM_LINE")")
ISSUES_FOUND=true
fi
done

# Check if any issues were found
if [ "$ISSUES_FOUND" = true ]; then
for err in "${ERRORS[@]}"; do
echo -e "$err"
done
echo "Envoy-gloo extensions_build_config.bzl is not up to date with the upstream version."
exit 1
else
echo "Envoy-gloo extensions_build_config.bzl is up to date with the upstream version."
fi

# Cleanup
rm upstream_file.tmp
1 change: 1 addition & 0 deletions ci/do_ci.sh
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@ mkdir -p "$ENVOY_GLOO_BIN_DIR"
bazel run @envoy//tools/zstd:zstd -- --stdout -d /build/envoy/x64/bin/release.tar.zst \
| tar xfO - envoy > "$ENVOY_GLOO_BIN_DIR/envoy"

chmod +x "${ENVOY_GLOO_BIN_DIR}/envoy"
ENVOY_GLOO_STRIPPED_DIR="${ENVOY_GLOO_BIN_DIR}_stripped"
mkdir -p "$ENVOY_GLOO_STRIPPED_DIR"
cp "${ENVOY_GLOO_BIN_DIR}/envoy" "${ENVOY_GLOO_STRIPPED_DIR}/envoy"
Expand Down

0 comments on commit 7f3ec49

Please sign in to comment.