diff --git a/Cargo.toml b/Cargo.toml index 078c3de9f4..d20bc4303b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -44,34 +44,35 @@ quilkin-macros = { version = "0.3.0-dev", path = "./macros" } base64 = "0.13.0" base64-serde = "0.6.1" bytes = { version = "1.1.0", features = ["serde"] } -clap = { version = "3.1", features = ["cargo"] } +clap = { version = "3.1", features = ["cargo", "derive", "env"] } dashmap = "4.0.2" either = "1.6.1" +eyre = "0.6.5" +futures = "0.3.17" hyper = "0.14.15" +ipnetwork = "0.18.0" num_cpus = "1.13.0" parking_lot = "0.11.2" prometheus = { version = "0.13.0", default-features = false } prost = "=0.9.0" prost-types = "=0.9.0" rand = "0.8.4" +regex = "1.5.4" +schemars = { version = "0.8.8", features = ["bytes"] } serde = { version = "1.0.130", features = ["derive", "rc"] } serde_json = "1.0.68" serde_regex = "1.1.0" serde_yaml = "0.8.21" snap = "1.0.5" +stable-eyre = "0.2.2" +thiserror = "1.0.30" tokio = { version = "1.16.1", features = ["rt-multi-thread", "signal", "test-util", "parking_lot"] } tokio-stream = "0.1.8" tonic = "0.6.1" tracing = {version = "0.1"} tracing-subscriber = { version = "0.3", features = ["json"] } -uuid = { version = "0.8.2", default-features = false, features = ["v4"] } -thiserror = "1.0.30" tryhard = "0.4.0" -eyre = "0.6.5" -stable-eyre = "0.2.2" -ipnetwork = "0.18.0" -futures = "0.3.17" -regex = "1.5.4" +uuid = { version = "0.8.2", default-features = false, features = ["v4"] } [target.'cfg(target_os = "linux")'.dependencies] sys-info = "0.9.0" diff --git a/build/build-image/Dockerfile b/build/build-image/Dockerfile index 03568c01da..9895540e2d 100644 --- a/build/build-image/Dockerfile +++ b/build/build-image/Dockerfile @@ -26,7 +26,7 @@ ENV RUSTUP_HOME=/usr/local/rustup \ # Install packages RUN set -eux && \ apt-get update && \ - apt-get install -y wget zip build-essential libssl-dev pkg-config python3-pip && \ + apt-get install -y jq wget zip build-essential libssl-dev pkg-config python3-pip && \ pip3 install live-server # Install Go diff --git a/docs/book.toml b/docs/book.toml index 420102ca06..f308768dcc 100644 --- a/docs/book.toml +++ b/docs/book.toml @@ -4,4 +4,10 @@ multilingual = false src = "src" title = "Quilkin Book" +[preprocessor.links] +after = ["quilkin"] + +[preprocessor.quilkin] +command = "./preprocessor.sh" + [output.html] diff --git a/docs/preprocessor.sh b/docs/preprocessor.sh new file mode 100755 index 0000000000..7f0026b036 --- /dev/null +++ b/docs/preprocessor.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +cargo run -q --manifest-path ../Cargo.toml -- -q generate-config-schema -o ../target + +echo $(jq -M -c .[1] <&0) diff --git a/docs/src/SUMMARY.md b/docs/src/SUMMARY.md index 8b8b02743a..f2a7e444a4 100644 --- a/docs/src/SUMMARY.md +++ b/docs/src/SUMMARY.md @@ -11,13 +11,14 @@ - [Proxy Configuration](./proxy-configuration.md) - [Filters](./filters.md) - [Capture](./filters/capture.md) - - [Concatenate Bytes](./filters/concatenate_bytes.md) - [Compress](./filters/compress.md) + - [Concatenate Bytes](./filters/concatenate_bytes.md) - [Debug](./filters/debug.md) + - [Firewall](./filters/firewall.md) - [Load Balancer](./filters/load_balancer.md) - [Local Rate Limit](./filters/local_rate_limit.md) + - [Matches](./filters/matches.md) - [Token Router](./filters/token_router.md) - - [Firewall](./filters/firewall.md) - [Writing Custom Filters](./filters/writing_custom_filters.md) - [Integrations](./integrations.md) - [Administration](./admin.md) diff --git a/docs/src/filters/capture.md b/docs/src/filters/capture.md index c5fd428606..4cce2680b4 100644 --- a/docs/src/filters/capture.md +++ b/docs/src/filters/capture.md @@ -5,7 +5,7 @@ The `CaptureBytes` filter's job is to find a series of bytes within a packet, an down the chain. This is often used as a way of retrieving authentication tokens from a packet, and used in combination with -[ConcatenateBytes](./concatenate_bytes.md) and +[ConcatenateBytes](./concatenate_bytes.md) and [TokenRouter](token_router.md) filter to provide common packet routing utilities. ### Capture strategies @@ -52,36 +52,12 @@ static: ### Configuration Options ([Rust Doc](../../api/quilkin/filters/capture/struct.Config.html)) ```yaml -properties: - strategy: - type: object - description: | - The selected strategy for capturing the series of bytes from the incoming packet. - - SUFFIX: Retrieve bytes from the end of the packet. - - PREFIX: Retrieve bytes from the beginnning of the packet. - default: "SUFFIX" - enum: ['PREFIX', 'SUFFIX'] - metadataKey: - type: string - default: quilkin.dev/captured - description: | - The key under which the captured bytes are stored in the Filter invocation values. - size: - type: integer - description: | - The number of bytes in the packet to capture using the applied strategy. - remove: - type: boolean - default: false - description: | - Whether or not to remove the captured bytes from the packet before passing it along to the next filter in the - chain. - required: ['size'] +{{#include ../../../target/quilkin.extensions.filters.capture.v1alpha1.yaml}} ``` ### Metrics -* `quilkin_filter_Capture_packets_dropped` +* `quilkin_filter_Capture_packets_dropped` A counter of the total number of packets that have been dropped due to their length being less than the configured `size`. diff --git a/docs/src/filters/compress.md b/docs/src/filters/compress.md index 3fa6de92c7..eee5c5396b 100644 --- a/docs/src/filters/compress.md +++ b/docs/src/filters/compress.md @@ -1,6 +1,6 @@ # Compress -The `Compress` filter's job is to provide a variety of compression implementations for compression +The `Compress` filter's job is to provide a variety of compression implementations for compression and subsequent decompression of UDP data when sent between systems, such as a game client and game server. #### Filter name @@ -27,9 +27,9 @@ static: # quilkin::Builder::from(std::sync::Arc::new(config)).validate().unwrap(); ``` -The above example shows a proxy that could be used with a typical game client, where the original client data is -sent to the local listening port and then compressed when heading up to a dedicated game server, and then -decompressed when traffic is returned from the dedicated game server before being handed back to game client. +The above example shows a proxy that could be used with a typical game client, where the original client data is +sent to the local listening port and then compressed when heading up to a dedicated game server, and then +decompressed when traffic is returned from the dedicated game server before being handed back to game client. > It is worth noting that since the Compress filter modifies the *entire packet*, it is worth paying special attention to the order it is placed in your [Filter configuration](../filters.md). Most of the time it will likely be @@ -38,38 +38,14 @@ decompressed when traffic is returned from the dedicated game server before bein ### Configuration Options ([Rust Doc](../../api/quilkin/filters/compress/struct.Config.html)) ```yaml -properties: - on_read: - '$ref': '#/definitions/action' - description: | - Whether to compress, decompress or do nothing when reading packets from the local listening port - on_write: - '$ref': '#/definitions/action' - description: | - Whether to compress, decompress or do nothing when writing packets to the local listening port - mode: - type: string - description: | - The compression implementation to use on the incoming and outgoing packets. See "Compression Modes" for details. - enum: - - SNAPPY - default: SNAPPY - -definitions: - action: - type: string - enum: - - DO_NOTHING - - COMPRESS - - DECOMPRESS - default: DO_NOTHING +{{#include ../../../target/quilkin.extensions.filters.compress.v1alpha1.yaml}} ``` #### Compression Modes ##### Snappy -> Snappy is a compression/decompression library. It does not aim for maximum compression, or compatibility with any +> Snappy is a compression/decompression library. It does not aim for maximum compression, or compatibility with any > other compression library; instead, it aims for very high speeds and reasonable compression. Currently, this filter only provides the [Snappy](https://github.com/google/snappy/) compression format via the @@ -78,7 +54,7 @@ provided in the future. ### Metrics * `quilkin_filter_Compress_packets_dropped_total` - Total number of packets dropped as they could not be processed. + Total number of packets dropped as they could not be processed. * Labels: * `action`: The action that could not be completed successfully, thereby causing the packet to be dropped. * `Compress`: Compressing the packet with the configured `mode` was attempted. diff --git a/docs/src/filters/concatenate_bytes.md b/docs/src/filters/concatenate_bytes.md index de3e0874cf..165e309a13 100644 --- a/docs/src/filters/concatenate_bytes.md +++ b/docs/src/filters/concatenate_bytes.md @@ -30,23 +30,7 @@ static: ### Configuration Options ([Rust Doc](../../api/quilkin/filters/concatenate_bytes/struct.Config.html)) ```yaml -properties: - on_read: - type: string - description: | - Either append or prepend the `bytes` data to each packet filtered on read of the listening port. - default: DO_NOTHING - enum: ['DO_NOTHING', 'APPEND', 'PREPEND'] - on_write: - type: string - description: | - Either append or prepend the `bytes` data to each packet filtered on write of the listening port. - default: DO_NOTHING - enum: ['DO_NOTHING', 'APPEND', 'PREPEND'] - bytes: - type: string - description: | - Base64 encoded string of the byte array to add to each packet as it is filtered. +{{#include ../../../target/quilkin.extensions.filters.concatenate_bytes.v1alpha1.yaml}} ``` ### Metrics diff --git a/docs/src/filters/debug.md b/docs/src/filters/debug.md index 4d312571db..385655623b 100644 --- a/docs/src/filters/debug.md +++ b/docs/src/filters/debug.md @@ -29,11 +29,7 @@ static: ### Configuration Options ([Rust Doc](../../api/quilkin/filters/debug/struct.Config.html)) ```yaml -properties: - id: - type: string - description: | - An identifier that will be included with each log message. +{{#include ../../../target/quilkin.extensions.filters.debug.v1alpha1.yaml}} ``` diff --git a/docs/src/filters/firewall.md b/docs/src/filters/firewall.md index 691f504a52..166fef2d46 100644 --- a/docs/src/filters/firewall.md +++ b/docs/src/filters/firewall.md @@ -38,41 +38,7 @@ static: ### Configuration Options ([Rust Doc](../../api/quilkin/filters/firewall/struct.Config.html)) ```yaml -properties: - on_read: - '$ref': '#/definitions/rules' - description: Rules to match against when reading packets to the local listening port. - on_write: - type: array - '$ref': '#/definitions/rules' - description: Rules to match against when writing packets to the local listening port. - -definitions: - rules: - type: array - description: Rules to match against when writing packets to the local listening port. - items: - type: object - properties: - action: - type: string - description: | - Whether or not a matching Rule should Allow or Deny access - - DENY: If the rule matches, block the traffic. - - ALLOW: If the rule matches, allow the traffic through. - enum: ['ALLOW', 'DENY'] - source: - type: string - description: A CIDR network range, either in a v4 or v6 format. - ports: - type: array - description: Array of singular ports or port ranges to match against. - items: - type: string - description: | - Either in the format of "10" for a singular port or "10-100" for a port range where - min is inclusive, and max is exclusive. - required: ['action', 'source', 'ports'] +{{#include ../../../target/quilkin.extensions.filters.firewall.v1alpha1.yaml}} ``` #### Rule Evaluation diff --git a/docs/src/filters/load_balancer.md b/docs/src/filters/load_balancer.md index efa2ea7662..2cedc146ea 100644 --- a/docs/src/filters/load_balancer.md +++ b/docs/src/filters/load_balancer.md @@ -33,16 +33,7 @@ In the example above, packets will be distributed by selecting endpoints in turn ### Configuration Options ([Rust Doc](../../api/quilkin/filters/load_balancer/struct.Config.html)) ```yaml -properties: - policy: - type: string - description: | - The load balancing policy with which to distribute packets among endpoints. - enum: - - ROUND_ROBIN # Send packets by selecting endpoints in turn. - - RANDOM # Send packets by randomly selecting endpoints. - - HASH # Send packets by hashing the source IP and port. - default: ROUND_ROBIN +{{#include ../../../target/quilkin.extensions.filters.load_balancer.v1alpha1.yaml}} ``` ### Metrics diff --git a/docs/src/filters/local_rate_limit.md b/docs/src/filters/local_rate_limit.md index 7e92d38afa..85ced02f74 100644 --- a/docs/src/filters/local_rate_limit.md +++ b/docs/src/filters/local_rate_limit.md @@ -42,21 +42,7 @@ To configure a rate limiter, we specify the maximum rate at which the proxy is a ### Configuration Options ([Rust Doc](../../api/quilkin/filters/local_rate_limit/struct.Config.html)) ```yaml -properties: - max_packets: - type: integer - description: | - The maximum number of packets allowed to be forwarded over the given duration. - minimum: 0 - - period: - type: string - description: | - The duration in seconds overwhich `max_packets` applies. - default: 1 # 1 second - minimum: 1 - -required: [ 'max_packets' ] +{{#include ../../../target/quilkin.extensions.filters.local_rate_limit.v1alpha1.yaml}} ``` diff --git a/docs/src/filters/matches.md b/docs/src/filters/matches.md index 0456b953b2..3b09649bdc 100644 --- a/docs/src/filters/matches.md +++ b/docs/src/filters/matches.md @@ -40,4 +40,10 @@ static: # quilkin::Builder::from(std::sync::Arc::new(config)).validate().unwrap(); ``` +### Configuration Options ([Rust Doc](../../api/quilkin/filters/matches/struct.Config.html)) + +```yaml +{{#include ../../../target/quilkin.extensions.filters.matches.v1alpha1.yaml}} +``` + View the [Matches](../../api/quilkin/filters/matches/struct.Config.html) filter documentation for more details. diff --git a/docs/src/filters/token_router.md b/docs/src/filters/token_router.md index 814da5cbf1..9c106612d1 100644 --- a/docs/src/filters/token_router.md +++ b/docs/src/filters/token_router.md @@ -43,12 +43,7 @@ View the [CaptureBytes](./capture.md) filter documentation for more details. ### Configuration Options ([Rust Doc](../../api/quilkin/filters/token_router/struct.Config.html)) ```yaml -properties: - metadataKey: - type: string - default: quilkin.dev/captured - description: | - The key under which the token is stored in the Filter dynamic metadata. +{{#include ../../../target/quilkin.extensions.filters.token_router.v1alpha1.yaml}} ``` ### Metrics diff --git a/docs/src/filters/writing_custom_filters.md b/docs/src/filters/writing_custom_filters.md index 568d3a95a4..0b751558c6 100644 --- a/docs/src/filters/writing_custom_filters.md +++ b/docs/src/filters/writing_custom_filters.md @@ -111,6 +111,11 @@ impl FilterFactory for GreetFilterFactory { fn name(&self) -> &'static str { NAME } + + fn config_schema(&self) -> schemars::schema::RootSchema { + schemars::schema_for!(serde_json::Value) + } + fn create_filter(&self, _: CreateFilterArgs) -> Result { let filter: Box = Box::new(Greet); Ok(FilterInstance::new(serde_json::Value::Null, filter)) @@ -235,6 +240,11 @@ impl FilterFactory for GreetFilterFactory { fn name(&self) -> &'static str { NAME } + + fn config_schema(&self) -> schemars::schema::RootSchema { + schemars::schema_for!(serde_json::Value) + } + fn create_filter(&self, args: CreateFilterArgs) -> Result { let config = match args.config.unwrap() { ConfigType::Static(config) => { diff --git a/docs/src/quickstart-agones-xonotic.md b/docs/src/quickstart-agones-xonotic.md index b3f4e2e6d3..40077db573 100644 --- a/docs/src/quickstart-agones-xonotic.md +++ b/docs/src/quickstart-agones-xonotic.md @@ -112,7 +112,7 @@ replace the `${GAMESERVER_IP}` and `${GAMESERVER_PORT}` values in your copy of ` Run this configuration locally as: ```shell -quilkin run -c ./client-compress.yaml` +quilkin -c ./client-compress.yaml run` ``` Now we can connect to the local client proxy on "127.0.0.1:7000" via the "Multiplayer > Address" field in the diff --git a/docs/src/quickstart-netcat.md b/docs/src/quickstart-netcat.md index 8ed8fe7b57..f95dcc9b4b 100644 --- a/docs/src/quickstart-netcat.md +++ b/docs/src/quickstart-netcat.md @@ -38,13 +38,13 @@ a single endpoint of 127.0.0.1, port 8000. Let's start Quilkin with the above configuration: ```shell -quilkin run --config proxy.yaml +quilkin --config proxy.yaml run ``` You should see an output like the following: ```shell -$ quilkin run --config proxy.yaml +$ quilkin --config proxy.yaml run {"msg":"Starting Quilkin","level":"INFO","ts":"2021-04-25T19:27:22.535174615-07:00","source":"run","version":"0.1.0-dev"} {"msg":"Starting","level":"INFO","ts":"2021-04-25T19:27:22.535315827-07:00","source":"server::Server","port":7000} {"msg":"Starting admin endpoint","level":"INFO","ts":"2021-04-25T19:27:22.535550572-07:00","source":"proxy::Admin","address":"[::]:9091"} diff --git a/docs/src/using.md b/docs/src/using.md index 6e7940daa3..076a561134 100644 --- a/docs/src/using.md +++ b/docs/src/using.md @@ -15,7 +15,7 @@ The release binary can be downloaded from the Quilkin needs to be run with an accompanying [configuration file](./proxy-configuration.md), like so: -`quilkin run --config="configuration.yaml"` +`quilkin --config="configuration.yaml" run` To view debug output, run the same command with the `quilkin-debug` binary. diff --git a/examples/agones-xonotic/README.md b/examples/agones-xonotic/README.md index e6cc9c24a8..77f8bdefad 100644 --- a/examples/agones-xonotic/README.md +++ b/examples/agones-xonotic/README.md @@ -25,7 +25,7 @@ Instead of connecting Xonotic directly, take the IP and port from the Agones hos `${GAMESERVER_IP}` and `${GAMESERVER_PORT}` values in a local copy of `client-compress.yaml`. Run this configuration locally as: -`quilkin run -c ./client-compress.yaml` +`quilkin -c ./client-compress.yaml run` From there connect to the local client proxy on "127.0.0.1:7000" via the "Multiplayer > Address" field in the Xonotic client, and Quilkin will take care of compressing the data for you without having to change either the diff --git a/examples/quilkin-filter-example/Cargo.toml b/examples/quilkin-filter-example/Cargo.toml index e2db318aa9..c00639925b 100644 --- a/examples/quilkin-filter-example/Cargo.toml +++ b/examples/quilkin-filter-example/Cargo.toml @@ -34,6 +34,7 @@ prost-types = "0.9.0" serde = "1.0" serde_yaml = "0.8" bytes = "1.1.0" +schemars = "0.8.8" [build-dependencies] prost-build = "0.9.0" diff --git a/examples/quilkin-filter-example/src/main.rs b/examples/quilkin-filter-example/src/main.rs index ec8d731d01..5b97fef648 100644 --- a/examples/quilkin-filter-example/src/main.rs +++ b/examples/quilkin-filter-example/src/main.rs @@ -24,7 +24,7 @@ use serde::{Deserialize, Serialize}; use std::convert::TryFrom; // ANCHOR: serde_config -#[derive(Serialize, Deserialize, Debug)] +#[derive(Serialize, Deserialize, Debug, schemars::JsonSchema)] struct Config { greeting: String, } @@ -71,6 +71,11 @@ impl FilterFactory for GreetFilterFactory { fn name(&self) -> &'static str { NAME } + + fn config_schema(&self) -> schemars::schema::RootSchema { + schemars::schema_for!(Config) + } + fn create_filter(&self, args: CreateFilterArgs) -> Result { let (config_json, config) = self .require_config(args.config)? @@ -84,8 +89,11 @@ impl FilterFactory for GreetFilterFactory { // ANCHOR: run #[tokio::main] async fn main() { - quilkin::run(vec![self::factory()].into_iter()) - .await - .unwrap(); + quilkin::run( + quilkin::Config::builder().build(), + vec![self::factory()].into_iter(), + ) + .await + .unwrap(); } // ANCHOR_END: run diff --git a/image/Dockerfile b/image/Dockerfile index cf5c2c374b..2815d34470 100644 --- a/image/Dockerfile +++ b/image/Dockerfile @@ -28,4 +28,4 @@ COPY ./target/x86_64-unknown-linux-gnu/debug/quilkin . FROM $PROFILE USER nonroot:nonroot -ENTRYPOINT ["/quilkin", "run", "--config", "/etc/quilkin/quilkin.yaml"] +ENTRYPOINT ["/quilkin", "--config", "/etc/quilkin/quilkin.yaml", "run"] diff --git a/src/config.rs b/src/config.rs index 37419868e6..bb464a6007 100644 --- a/src/config.rs +++ b/src/config.rs @@ -56,35 +56,9 @@ pub struct Config { } impl Config { - /// Attempts to locate and parse a `Config` located at either `path`, the - /// `$QUILKIN_CONFIG` environment variable if set, the current directory, - /// or the `/etc/quilkin` directory (on unix platforms only). Returns an - /// error if the found configuration is invalid, or if no configuration - /// could be found at any location. - pub fn find(path: Option<&str>) -> crate::Result { - const ENV_CONFIG_PATH: &str = "QUILKIN_CONFIG"; - const CONFIG_FILE: &str = "quilkin.yaml"; - - let config_env = std::env::var(ENV_CONFIG_PATH).ok(); - - let config_path = std::path::Path::new( - path.or_else(|| config_env.as_deref()) - .unwrap_or(CONFIG_FILE), - ) - .canonicalize()?; - - tracing::info!(path = %config_path.display(), "Found configuration file"); - - std::fs::File::open(&config_path) - .or_else(|error| { - if cfg!(unix) { - std::fs::File::open("/etc/quilkin/quilkin.yaml") - } else { - Err(error) - } - }) - .map_err(From::from) - .and_then(|file| Self::from_reader(file).map_err(From::from)) + /// Returns a new empty [`Builder`] for [`Config`]. + pub fn builder() -> Builder { + Builder::empty() } /// Attempts to deserialize `input` as a YAML object representing `Self`. diff --git a/src/config/config_type.rs b/src/config/config_type.rs index db42137884..c1f5d19763 100644 --- a/src/config/config_type.rs +++ b/src/config/config_type.rs @@ -22,11 +22,13 @@ use crate::filters::{ConvertProtoConfigError, Error}; /// The configuration of a [`Filter`][crate::filters::Filter] from either a /// static or dynamic source. -#[derive(Clone, Debug, PartialEq)] +#[derive(Clone, Debug, PartialEq, schemars::JsonSchema)] pub enum ConfigType { /// Static configuration from YAML. + #[schemars(with = "serde_json::Value")] Static(serde_yaml::Value), /// Dynamic configuration from Protobuf. + #[schemars(skip)] Dynamic(prost_types::Any), } diff --git a/src/filters/capture.rs b/src/filters/capture.rs index 00998575cb..aef3203db0 100644 --- a/src/filters/capture.rs +++ b/src/filters/capture.rs @@ -96,6 +96,10 @@ impl FilterFactory for CaptureFactory { NAME } + fn config_schema(&self) -> schemars::schema::RootSchema { + schemars::schema_for!(Config) + } + fn create_filter(&self, args: CreateFilterArgs) -> Result { let (config_json, config) = self .require_config(args.config)? diff --git a/src/filters/capture/affix.rs b/src/filters/capture/affix.rs index d1ae743ec0..1bc2525cf5 100644 --- a/src/filters/capture/affix.rs +++ b/src/filters/capture/affix.rs @@ -18,7 +18,7 @@ fn is_valid_size(contents: &[u8], size: u32, metrics: &Metrics) -> bool { } /// Capture from the start of the packet. -#[derive(serde::Serialize, serde::Deserialize, Debug, PartialEq)] +#[derive(serde::Serialize, serde::Deserialize, Debug, PartialEq, schemars::JsonSchema)] pub struct Prefix { /// Whether captured bytes are removed from the original packet. #[serde(default)] @@ -40,7 +40,7 @@ impl super::CaptureStrategy for Prefix { } /// Capture from the end of the packet. -#[derive(serde::Serialize, serde::Deserialize, Debug, PartialEq)] +#[derive(serde::Serialize, serde::Deserialize, Debug, PartialEq, schemars::JsonSchema)] pub struct Suffix { /// Whether captured bytes are removed from the original packet. pub size: u32, diff --git a/src/filters/capture/config.rs b/src/filters/capture/config.rs index 139b826f65..42df80818d 100644 --- a/src/filters/capture/config.rs +++ b/src/filters/capture/config.rs @@ -22,7 +22,7 @@ use super::{proto, Prefix, Regex, Suffix}; use crate::filters::{metadata::CAPTURED_BYTES, ConvertProtoConfigError}; /// Strategy to apply for acquiring a set of bytes in the UDP packet -#[derive(Serialize, Deserialize, Debug, PartialEq)] +#[derive(Serialize, Deserialize, Debug, PartialEq, schemars::JsonSchema)] #[serde(tag = "kind")] pub enum Strategy { /// Looks for the set of bytes at the beginning of the packet @@ -46,7 +46,7 @@ impl Strategy { } } -#[derive(Debug, PartialEq)] +#[derive(Debug, PartialEq, schemars::JsonSchema)] pub struct Config { /// The key to use when storing the captured value in the filter context. /// If a match was found it is available diff --git a/src/filters/capture/regex.rs b/src/filters/capture/regex.rs index e8ee3da42f..0f52c0d3a5 100644 --- a/src/filters/capture/regex.rs +++ b/src/filters/capture/regex.rs @@ -3,10 +3,11 @@ use crate::metadata::Value; use super::Metrics; /// Capture from the start of the packet. -#[derive(serde::Serialize, serde::Deserialize, Debug)] +#[derive(serde::Serialize, serde::Deserialize, Debug, schemars::JsonSchema)] pub struct Regex { /// The regular expression to use for capture. #[serde(with = "serde_regex")] + #[schemars(with = "String")] pub pattern: regex::bytes::Regex, } diff --git a/src/filters/compress.rs b/src/filters/compress.rs index a691f67354..57f4cd917f 100644 --- a/src/filters/compress.rs +++ b/src/filters/compress.rs @@ -158,6 +158,10 @@ impl FilterFactory for CompressFactory { NAME } + fn config_schema(&self) -> schemars::schema::RootSchema { + schemars::schema_for!(Config) + } + fn create_filter(&self, args: CreateFilterArgs) -> Result { let (config_json, config) = self .require_config(args.config)? diff --git a/src/filters/compress/config.rs b/src/filters/compress/config.rs index 77d8530d7a..e361695c9b 100644 --- a/src/filters/compress/config.rs +++ b/src/filters/compress/config.rs @@ -16,6 +16,7 @@ use std::convert::TryFrom; +use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use super::compressor::{Compressor, Snappy}; @@ -25,7 +26,7 @@ use super::quilkin::extensions::filters::compress::v1alpha1::{ use crate::{filters::ConvertProtoConfigError, map_proto_enum}; /// The library to use when compressing. -#[derive(Clone, Copy, Deserialize, Debug, PartialEq, Serialize)] +#[derive(Clone, Copy, Deserialize, Debug, PartialEq, Serialize, JsonSchema)] #[non_exhaustive] pub enum Mode { // we only support one mode for now, but adding in the config option to @@ -49,7 +50,7 @@ impl Default for Mode { } /// Whether to do nothing, compress or decompress the packet. -#[derive(Clone, Copy, Deserialize, Debug, PartialEq, Serialize)] +#[derive(Clone, Copy, Deserialize, Debug, PartialEq, Serialize, JsonSchema)] pub enum Action { #[serde(rename = "DO_NOTHING")] DoNothing, @@ -65,7 +66,7 @@ impl Default for Action { } } -#[derive(Clone, Copy, Deserialize, Debug, PartialEq, Serialize)] +#[derive(Clone, Copy, Deserialize, Debug, PartialEq, Serialize, JsonSchema)] #[non_exhaustive] pub struct Config { #[serde(default)] diff --git a/src/filters/concatenate_bytes.rs b/src/filters/concatenate_bytes.rs index 1bb4c0f433..f61768fcc6 100644 --- a/src/filters/concatenate_bytes.rs +++ b/src/filters/concatenate_bytes.rs @@ -86,6 +86,10 @@ impl FilterFactory for ConcatBytesFactory { NAME } + fn config_schema(&self) -> schemars::schema::RootSchema { + schemars::schema_for!(Config) + } + fn create_filter(&self, args: CreateFilterArgs) -> Result { let (config_json, config) = self .require_config(args.config)? diff --git a/src/filters/concatenate_bytes/config.rs b/src/filters/concatenate_bytes/config.rs index 329ac71f99..f7e574aeef 100644 --- a/src/filters/concatenate_bytes/config.rs +++ b/src/filters/concatenate_bytes/config.rs @@ -19,6 +19,7 @@ crate::include_proto!("quilkin.extensions.filters.concatenate_bytes.v1alpha1"); use std::convert::TryFrom; use base64_serde::base64_serde_type; +use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use crate::{filters::prelude::*, map_proto_enum}; @@ -29,7 +30,7 @@ pub use self::quilkin::extensions::filters::concatenate_bytes::v1alpha1::Concate base64_serde_type!(Base64Standard, base64::STANDARD); -#[derive(Serialize, Deserialize, Debug, PartialEq)] +#[derive(Serialize, Deserialize, Debug, PartialEq, JsonSchema)] pub enum Strategy { #[serde(rename = "APPEND")] Append, @@ -46,7 +47,7 @@ impl Default for Strategy { } /// Config represents a `ConcatenateBytes` filter configuration. -#[derive(Serialize, Deserialize, Debug, PartialEq)] +#[derive(Serialize, Deserialize, Debug, PartialEq, JsonSchema)] #[non_exhaustive] pub struct Config { /// Whether or not to `append` or `prepend` or `do nothing` on Filter `Read` @@ -56,7 +57,10 @@ pub struct Config { #[serde(default)] pub on_write: Strategy, - #[serde(with = "Base64Standard")] + #[serde( + deserialize_with = "Base64Standard::deserialize", + serialize_with = "Base64Standard::serialize" + )] pub bytes: Vec, } diff --git a/src/filters/debug.rs b/src/filters/debug.rs index 08d453034d..6a8d4c3535 100644 --- a/src/filters/debug.rs +++ b/src/filters/debug.rs @@ -80,6 +80,10 @@ impl FilterFactory for DebugFactory { NAME } + fn config_schema(&self) -> schemars::schema::RootSchema { + schemars::schema_for!(Config) + } + fn create_filter(&self, args: CreateFilterArgs) -> Result { let config: Option<(_, Config)> = args .config @@ -99,7 +103,7 @@ impl FilterFactory for DebugFactory { } /// A Debug filter's configuration. -#[derive(Serialize, Deserialize, Debug)] +#[derive(Serialize, Deserialize, Debug, schemars::JsonSchema)] pub struct Config { /// Identifier that will be optionally included with each log message. pub id: Option, diff --git a/src/filters/factory.rs b/src/filters/factory.rs index cb929d2a0c..d317613d12 100644 --- a/src/filters/factory.rs +++ b/src/filters/factory.rs @@ -58,6 +58,9 @@ pub trait FilterFactory: Sync + Send { /// `quilkin.extensions.filters.debug_filter.v1alpha1.Debug` fn name(&self) -> &'static str; + /// Returns the schema for the configuration of the [`Filter`]. + fn config_schema(&self) -> schemars::schema::RootSchema; + /// Returns a filter based on the provided arguments. fn create_filter(&self, args: CreateFilterArgs) -> Result; diff --git a/src/filters/firewall.rs b/src/filters/firewall.rs index bce31df397..c7eaf89b95 100644 --- a/src/filters/firewall.rs +++ b/src/filters/firewall.rs @@ -49,6 +49,10 @@ impl FilterFactory for FirewallFactory { NAME } + fn config_schema(&self) -> schemars::schema::RootSchema { + schemars::schema_for!(Config) + } + fn create_filter(&self, args: CreateFilterArgs) -> Result { let (config_json, config) = self .require_config(args.config)? diff --git a/src/filters/firewall/config.rs b/src/filters/firewall/config.rs index 8e5776588d..3c5d38ca7c 100644 --- a/src/filters/firewall/config.rs +++ b/src/filters/firewall/config.rs @@ -17,6 +17,7 @@ use std::{convert::TryFrom, fmt, fmt::Formatter, net::SocketAddr, ops::Range}; use ipnetwork::IpNetwork; +use schemars::JsonSchema; use serde::de::{self, Visitor}; use serde::{Deserialize, Deserializer, Serialize, Serializer}; @@ -29,7 +30,7 @@ use super::quilkin::extensions::filters::firewall::v1alpha1::{ /// Represents how a Firewall filter is configured for read and write /// operations. -#[derive(Clone, Deserialize, Debug, PartialEq, Serialize)] +#[derive(Clone, Deserialize, Debug, PartialEq, Serialize, JsonSchema)] #[non_exhaustive] pub struct Config { pub on_read: Vec, @@ -37,7 +38,7 @@ pub struct Config { } /// Whether or not a matching [Rule] should Allow or Deny access -#[derive(Clone, Deserialize, Debug, PartialEq, Serialize)] +#[derive(Clone, Deserialize, Debug, PartialEq, Serialize, JsonSchema)] pub enum Action { /// Matching rules will allow packets through. #[serde(rename = "ALLOW")] @@ -48,10 +49,11 @@ pub enum Action { } /// Combination of CIDR range, port range and action to take. -#[derive(Clone, Deserialize, Debug, PartialEq, Serialize)] +#[derive(Clone, Deserialize, Debug, PartialEq, Serialize, JsonSchema)] pub struct Rule { pub action: Action, /// ipv4 or ipv6 CIDR address. + #[schemars(with = "String")] pub source: IpNetwork, pub ports: Vec, } @@ -98,7 +100,7 @@ pub enum PortRangeError { } /// Range of matching ports that are configured against a [Rule]. -#[derive(Clone, Debug, PartialEq)] +#[derive(Clone, Debug, PartialEq, JsonSchema)] pub struct PortRange(Range); impl PortRange { diff --git a/src/filters/load_balancer.rs b/src/filters/load_balancer.rs index 3d36ae95df..84f446be50 100644 --- a/src/filters/load_balancer.rs +++ b/src/filters/load_balancer.rs @@ -50,6 +50,10 @@ impl FilterFactory for LoadBalancerFilterFactory { NAME } + fn config_schema(&self) -> schemars::schema::RootSchema { + schemars::schema_for!(Config) + } + fn create_filter(&self, args: CreateFilterArgs) -> Result { let (config_json, config) = self .require_config(args.config)? diff --git a/src/filters/load_balancer/config.rs b/src/filters/load_balancer/config.rs index 453ed65a6a..19d123e630 100644 --- a/src/filters/load_balancer/config.rs +++ b/src/filters/load_balancer/config.rs @@ -18,6 +18,7 @@ crate::include_proto!("quilkin.extensions.filters.load_balancer.v1alpha1"); use std::convert::TryFrom; +use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use self::quilkin::extensions::filters::load_balancer::v1alpha1::load_balancer::Policy as ProtoPolicy; @@ -29,7 +30,7 @@ use crate::{filters::ConvertProtoConfigError, map_proto_enum}; pub use self::quilkin::extensions::filters::load_balancer::v1alpha1::LoadBalancer as ProtoConfig; /// The configuration for [`load_balancer`][super]. -#[derive(Serialize, Deserialize, Debug, PartialEq)] +#[derive(Serialize, Deserialize, Debug, PartialEq, JsonSchema)] #[non_exhaustive] pub struct Config { #[serde(default)] @@ -59,7 +60,7 @@ impl TryFrom for Config { /// Policy represents how a [`load_balancer`][super] distributes /// packets across endpoints. -#[derive(Debug, Deserialize, Serialize, Eq, PartialEq)] +#[derive(Debug, Deserialize, Serialize, Eq, PartialEq, JsonSchema)] pub enum Policy { /// Send packets to endpoints in turns. #[serde(rename = "ROUND_ROBIN")] diff --git a/src/filters/local_rate_limit.rs b/src/filters/local_rate_limit.rs index 1b59a5bdd1..055a0ffbc1 100644 --- a/src/filters/local_rate_limit.rs +++ b/src/filters/local_rate_limit.rs @@ -178,6 +178,10 @@ impl FilterFactory for LocalRateLimitFactory { NAME } + fn config_schema(&self) -> schemars::schema::RootSchema { + schemars::schema_for!(Config) + } + fn create_filter(&self, args: CreateFilterArgs) -> Result { let (config_json, config) = self .require_config(args.config)? @@ -199,7 +203,7 @@ impl FilterFactory for LocalRateLimitFactory { } /// Config represents a [self]'s configuration. -#[derive(Serialize, Deserialize, Debug, PartialEq)] +#[derive(Serialize, Deserialize, Debug, PartialEq, schemars::JsonSchema)] pub struct Config { /// The maximum number of packets allowed to be forwarded by the rate /// limiter in a given duration. diff --git a/src/filters/matches.rs b/src/filters/matches.rs index f579060f02..d07c7830e1 100644 --- a/src/filters/matches.rs +++ b/src/filters/matches.rs @@ -163,6 +163,10 @@ impl FilterFactory for MatchesFactory { NAME } + fn config_schema(&self) -> schemars::schema::RootSchema { + schemars::schema_for!(Config) + } + fn create_filter(&self, args: CreateFilterArgs) -> Result { let (config_json, config) = self .require_config(args.config)? @@ -177,7 +181,7 @@ impl FilterFactory for MatchesFactory { } /// Configuration for the [`factory`]. -#[derive(Debug, serde::Deserialize, serde::Serialize, PartialEq)] +#[derive(Debug, serde::Deserialize, serde::Serialize, PartialEq, schemars::JsonSchema)] #[serde(deny_unknown_fields)] pub struct Config { /// Configuration for [`Filter::read`]. @@ -231,7 +235,7 @@ impl TryFrom for DirectionalConfig { } /// Configuration for a specific direction. -#[derive(Debug, serde::Deserialize, serde::Serialize, PartialEq)] +#[derive(Debug, serde::Deserialize, serde::Serialize, PartialEq, schemars::JsonSchema)] pub struct DirectionalConfig { /// The key for the metadata to compare against. #[serde(rename = "metadataKey")] @@ -245,7 +249,7 @@ pub struct DirectionalConfig { /// A specific match branch. The filter is run when `value` matches the value /// defined in `metadata_key`. -#[derive(Debug, serde::Deserialize, serde::Serialize, PartialEq)] +#[derive(Debug, serde::Deserialize, serde::Serialize, PartialEq, schemars::JsonSchema)] pub struct Branch { /// The value to compare against the dynamic metadata. pub value: crate::metadata::Value, @@ -273,7 +277,7 @@ impl TryFrom for Branch { } /// The behaviour when the none of branches match. -#[derive(Debug, PartialEq)] +#[derive(Debug, PartialEq, schemars::JsonSchema)] pub enum Fallthrough { /// The packet will be passed onto the next filter. Pass, diff --git a/src/filters/set.rs b/src/filters/set.rs index 75fd683157..76b733366f 100644 --- a/src/filters/set.rs +++ b/src/filters/set.rs @@ -71,6 +71,19 @@ impl FilterSet { pub fn with(filters: impl IntoIterator) -> Self { Self::from_iter(filters) } + + /// Returns a [`DynFilterFactory`] if one matches `id`, otherwise returns + /// `None`. + pub fn get(&self, id: &str) -> Option<&DynFilterFactory> { + self.0.get(id) + } + + /// Returns a by reference iterator over the set of filters. + pub fn iter(&self) -> Iter { + Iter { + inner: self.0.iter(), + } + } } impl> From for FilterSet { @@ -114,3 +127,16 @@ impl Iterator for IntoIter { self.inner.next().map(|(_, v)| v) } } + +/// Iterator over a set of [`DynFilterFactory`]s. +pub struct Iter<'r> { + inner: std::collections::hash_map::Iter<'r, &'static str, DynFilterFactory>, +} + +impl<'r> Iterator for Iter<'r> { + type Item = &'r DynFilterFactory; + + fn next(&mut self) -> Option { + self.inner.next().map(|(_, v)| v) + } +} diff --git a/src/filters/token_router.rs b/src/filters/token_router.rs index 74b2e7aca1..4073dd86f1 100644 --- a/src/filters/token_router.rs +++ b/src/filters/token_router.rs @@ -72,6 +72,10 @@ impl FilterFactory for TokenRouterFactory { NAME } + fn config_schema(&self) -> schemars::schema::RootSchema { + schemars::schema_for!(Config) + } + fn create_filter(&self, args: CreateFilterArgs) -> Result { let (config_json, config) = args .config @@ -142,7 +146,7 @@ impl Filter for TokenRouter { } } -#[derive(Serialize, Deserialize, Debug, PartialEq)] +#[derive(Serialize, Deserialize, Debug, PartialEq, schemars::JsonSchema)] #[serde(default)] pub struct Config { /// the key to use when retrieving the token from the Filter's dynamic metadata diff --git a/src/lib.rs b/src/lib.rs index df02462b57..11e1dd20d9 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -37,7 +37,7 @@ pub type Result = std::result::Result; pub use self::{ config::Config, proxy::{Builder, PendingValidation, Server, Validated}, - runner::{run, run_with_config}, + runner::run, }; pub use quilkin_macros::include_proto; diff --git a/src/main.rs b/src/main.rs index 4a745d2d6f..35d28ef012 100644 --- a/src/main.rs +++ b/src/main.rs @@ -14,15 +14,55 @@ * limitations under the License. */ -use clap::{Arg, Command}; -use std::sync::Arc; +use std::path::PathBuf; + use tracing::info; const VERSION: &str = env!("CARGO_PKG_VERSION"); +#[derive(clap::Parser)] +struct Cli { + #[clap( + short, + long, + env = "QUILKIN_CONFIG", + default_value = "quilkin.yaml", + help = "The YAML configuration file." + )] + config: PathBuf, + #[clap( + short, + long, + env, + help = "Whether Quilkin will report any results to stdout/stderr." + )] + quiet: bool, + #[clap(subcommand)] + command: Commands, +} + +#[derive(clap::Subcommand)] +enum Commands { + Run, + GenerateConfigSchema { + #[clap( + short, + long, + default_value = ".", + help = "The directory to write configuration files." + )] + output_directory: PathBuf, + #[clap( + min_values = 1, + default_value = "all", + help = "A list of one or more filter IDs to generate or 'all' to generate all available filter schemas." + )] + filter_ids: Vec, + }, +} + #[tokio::main] async fn main() -> quilkin::Result<()> { - tracing_subscriber::fmt().json().with_target(false).init(); stable_eyre::install()?; let version: std::borrow::Cow<'static, str> = if cfg!(debug_assertions) { format!("{VERSION}+debug").into() @@ -30,34 +70,66 @@ async fn main() -> quilkin::Result<()> { VERSION.into() }; - let config_arg = Arg::new("config") - .short('c') - .long("config") - .value_name("CONFIG") - .help("The YAML configuration file") - .takes_value(true); - - let cli = Command::new(clap::crate_name!()) - .version(&*version) - .about(clap::crate_description!()) - .subcommand_required(true) - .arg_required_else_help(true) - .subcommand( - Command::new("run") - .about("Start Quilkin process.") - .arg(config_arg.clone()), - ) - .get_matches(); + let cli = ::parse(); + + if !cli.quiet { + tracing_subscriber::fmt().json().with_target(false).init(); + } info!(version = &*version, "Starting Quilkin"); - match cli.subcommand() { - Some(("run", matches)) => { - let config = quilkin::config::Config::find(matches.value_of("config")).map(Arc::new)?; + match cli.command { + Commands::Run => { + let config = std::fs::File::open(cli.config) + .or_else(|error| { + if cfg!(unix) { + std::fs::File::open("/etc/quilkin/quilkin.yaml") + } else { + Err(error) + } + }) + .map_err(eyre::Error::from) + .and_then(|file| quilkin::Config::from_reader(file).map_err(From::from))?; - quilkin::run_with_config(config, vec![]).await + quilkin::run(config, vec![]).await } - Some((cmd, _)) => panic!("Unimplemented subcommand: {}", cmd), - None => unreachable!(), + Commands::GenerateConfigSchema { + output_directory, + filter_ids, + } => { + let set = quilkin::filters::FilterSet::default(); + type SchemaIterator<'r> = + Box + 'r>; + + let schemas = (filter_ids.len() == 1 && filter_ids[0].to_lowercase() == "all") + .then(|| { + Box::new( + set.iter() + .map(|factory| (factory.name(), factory.config_schema())), + ) as SchemaIterator + }) + .unwrap_or_else(|| { + Box::new(filter_ids.iter().filter_map(|id| { + let item = set.get(id); + + if item.is_none() { + tracing::error!("{id} not found in filter set."); + } + + item.map(|item| (item.name(), item.config_schema())) + })) as SchemaIterator + }); + + for (id, schema) in schemas { + let mut path = output_directory.join(id); + path.set_extension("yaml"); + + tracing::info!("Writing {id} schema to {}", path.display()); + + std::fs::write(path, serde_yaml::to_string(&schema)?)?; + } + + Ok(()) + } } } diff --git a/src/metadata.rs b/src/metadata.rs index c261a0a083..d235f40735 100644 --- a/src/metadata.rs +++ b/src/metadata.rs @@ -23,7 +23,9 @@ pub type DynamicMetadata = HashMap, Value>; pub const KEY: &str = "quilkin.dev"; -#[derive(Clone, Debug, PartialOrd, serde::Serialize, serde::Deserialize, Eq, Ord)] +#[derive( + Clone, Debug, PartialOrd, serde::Serialize, serde::Deserialize, Eq, Ord, schemars::JsonSchema, +)] #[serde(untagged)] pub enum Value { Bool(bool), diff --git a/src/runner.rs b/src/runner.rs index 61428c7129..521dff7513 100644 --- a/src/runner.rs +++ b/src/runner.rs @@ -29,22 +29,16 @@ use crate::{ #[cfg(doc)] use crate::filters::FilterFactory; -/// Calls [`run`] with the [`Config`] found by [`Config::find`] and the -/// default [`FilterSet`]. -pub async fn run(filter_factories: impl IntoIterator) -> Result<()> { - run_with_config(Config::find(None).map(Arc::new)?, filter_factories).await -} - /// Start and run a proxy. Any passed in [`FilterFactory`]s are included /// alongside the default filter factories. -pub async fn run_with_config( - config: Arc, +pub async fn run( + config: Config, filter_factories: impl IntoIterator, ) -> Result<()> { let span = span!(Level::INFO, "source::run"); let _enter = span.enter(); - let server = Builder::from(config) + let server = Builder::from(Arc::new(config)) .with_filter_registry(FilterRegistry::new(FilterSet::default_with( filter_factories.into_iter(), ))) diff --git a/src/test_utils.rs b/src/test_utils.rs index a85333908a..aae62b108a 100644 --- a/src/test_utils.rs +++ b/src/test_utils.rs @@ -34,6 +34,10 @@ impl FilterFactory for TestFilterFactory { "TestFilter" } + fn config_schema(&self) -> schemars::schema::RootSchema { + schemars::schema_for_value!(serde_json::Value::Null) + } + fn create_filter(&self, _: CreateFilterArgs) -> Result { Ok(Self::create_empty_filter()) } diff --git a/src/xds/listener.rs b/src/xds/listener.rs index 0c3ed2015b..08945986e5 100644 --- a/src/xds/listener.rs +++ b/src/xds/listener.rs @@ -206,7 +206,7 @@ mod tests { // A simple filter that will be used in the following tests. // It appends a string to each payload. const APPEND_TYPE_URL: &str = "filter.append"; - #[derive(Clone, PartialEq, Serialize, Deserialize)] + #[derive(Clone, PartialEq, Serialize, Deserialize, schemars::JsonSchema)] pub struct Append { pub value: Option, } @@ -247,6 +247,10 @@ mod tests { APPEND_TYPE_URL } + fn config_schema(&self) -> schemars::schema::RootSchema { + schemars::schema_for!(Append) + } + fn create_filter(&self, args: CreateFilterArgs) -> Result { let (config_json, filter) = args .config