Add auto-generation of JSON schema.

This commit adds support for Quilkin to generate JSON schema for any of the included filters, and adds some glue to make it so that these schemas are automatically included in the mdBook documentation.
googleforgames · Feb 7, 2022 · 369e25f · 369e25f
1 parent 2fdc8ac
commit 369e25f
Show file tree

Hide file tree

Showing 41 changed files with 259 additions and 239 deletions.
diff --git a/Cargo.toml b/Cargo.toml
@@ -44,7 +44,7 @@ quilkin-macros = { version = "0.3.0-dev", path = "./macros" }
 base64 = "0.13.0"
 base64-serde = "0.6.1"
 bytes = { version = "1.1.0", features = ["serde"] }
-clap = { version = "3", features = ["cargo"] }
+clap = { version = "3", features = ["cargo", "derive", "env"] }
 dashmap = "4.0.2"
 either = "1.6.1"
 hyper = "0.14.15"
@@ -70,6 +70,7 @@ eyre = "0.6.5"
 stable-eyre = "0.2.2"
 ipnetwork = "0.18.0"
 futures = "0.3.17"
+schemars = "0.8.8"
 
 [target.'cfg(target_os = "linux")'.dependencies]
 sys-info = "0.9.0"

diff --git a/build/build-image/Dockerfile b/build/build-image/Dockerfile
@@ -26,7 +26,7 @@ ENV RUSTUP_HOME=/usr/local/rustup \
 # Install packages
 RUN set -eux && \
     apt-get update && \
-    apt-get install -y wget zip build-essential libssl-dev pkg-config python3-pip && \
+    apt-get install -y jq wget zip build-essential libssl-dev pkg-config python3-pip && \
     pip3 install live-server
 
 # Install Go

diff --git a/docs/book.toml b/docs/book.toml
@@ -4,4 +4,10 @@ multilingual = false
 src = "src"
 title = "Quilkin Book"
 
+[preprocessor.links]
+after = ["quilkin"]
+
+[preprocessor.quilkin]
+command = "./preprocessor.sh"
+
 [output.html]
diff --git a/docs/preprocessor.sh b/docs/preprocessor.sh
@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+cargo run -q --manifest-path ../Cargo.toml -- -q generate-config-schema -o ../target
+
+echo $(jq -M -c .[1] <&0)
diff --git a/docs/src/filters/capture_bytes.md b/docs/src/filters/capture_bytes.md
@@ -36,31 +36,7 @@ static:
 ### Configuration Options ([Rust Doc](../../api/quilkin/filters/capture_bytes/struct.Config.html))
 
 ```yaml
-properties:
-  strategy:
-    type: string
-    description: |
-      The selected strategy for capturing the series of bytes from the incoming packet.
-       - SUFFIX: Retrieve bytes from the end of the packet.
-       - PREFIX: Retrieve bytes from the beginnning of the packet.
-    default: "SUFFIX"
-    enum: ['PREFIX', 'SUFFIX']
-  metadataKey:
-    type: string
-    default: quilkin.dev/captured_bytes
-    description: | 
-      The key under which the captured bytes are stored in the Filter invocation values.
-  size:
-    type: integer
-    description: |
-      The number of bytes in the packet to capture using the applied strategy.
-  remove:
-    type: boolean
-    default: false
-    description: |
-      Whether or not to remove the captured bytes from the packet before passing it along to the next filter in the
-      chain.
-  required: ['size']
+{{#include ../../../target/quilkin.extensions.filters.capture_bytes.v1alpha1.yaml}}
 ```
 
 ### Metrics

diff --git a/docs/src/filters/compress.md b/docs/src/filters/compress.md
@@ -1,6 +1,6 @@
 # Compress
 
-The `Compress` filter's job is to provide a variety of compression implementations for compression 
+The `Compress` filter's job is to provide a variety of compression implementations for compression
 and subsequent decompression of UDP data when sent between systems, such as a game client and game server.
 
 #### Filter name
@@ -27,9 +27,9 @@ static:
 # quilkin::Builder::from(std::sync::Arc::new(config)).validate().unwrap();
 ```
 
-The above example shows a proxy that could be used with a typical game client, where the original client data is 
-sent to the local listening port and then compressed when heading up to a dedicated game server, and then 
-decompressed when traffic is returned from the dedicated game server before being handed back to game client. 
+The above example shows a proxy that could be used with a typical game client, where the original client data is
+sent to the local listening port and then compressed when heading up to a dedicated game server, and then
+decompressed when traffic is returned from the dedicated game server before being handed back to game client.
 
 > It is worth noting that since the Compress filter modifies the *entire packet*, it is worth paying special
   attention to the order it is placed in your [Filter configuration](../filters.md). Most of the time it will likely be
@@ -38,38 +38,14 @@ decompressed when traffic is returned from the dedicated game server before bein
 ### Configuration Options ([Rust Doc](../../api/quilkin/filters/compress/struct.Config.html))
 
 ```yaml
-properties:
-  on_read:
-    '$ref': '#/definitions/action'
-    description: |
-      Whether to compress, decompress or do nothing when reading packets from the local listening port
-  on_write:
-    '$ref': '#/definitions/action'
-    description: |
-      Whether to compress, decompress or do nothing when writing packets to the local listening port
-  mode:
-    type: string
-    description: |
-      The compression implementation to use on the incoming and outgoing packets. See "Compression Modes" for details.
-    enum:
-      - SNAPPY
-    default: SNAPPY
-
-definitions:
-  action:
-    type: string
-    enum:
-      - DO_NOTHING
-      - COMPRESS
-      - DECOMPRESS
-    default: DO_NOTHING
+{{#include ../../../target/quilkin.extensions.filters.compress.v1alpha1.yaml}}
 ```
 
 #### Compression Modes
 
 ##### Snappy
 
-> Snappy is a compression/decompression library. It does not aim for maximum compression, or compatibility with any 
+> Snappy is a compression/decompression library. It does not aim for maximum compression, or compatibility with any
 > other compression library; instead, it aims for very high speeds and reasonable compression.
 
 Currently, this filter only provides the [Snappy](https://github.com/google/snappy/) compression format via the
@@ -78,7 +54,7 @@ provided in the future.
 
 ### Metrics
 * `quilkin_filter_Compress_packets_dropped_total`
-  Total number of packets dropped as they could not be processed.  
+  Total number of packets dropped as they could not be processed.
     * Labels:
       * `action`: The action that could not be completed successfully, thereby causing the packet to be dropped.
         * `Compress`: Compressing the packet with the configured `mode` was attempted.

diff --git a/docs/src/filters/concatenate_bytes.md b/docs/src/filters/concatenate_bytes.md
@@ -30,23 +30,7 @@ static:
 ### Configuration Options ([Rust Doc](../../api/quilkin/filters/concatenate_bytes/struct.Config.html))
 
 ```yaml
-properties:
-  on_read:
-    type: string
-    description: |
-      Either append or prepend the `bytes` data to each packet filtered on read of the listening port.
-    default: DO_NOTHING
-    enum: ['DO_NOTHING', 'APPEND', 'PREPEND']
-  on_write:
-    type: string
-    description: |
-      Either append or prepend the `bytes` data to each packet filtered on write of the listening port.
-    default: DO_NOTHING
-    enum: ['DO_NOTHING', 'APPEND', 'PREPEND']    
-  bytes:
-    type: string
-    description: |
-      Base64 encoded string of the byte array to add to each packet as it is filtered.
+{{#include ../../../target/quilkin.extensions.filters.concatenate_bytes.v1alpha1.yaml}}
 ```
 
 ### Metrics

diff --git a/docs/src/filters/debug.md b/docs/src/filters/debug.md
@@ -29,11 +29,7 @@ static:
 ### Configuration Options ([Rust Doc](../../api/quilkin/filters/debug/struct.Config.html))
 
 ```yaml
-properties:
-  id:
-    type: string
-    description: |
-      An identifier that will be included with each log message.
+{{#include ../../../target/quilkin.extensions.filters.debug.v1alpha1.yaml}}
 ```
 
 

diff --git a/docs/src/filters/firewall.md b/docs/src/filters/firewall.md
@@ -38,41 +38,7 @@ static:
 ### Configuration Options ([Rust Doc](../../api/quilkin/filters/firewall/struct.Config.html))
 
 ```yaml
-properties:
-  on_read:
-    '$ref': '#/definitions/rules'
-    description: Rules to match against when reading packets to the local listening port.
-  on_write:
-    type: array
-    '$ref': '#/definitions/rules'
-    description: Rules to match against when writing packets to the local listening port.
-
-definitions:
-  rules:
-    type: array
-    description: Rules to match against when writing packets to the local listening port.
-    items:
-      type: object
-      properties:
-        action:
-          type: string
-          description: |
-            Whether or not a matching Rule should Allow or Deny access
-            - DENY: If the rule matches, block the traffic.
-            - ALLOW: If the rule matches, allow the traffic through.
-          enum: ['ALLOW', 'DENY']
-        source:
-          type: string
-          description: A CIDR network range, either in a v4 or v6 format.
-        ports:
-          type: array
-          description: Array of singular ports or port ranges to match against.
-          items:
-            type: string
-            description: |
-              Either in the format of "10" for a singular port or "10-100" for a port range where 
-              min is inclusive, and max is exclusive.
-      required: ['action', 'source', 'ports']
+{{#include ../../../target/quilkin.extensions.filters.firewall.v1alpha1.yaml}}
 ```
 
 #### Rule Evaluation

diff --git a/docs/src/filters/load_balancer.md b/docs/src/filters/load_balancer.md
@@ -33,16 +33,7 @@ In the example above, packets will be distributed by selecting endpoints in turn
 ### Configuration Options ([Rust Doc](../../api/quilkin/filters/load_balancer/struct.Config.html))
 
 ```yaml
-properties:
-  policy:
-    type: string
-    description: |
-      The load balancing policy with which to distribute packets among endpoints.
-    enum:
-      - ROUND_ROBIN # Send packets by selecting endpoints in turn.
-      - RANDOM      # Send packets by randomly selecting endpoints.
-      - HASH        # Send packets by hashing the source IP and port.
-    default: ROUND_ROBIN
+{{#include ../../../target/quilkin.extensions.filters.load_balancer.v1alpha1.yaml}}
 ```
 
 ### Metrics

diff --git a/docs/src/filters/local_rate_limit.md b/docs/src/filters/local_rate_limit.md
@@ -42,21 +42,7 @@ To configure a rate limiter, we specify the maximum rate at which the proxy is a
 ### Configuration Options ([Rust Doc](../../api/quilkin/filters/local_rate_limit/struct.Config.html))
 
 ```yaml
-properties:
-  max_packets:
-    type: integer
-    description: |
-      The maximum number of packets allowed to be forwarded over the given duration.
-    minimum: 0
-
-  period:
-    type: string
-    description: |
-      The duration in seconds overwhich `max_packets` applies.
-    default: 1 # 1 second
-    minimum: 1
-
-required: [ 'max_packets' ]
+{{#include ../../../target/quilkin.extensions.filters.local_rate_limit.v1alpha1.yaml}}
 ```
 
 

diff --git a/docs/src/filters/token_router.md b/docs/src/filters/token_router.md
@@ -43,12 +43,7 @@ View the [CaptureBytes](./capture_bytes.md) filter documentation for more detail
 ### Configuration Options ([Rust Doc](../../api/quilkin/filters/token_router/struct.Config.html))
 
 ```yaml
-properties:
-  metadataKey:
-    type: string
-    default: quilkin.dev/captured_bytes
-    description: | 
-      The key under which the token is stored in the Filter dynamic metadata.
+{{#include ../../../target/quilkin.extensions.filters.token_router.v1alpha1.yaml}}
 ```
 
 ### Metrics

diff --git a/docs/src/filters/writing_custom_filters.md b/docs/src/filters/writing_custom_filters.md
@@ -111,6 +111,11 @@ impl FilterFactory for GreetFilterFactory {
     fn name(&self) -> &'static str {
         NAME
     }
+
+    fn config_schema(&self) -> schemars::schema::RootSchema {
+        schemars::schema_for!(serde_json::Value)
+    }
+
     fn create_filter(&self, _: CreateFilterArgs) -> Result<FilterInstance, Error> {
         let filter: Box<dyn Filter> = Box::new(Greet);
         Ok(FilterInstance::new(serde_json::Value::Null, filter))
@@ -235,6 +240,11 @@ impl FilterFactory for GreetFilterFactory {
     fn name(&self) -> &'static str {
         NAME
     }
+
+    fn config_schema(&self) -> schemars::schema::RootSchema {
+        schemars::schema_for!(serde_json::Value)
+    }
+
     fn create_filter(&self, args: CreateFilterArgs) -> Result<FilterInstance, Error> {
         let config = match args.config.unwrap() {
           ConfigType::Static(config) => {

diff --git a/docs/src/quickstart-agones-xonotic.md b/docs/src/quickstart-agones-xonotic.md
@@ -112,7 +112,7 @@ replace the `${GAMESERVER_IP}` and `${GAMESERVER_PORT}` values in your copy of `
 Run this configuration locally as:
 
 ```shell
-quilkin run -c ./client-compress.yaml`
+quilkin -c ./client-compress.yaml run`
 ```
 
 Now we can connect to the local client proxy on "127.0.0.1:7000" via the "Multiplayer > Address" field in the

diff --git a/docs/src/quickstart-netcat.md b/docs/src/quickstart-netcat.md
@@ -38,13 +38,13 @@ a single endpoint of 127.0.0.1, port 8000.
 Let's start Quilkin with the above configuration:
 
 ```shell
-quilkin run --config proxy.yaml
+quilkin --config proxy.yaml run
 ```
 
 You should see an output like the following:
 
 ```shell
-$ quilkin run --config proxy.yaml
+$ quilkin --config proxy.yaml run
 {"msg":"Starting Quilkin","level":"INFO","ts":"2021-04-25T19:27:22.535174615-07:00","source":"run","version":"0.1.0-dev"}
 {"msg":"Starting","level":"INFO","ts":"2021-04-25T19:27:22.535315827-07:00","source":"server::Server","port":7000}
 {"msg":"Starting admin endpoint","level":"INFO","ts":"2021-04-25T19:27:22.535550572-07:00","source":"proxy::Admin","address":"[::]:9091"}

diff --git a/docs/src/using.md b/docs/src/using.md
@@ -15,7 +15,7 @@ The release binary can be downloaded from the
 
 Quilkin needs to be run with an accompanying [configuration file](./proxy-configuration.md), like so:
 
-`quilkin run --config="configuration.yaml"`
+`quilkin --config="configuration.yaml run"`
 
 To view debug output, run the same command with the `quilkin-debug` binary.
 

diff --git a/examples/agones-xonotic/README.md b/examples/agones-xonotic/README.md
@@ -25,7 +25,7 @@ Instead of connecting Xonotic directly, take the IP and port from the Agones hos
 `${GAMESERVER_IP}` and `${GAMESERVER_PORT}` values in a local copy of `client-compress.yaml`. Run this configuration 
 locally as:
 
-`quilkin run -c ./client-compress.yaml`
+`quilkin -c ./client-compress.yaml run`
 
 From there connect to the local client proxy on "127.0.0.1:7000" via the "Multiplayer > Address" field in the 
 Xonotic client, and Quilkin will take care of compressing the data for you without having to change either the 

diff --git a/examples/quilkin-filter-example/Cargo.toml b/examples/quilkin-filter-example/Cargo.toml
@@ -34,6 +34,7 @@ prost-types = "0.9.0"
 serde = "1.0"
 serde_yaml = "0.8"
 bytes = "1.1.0"
+schemars = "0.8.8"
 
 [build-dependencies]
 prost-build = "0.9.0"
diff --git a/examples/quilkin-filter-example/src/main.rs b/examples/quilkin-filter-example/src/main.rs
@@ -24,7 +24,7 @@ use serde::{Deserialize, Serialize};
 use std::convert::TryFrom;
 
 // ANCHOR: serde_config
-#[derive(Serialize, Deserialize, Debug)]
+#[derive(Serialize, Deserialize, Debug, schemars::JsonSchema)]
 struct Config {
     greeting: String,
 }
@@ -71,6 +71,11 @@ impl FilterFactory for GreetFilterFactory {
     fn name(&self) -> &'static str {
         NAME
     }
+
+    fn config_schema(&self) -> schemars::schema::RootSchema {
+        schemars::schema_for!(Config)
+    }
+
     fn create_filter(&self, args: CreateFilterArgs) -> Result<FilterInstance, Error> {
         let (config_json, config) = self
             .require_config(args.config)?
@@ -84,8 +89,11 @@ impl FilterFactory for GreetFilterFactory {
 // ANCHOR: run
 #[tokio::main]
 async fn main() {
-    quilkin::run(vec![self::factory()].into_iter())
-        .await
-        .unwrap();
+    quilkin::run(
+        quilkin::Config::builder().build(),
+        vec![self::factory()].into_iter(),
+    )
+    .await
+    .unwrap();
 }
 // ANCHOR_END: run