Add auto-generation of JSON schema.

This commit adds support for Quilkin to generate JSON schema for any of the included filters, and adds some glue to make it so that these schemas are automatically included in the mdBook documentation.
googleforgames · Feb 22, 2022 · c6fc7de · c6fc7de
1 parent 44903d1
commit c6fc7de
Show file tree

Hide file tree

Showing 48 changed files with 309 additions and 260 deletions.
diff --git a/Cargo.toml b/Cargo.toml
@@ -44,34 +44,35 @@ quilkin-macros = { version = "0.3.0-dev", path = "./macros" }
 base64 = "0.13.0"
 base64-serde = "0.6.1"
 bytes = { version = "1.1.0", features = ["serde"] }
-clap = { version = "3.1", features = ["cargo"] }
+clap = { version = "3.1", features = ["cargo", "derive", "env"] }
 dashmap = "4.0.2"
 either = "1.6.1"
+eyre = "0.6.5"
+futures = "0.3.17"
 hyper = "0.14.15"
+ipnetwork = "0.18.0"
 num_cpus = "1.13.0"
 parking_lot = "0.11.2"
 prometheus = { version = "0.13.0", default-features = false }
 prost = "=0.9.0"
 prost-types = "=0.9.0"
 rand = "0.8.4"
+regex = "1.5.4"
+schemars = { version = "0.8.8", features = ["bytes"] }
 serde = { version = "1.0.130", features = ["derive", "rc"] }
 serde_json = "1.0.68"
 serde_regex = "1.1.0"
 serde_yaml = "0.8.21"
 snap = "1.0.5"
+stable-eyre = "0.2.2"
+thiserror = "1.0.30"
 tokio = { version = "1.16.1", features = ["rt-multi-thread", "signal", "test-util", "parking_lot"] }
 tokio-stream = "0.1.8"
 tonic = "0.6.1"
 tracing = {version = "0.1"}
 tracing-subscriber = { version = "0.3", features = ["json"] }
-uuid = { version = "0.8.2", default-features = false, features = ["v4"] }
-thiserror = "1.0.30"
 tryhard = "0.4.0"
-eyre = "0.6.5"
-stable-eyre = "0.2.2"
-ipnetwork = "0.18.0"
-futures = "0.3.17"
-regex = "1.5.4"
+uuid = { version = "0.8.2", default-features = false, features = ["v4"] }
 
 [target.'cfg(target_os = "linux")'.dependencies]
 sys-info = "0.9.0"

diff --git a/build/build-image/Dockerfile b/build/build-image/Dockerfile
@@ -26,7 +26,7 @@ ENV RUSTUP_HOME=/usr/local/rustup \
 # Install packages
 RUN set -eux && \
     apt-get update && \
-    apt-get install -y wget zip build-essential libssl-dev pkg-config python3-pip && \
+    apt-get install -y jq wget zip build-essential libssl-dev pkg-config python3-pip && \
     pip3 install live-server
 
 # Install Go

diff --git a/docs/book.toml b/docs/book.toml
@@ -4,4 +4,10 @@ multilingual = false
 src = "src"
 title = "Quilkin Book"
 
+[preprocessor.links]
+after = ["quilkin"]
+
+[preprocessor.quilkin]
+command = "./preprocessor.sh"
+
 [output.html]
diff --git a/docs/preprocessor.sh b/docs/preprocessor.sh
@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -euo pipefail
+
+cargo run -q --manifest-path ../Cargo.toml -- -q generate-config-schema -o ../target
+
+echo $(jq -M -c .[1] <&0)
diff --git a/docs/src/SUMMARY.md b/docs/src/SUMMARY.md
@@ -11,13 +11,14 @@
     - [Proxy Configuration](./proxy-configuration.md)
 - [Filters](./filters.md)
     - [Capture](./filters/capture.md)
-    - [Concatenate Bytes](./filters/concatenate_bytes.md)
     - [Compress](./filters/compress.md)
+    - [Concatenate Bytes](./filters/concatenate_bytes.md)
     - [Debug](./filters/debug.md)
+    - [Firewall](./filters/firewall.md)
     - [Load Balancer](./filters/load_balancer.md)
     - [Local Rate Limit](./filters/local_rate_limit.md)
+    - [Matches](./filters/matches.md)
     - [Token Router](./filters/token_router.md)
-    - [Firewall](./filters/firewall.md)
     - [Writing Custom Filters](./filters/writing_custom_filters.md)
 - [Integrations](./integrations.md)
 - [Administration](./admin.md)

diff --git a/docs/src/filters/capture.md b/docs/src/filters/capture.md
@@ -5,7 +5,7 @@ The `CaptureBytes` filter's job is to find a series of bytes within a packet, an
 down the chain.
 
 This is often used as a way of retrieving authentication tokens from a packet, and used in combination with
-[ConcatenateBytes](./concatenate_bytes.md) and 
+[ConcatenateBytes](./concatenate_bytes.md) and
 [TokenRouter](token_router.md) filter to provide common packet routing utilities.
 
 ### Capture strategies
@@ -52,36 +52,12 @@ static:
 ### Configuration Options ([Rust Doc](../../api/quilkin/filters/capture/struct.Config.html))
 
 ```yaml
-properties:
-  strategy:
-    type: object
-    description: |
-      The selected strategy for capturing the series of bytes from the incoming packet.
-       - SUFFIX: Retrieve bytes from the end of the packet.
-       - PREFIX: Retrieve bytes from the beginnning of the packet.
-    default: "SUFFIX"
-    enum: ['PREFIX', 'SUFFIX']
-  metadataKey:
-    type: string
-    default: quilkin.dev/captured
-    description: | 
-      The key under which the captured bytes are stored in the Filter invocation values.
-  size:
-    type: integer
-    description: |
-      The number of bytes in the packet to capture using the applied strategy.
-  remove:
-    type: boolean
-    default: false
-    description: |
-      Whether or not to remove the captured bytes from the packet before passing it along to the next filter in the
-      chain.
-  required: ['size']
+{{#include ../../../target/quilkin.extensions.filters.capture.v1alpha1.yaml}}
 ```
 
 ### Metrics
 
-* `quilkin_filter_Capture_packets_dropped`  
+* `quilkin_filter_Capture_packets_dropped`
   A counter of the total number of packets that have been dropped due to their length being less than the configured
   `size`.
 

diff --git a/docs/src/filters/compress.md b/docs/src/filters/compress.md
@@ -1,6 +1,6 @@
 # Compress
 
-The `Compress` filter's job is to provide a variety of compression implementations for compression 
+The `Compress` filter's job is to provide a variety of compression implementations for compression
 and subsequent decompression of UDP data when sent between systems, such as a game client and game server.
 
 #### Filter name
@@ -27,9 +27,9 @@ static:
 # quilkin::Builder::from(std::sync::Arc::new(config)).validate().unwrap();
 ```
 
-The above example shows a proxy that could be used with a typical game client, where the original client data is 
-sent to the local listening port and then compressed when heading up to a dedicated game server, and then 
-decompressed when traffic is returned from the dedicated game server before being handed back to game client. 
+The above example shows a proxy that could be used with a typical game client, where the original client data is
+sent to the local listening port and then compressed when heading up to a dedicated game server, and then
+decompressed when traffic is returned from the dedicated game server before being handed back to game client.
 
 > It is worth noting that since the Compress filter modifies the *entire packet*, it is worth paying special
   attention to the order it is placed in your [Filter configuration](../filters.md). Most of the time it will likely be
@@ -38,38 +38,14 @@ decompressed when traffic is returned from the dedicated game server before bein
 ### Configuration Options ([Rust Doc](../../api/quilkin/filters/compress/struct.Config.html))
 
 ```yaml
-properties:
-  on_read:
-    '$ref': '#/definitions/action'
-    description: |
-      Whether to compress, decompress or do nothing when reading packets from the local listening port
-  on_write:
-    '$ref': '#/definitions/action'
-    description: |
-      Whether to compress, decompress or do nothing when writing packets to the local listening port
-  mode:
-    type: string
-    description: |
-      The compression implementation to use on the incoming and outgoing packets. See "Compression Modes" for details.
-    enum:
-      - SNAPPY
-    default: SNAPPY
-
-definitions:
-  action:
-    type: string
-    enum:
-      - DO_NOTHING
-      - COMPRESS
-      - DECOMPRESS
-    default: DO_NOTHING
+{{#include ../../../target/quilkin.extensions.filters.compress.v1alpha1.yaml}}
 ```
 
 #### Compression Modes
 
 ##### Snappy
 
-> Snappy is a compression/decompression library. It does not aim for maximum compression, or compatibility with any 
+> Snappy is a compression/decompression library. It does not aim for maximum compression, or compatibility with any
 > other compression library; instead, it aims for very high speeds and reasonable compression.
 
 Currently, this filter only provides the [Snappy](https://github.com/google/snappy/) compression format via the
@@ -78,7 +54,7 @@ provided in the future.
 
 ### Metrics
 * `quilkin_filter_Compress_packets_dropped_total`
-  Total number of packets dropped as they could not be processed.  
+  Total number of packets dropped as they could not be processed.
     * Labels:
       * `action`: The action that could not be completed successfully, thereby causing the packet to be dropped.
         * `Compress`: Compressing the packet with the configured `mode` was attempted.

diff --git a/docs/src/filters/concatenate_bytes.md b/docs/src/filters/concatenate_bytes.md
@@ -30,23 +30,7 @@ static:
 ### Configuration Options ([Rust Doc](../../api/quilkin/filters/concatenate_bytes/struct.Config.html))
 
 ```yaml
-properties:
-  on_read:
-    type: string
-    description: |
-      Either append or prepend the `bytes` data to each packet filtered on read of the listening port.
-    default: DO_NOTHING
-    enum: ['DO_NOTHING', 'APPEND', 'PREPEND']
-  on_write:
-    type: string
-    description: |
-      Either append or prepend the `bytes` data to each packet filtered on write of the listening port.
-    default: DO_NOTHING
-    enum: ['DO_NOTHING', 'APPEND', 'PREPEND']    
-  bytes:
-    type: string
-    description: |
-      Base64 encoded string of the byte array to add to each packet as it is filtered.
+{{#include ../../../target/quilkin.extensions.filters.concatenate_bytes.v1alpha1.yaml}}
 ```
 
 ### Metrics

diff --git a/docs/src/filters/debug.md b/docs/src/filters/debug.md
@@ -29,11 +29,7 @@ static:
 ### Configuration Options ([Rust Doc](../../api/quilkin/filters/debug/struct.Config.html))
 
 ```yaml
-properties:
-  id:
-    type: string
-    description: |
-      An identifier that will be included with each log message.
+{{#include ../../../target/quilkin.extensions.filters.debug.v1alpha1.yaml}}
 ```
 
 

diff --git a/docs/src/filters/firewall.md b/docs/src/filters/firewall.md
@@ -38,41 +38,7 @@ static:
 ### Configuration Options ([Rust Doc](../../api/quilkin/filters/firewall/struct.Config.html))
 
 ```yaml
-properties:
-  on_read:
-    '$ref': '#/definitions/rules'
-    description: Rules to match against when reading packets to the local listening port.
-  on_write:
-    type: array
-    '$ref': '#/definitions/rules'
-    description: Rules to match against when writing packets to the local listening port.
-
-definitions:
-  rules:
-    type: array
-    description: Rules to match against when writing packets to the local listening port.
-    items:
-      type: object
-      properties:
-        action:
-          type: string
-          description: |
-            Whether or not a matching Rule should Allow or Deny access
-            - DENY: If the rule matches, block the traffic.
-            - ALLOW: If the rule matches, allow the traffic through.
-          enum: ['ALLOW', 'DENY']
-        source:
-          type: string
-          description: A CIDR network range, either in a v4 or v6 format.
-        ports:
-          type: array
-          description: Array of singular ports or port ranges to match against.
-          items:
-            type: string
-            description: |
-              Either in the format of "10" for a singular port or "10-100" for a port range where 
-              min is inclusive, and max is exclusive.
-      required: ['action', 'source', 'ports']
+{{#include ../../../target/quilkin.extensions.filters.firewall.v1alpha1.yaml}}
 ```
 
 #### Rule Evaluation

diff --git a/docs/src/filters/load_balancer.md b/docs/src/filters/load_balancer.md
@@ -33,16 +33,7 @@ In the example above, packets will be distributed by selecting endpoints in turn
 ### Configuration Options ([Rust Doc](../../api/quilkin/filters/load_balancer/struct.Config.html))
 
 ```yaml
-properties:
-  policy:
-    type: string
-    description: |
-      The load balancing policy with which to distribute packets among endpoints.
-    enum:
-      - ROUND_ROBIN # Send packets by selecting endpoints in turn.
-      - RANDOM      # Send packets by randomly selecting endpoints.
-      - HASH        # Send packets by hashing the source IP and port.
-    default: ROUND_ROBIN
+{{#include ../../../target/quilkin.extensions.filters.load_balancer.v1alpha1.yaml}}
 ```
 
 ### Metrics

diff --git a/docs/src/filters/local_rate_limit.md b/docs/src/filters/local_rate_limit.md
@@ -42,21 +42,7 @@ To configure a rate limiter, we specify the maximum rate at which the proxy is a
 ### Configuration Options ([Rust Doc](../../api/quilkin/filters/local_rate_limit/struct.Config.html))
 
 ```yaml
-properties:
-  max_packets:
-    type: integer
-    description: |
-      The maximum number of packets allowed to be forwarded over the given duration.
-    minimum: 0
-
-  period:
-    type: string
-    description: |
-      The duration in seconds overwhich `max_packets` applies.
-    default: 1 # 1 second
-    minimum: 1
-
-required: [ 'max_packets' ]
+{{#include ../../../target/quilkin.extensions.filters.local_rate_limit.v1alpha1.yaml}}
 ```
 
 

diff --git a/docs/src/filters/matches.md b/docs/src/filters/matches.md
@@ -40,4 +40,10 @@ static:
 # quilkin::Builder::from(std::sync::Arc::new(config)).validate().unwrap();
 ```
 
+### Configuration Options ([Rust Doc](../../api/quilkin/filters/matches/struct.Config.html))
+
+```yaml
+{{#include ../../../target/quilkin.extensions.filters.matches.v1alpha1.yaml}}
+```
+
 View the [Matches](../../api/quilkin/filters/matches/struct.Config.html) filter documentation for more details.
diff --git a/docs/src/filters/token_router.md b/docs/src/filters/token_router.md
@@ -43,12 +43,7 @@ View the [CaptureBytes](./capture.md) filter documentation for more details.
 ### Configuration Options ([Rust Doc](../../api/quilkin/filters/token_router/struct.Config.html))
 
 ```yaml
-properties:
-  metadataKey:
-    type: string
-    default: quilkin.dev/captured
-    description: | 
-      The key under which the token is stored in the Filter dynamic metadata.
+{{#include ../../../target/quilkin.extensions.filters.token_router.v1alpha1.yaml}}
 ```
 
 ### Metrics

diff --git a/docs/src/filters/writing_custom_filters.md b/docs/src/filters/writing_custom_filters.md
@@ -111,6 +111,11 @@ impl FilterFactory for GreetFilterFactory {
     fn name(&self) -> &'static str {
         NAME
     }
+
+    fn config_schema(&self) -> schemars::schema::RootSchema {
+        schemars::schema_for!(serde_json::Value)
+    }
+
     fn create_filter(&self, _: CreateFilterArgs) -> Result<FilterInstance, Error> {
         let filter: Box<dyn Filter> = Box::new(Greet);
         Ok(FilterInstance::new(serde_json::Value::Null, filter))
@@ -235,6 +240,11 @@ impl FilterFactory for GreetFilterFactory {
     fn name(&self) -> &'static str {
         NAME
     }
+
+    fn config_schema(&self) -> schemars::schema::RootSchema {
+        schemars::schema_for!(serde_json::Value)
+    }
+
     fn create_filter(&self, args: CreateFilterArgs) -> Result<FilterInstance, Error> {
         let config = match args.config.unwrap() {
           ConfigType::Static(config) => {