Skip to content

Commit

Permalink
feat(http_server source): add all headers to the namespace metadata (v…
Browse files Browse the repository at this point in the history 
…ectordotdev#18922)

* feat(http_server source): add all headers to the namespace metadata

* feat(http_server source): allow wildcard matching in headers

* style: whitespace typo

* rework header glob matching, add docs and tests

* examples, docs, tests, error on misconfiguration

* fmt & clippy cleanup

* Generate docs

Signed-off-by: Jesse Szwedko <[email protected]>

* docs grammar adjustment

* add some code docs

Ref: LOG-19103

---------

Signed-off-by: Jesse Szwedko <[email protected]>
Co-authored-by: Jesse Szwedko <[email protected]>
Co-authored-by: neuronull <[email protected]>
  • Loading branch information
@sonnens @jszwedko
@neuronull @darinspivey
3 people authored and darinspivey committed Jan 24, 2024
1 parent 2641295 commit 3772b19
Show file tree
Hide file tree
Showing 3 changed files with 119 additions and 15 deletions.
122 changes: 109 additions & 13 deletions src/sources/http_server.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,10 +88,16 @@ pub struct SimpleHttpConfig {

/// A list of HTTP headers to include in the log event.
///
/// Accepts the wildcard (`*`) character for headers matching a specified pattern.
///
/// Specifying "*" results in all headers included in the log event.
///
/// These override any values included in the JSON payload with conflicting names.
#[serde(default)]
#[configurable(metadata(docs::examples = "User-Agent"))]
#[configurable(metadata(docs::examples = "X-My-Custom-Header"))]
#[configurable(metadata(docs::examples = "X-*"))]
#[configurable(metadata(docs::examples = "*"))]
headers: Vec<String>,

/// A list of URL query parameters to include in the log event.
Expand Down Expand Up @@ -323,6 +329,21 @@ fn remove_duplicates(mut list: Vec<String>, list_name: &str) -> Vec<String> {
list
}

#[derive(Clone)]
enum HttpConfigParamKind {
Glob(glob::Pattern),
Exact(String),
}

fn build_param_matcher(list: &[String]) -> crate::Result<Vec<HttpConfigParamKind>> {
list.iter()
.map(|s| match s.contains('*') {
true => Ok(HttpConfigParamKind::Glob(glob::Pattern::new(s)?)),
false => Ok(HttpConfigParamKind::Exact(s.to_string())),
})
.collect::<crate::Result<Vec<HttpConfigParamKind>>>()
}

#[async_trait::async_trait]
#[typetag::serde(name = "http_server")]
impl SourceConfig for SimpleHttpConfig {
Expand All @@ -331,7 +352,7 @@ impl SourceConfig for SimpleHttpConfig {
let log_namespace = cx.log_namespace(self.log_namespace);

let source = SimpleHttpSource {
headers: remove_duplicates(self.headers.clone(), "headers"),
headers: build_param_matcher(&remove_duplicates(self.headers.clone(), "headers"))?,
query_parameters: remove_duplicates(self.query_parameters.clone(), "query_parameters"),
path_key: self.path_key.clone(),
decoder,
Expand Down Expand Up @@ -377,7 +398,7 @@ impl SourceConfig for SimpleHttpConfig {

#[derive(Clone)]
struct SimpleHttpSource {
headers: Vec<String>,
headers: Vec<HttpConfigParamKind>,
query_parameters: Vec<String>,
path_key: OptionalValuePath,
decoder: Decoder,
Expand Down Expand Up @@ -407,17 +428,48 @@ impl HttpSource for SimpleHttpSource {
request_path.to_owned(),
);

// add each header to each event
for header_name in &self.headers {
let value = headers_config.get(header_name).map(HeaderValue::as_bytes);

self.log_namespace.insert_source_metadata(
SimpleHttpConfig::NAME,
log,
Some(LegacyKey::InsertIfEmpty(path!(header_name))),
path!("headers", header_name),
Value::from(value.map(Bytes::copy_from_slice)),
);
for h in &self.headers {
match h {
// Add each non-wildcard containing header that was specified
// in the `headers` config option to the event if an exact match
// is found.
HttpConfigParamKind::Exact(header_name) => {
let value =
headers_config.get(header_name).map(HeaderValue::as_bytes);

self.log_namespace.insert_source_metadata(
SimpleHttpConfig::NAME,
log,
Some(LegacyKey::InsertIfEmpty(path!(header_name))),
path!("headers", header_name),
Value::from(value.map(Bytes::copy_from_slice)),
);
}
// Add all headers that match against wildcard pattens specified
// in the `headers` config option to the event.
HttpConfigParamKind::Glob(header_pattern) => {
for header_name in headers_config.keys() {
if header_pattern.matches_with(
header_name.as_str(),
glob::MatchOptions::default(),
) {
let value = headers_config
.get(header_name)
.map(HeaderValue::as_bytes);

self.log_namespace.insert_source_metadata(
SimpleHttpConfig::NAME,
log,
Some(LegacyKey::InsertIfEmpty(path!(
header_name.as_str()
))),
path!("headers", header_name.as_str()),
Value::from(value.map(Bytes::copy_from_slice)),
);
}
}
}
};
}

self.log_namespace.insert_standard_vector_source_metadata(
Expand Down Expand Up @@ -977,11 +1029,13 @@ mod tests {
let mut headers = HeaderMap::new();
headers.insert("User-Agent", "test_client".parse().unwrap());
headers.insert("Upgrade-Insecure-Requests", "false".parse().unwrap());
headers.insert("X-Test-Header", "true".parse().unwrap());

let (rx, addr) = source(
vec![
"User-Agent".to_string(),
"Upgrade-Insecure-Requests".to_string(),
"X-*".to_string(),
"AbsentHeader".to_string(),
],
vec![],
Expand Down Expand Up @@ -1012,11 +1066,53 @@ mod tests {
assert_eq!(log["key1"], "value1".into());
assert_eq!(log["\"User-Agent\""], "test_client".into());
assert_eq!(log["\"Upgrade-Insecure-Requests\""], "false".into());
assert_eq!(log["\"x-test-header\""], "true".into());
assert_eq!(log["AbsentHeader"], Value::Null);
assert_event_metadata(log).await;
}
}

#[tokio::test]
async fn http_headers_wildcard() {
let mut events = assert_source_compliance(&HTTP_PUSH_SOURCE_TAGS, async {
let mut headers = HeaderMap::new();
headers.insert("User-Agent", "test_client".parse().unwrap());
headers.insert("X-Case-Sensitive-Value", "CaseSensitive".parse().unwrap());

let (rx, addr) = source(
vec!["*".to_string()],
vec![],
"http_path",
"/",
"POST",
StatusCode::OK,
true,
EventStatus::Delivered,
true,
None,
Some(JsonDeserializerConfig::default().into()),
)
.await;

spawn_ok_collect_n(
send_with_headers(addr, "{\"key1\":\"value1\"}", headers),
rx,
1,
)
.await
})
.await;

{
let event = events.remove(0);
let log = event.as_log();
assert_eq!(log["key1"], "value1".into());
assert_eq!(log["\"user-agent\""], "test_client".into());
assert_eq!(log["\"x-case-sensitive-value\""], "CaseSensitive".into());
assert_event_metadata(log).await;
}
}

#[tokio::test]
async fn http_query() {
let mut events = assert_source_compliance(&HTTP_PUSH_SOURCE_TAGS, async {
Expand Down
6 changes: 5 additions & 1 deletion website/cue/reference/components/sources/base/http.cue
View file
Original file line number Diff line number Diff line change
Expand Up @@ -287,12 +287,16 @@ base: components: sources: http: configuration: {
description: """
A list of HTTP headers to include in the log event.
Accepts the wildcard (`*`) character for headers matching a specified pattern.
Specifying "*" results in all headers included in the log event.
These override any values included in the JSON payload with conflicting names.
"""
required: false
type: array: {
default: []
items: type: string: examples: ["User-Agent", "X-My-Custom-Header"]
items: type: string: examples: ["User-Agent", "X-My-Custom-Header", "X-*", "*"]
}
}
method: {
Expand Down
6 changes: 5 additions & 1 deletion website/cue/reference/components/sources/base/http_server.cue
View file
Original file line number Diff line number Diff line change
Expand Up @@ -287,12 +287,16 @@ base: components: sources: http_server: configuration: {
description: """
A list of HTTP headers to include in the log event.
Accepts the wildcard (`*`) character for headers matching a specified pattern.
Specifying "*" results in all headers included in the log event.
These override any values included in the JSON payload with conflicting names.
"""
required: false
type: array: {
default: []
items: type: string: examples: ["User-Agent", "X-My-Custom-Header"]
items: type: string: examples: ["User-Agent", "X-My-Custom-Header", "X-*", "*"]
}
}
method: {
Expand Down

0 comments on commit 3772b19

Please sign in to comment.