Add env support to expressions

.github/PULL_REQUEST_TEMPLATE.md

@@ -3,4 +3,5 @@
 ## **Please check that the PR fulfills these requirements**
 - [ ] Tests for the changes have been added (for bug fixes / features)
 - [ ] Docs have been added / updated (for bug fixes / features)
+- [ ] Add a changelog entry (for non-trivial bug fixes / features)
 - [ ] CI passes

CHANGELOG.md

@@ -4,6 +4,12 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [Unreleased]
+### Added
+- Support for reading from environment in expressions
+- Ability to point to labels with fields
+
+
 ## [0.9.0] - 2020-07-07
 
-Initial Open Source Release
+Initial Open Source Release

docs/types/expression.md

@@ -6,10 +6,25 @@ being processed.
 
 For reference documentation of the expression language, see [here](https://github.com/antonmedv/expr/blob/master/docs/Language-Definition.md).
 
-In most cases, the record of the entry being processed can be accessed with the `$` variable in the expression. See the examples below for syntax.
+Available to the expressions are a few special variables:
+- `$record` contains the entry's record
+- `$labels` contains the entry's labels
+- `$tags` contains the entry's tags
+- `$timestamp` contains the entry's timestamp
+- `env()` is a function that allows you to read environment variables
 
 ## Examples
 
+### Add a label from an environment variable
+
+```yaml
+- id: add_stack_label
+  type: metadata
+  output: my_receiver
+  labels:
+    stack: 'EXPR(env("STACK"))'
+```
+
 ### Map severity values to standard values
 
 ```yaml
@@ -19,5 +34,5 @@ In most cases, the record of the entry being processed can be accessed with the
   ops:
     - add:
         field: severity
-        value_expr: '$.raw_severity in ["critical", "super_critical"] ? "error" : $.raw_severity'
+        value_expr: '$record.raw_severity in ["critical", "super_critical"] ? "error" : $record.raw_severity'
 ```

docs/types/field.md

@@ -1,18 +1,13 @@
 ## Fields
 
-_Fields_ are the primary way to tell carbon which fields of a log's record to use for the operations of its plugins.
+_Fields_ are the primary way to tell carbon which fields of an entry to use for the operations of its plugins.
 Most often, these will be things like fields to parse for a parser plugin, or the field to write a new value to.
 
-Fields are `.`-delimited strings which allow you to selected into nested records in the field. The root level is specified by `$` such as in `$.key1`, but since all fields are expected to be relative to root, the `$` is implied and be omitted. For example, in the record below, `nested_key` can be equivalently selected with `$.key2.nested_key` or `key2.nested_key`.
+Fields are `.`-delimited strings which allow you to select labels or records on the entry. Fields can currently be used to select labels or values on a record. To select a label, prefix your field with `$label.` such as with `$label.my_label`. For values on the record, use the prefix `$record.` such as `$record.my_value`.
 
-```json
-{
-  "key1": "value1",
-  "key2": {
-    "nested_key": "nested_value"
-  }
-}
-```
+Record fields can be nested arbitrarily deeply, such as `$record.my_value.my_nested_value`.
+
+If a field does not start with either `$label` or `$record`, `$record` is assumed. For example, `my_value` is equivalent to `$record.my_value`.
 
 ## Examples
 
@@ -27,20 +22,27 @@ Config:
     - add:
         field: "key3"
         value: "value3"
-    - remove: "key2.nested_key1"
+    - remove: "$record.key2.nested_key1"
+    - add:
+        field: "$labels.my_label"
+        value: "my_label_value"
 ```
 
 <table>
-<tr><td> Input record </td> <td> Output record </td></tr>
+<tr><td> Input entry </td> <td> Output entry </td></tr>
 <tr>
 <td>
 
 ```json
 {
-  "key1": "value1",
-  "key2": {
-    "nested_key1": "nested_value1",
-    "nested_key2": "nested_value2"
+  "timestamp": "",
+  "labels": {},
+  "record": {
+    "key1": "value1",
+    "key2": {
+      "nested_key1": "nested_value1",
+      "nested_key2": "nested_value2"
+    }
   }
 }
 ```
@@ -50,11 +52,17 @@ Config:
 
 ```json
 {
-  "key1": "value1",
-  "key2": {
-    "nested_key2": "nested_value2"
+  "timestamp": "",
+  "labels": {
+    "my_label": "my_label_value"
   },
-  "key3": "value3"
+  "record": {
+    "key1": "value1",
+    "key2": {
+      "nested_key2": "nested_value2"
+    },
+    "key3": "value3"
+  }
 }
 ```
 

entry/entry.go

@@ -28,19 +28,19 @@ func (entry *Entry) AddLabel(key, value string) {
 	entry.Labels[key] = value
 }
 
-func (entry *Entry) Get(field Field) (interface{}, bool) {
+func (entry *Entry) Get(field FieldInterface) (interface{}, bool) {
 	return field.Get(entry)
 }
 
-func (entry *Entry) Set(field Field, val interface{}) {
-	field.Set(entry, val, true)
+func (entry *Entry) Set(field FieldInterface, val interface{}) error {
+	return field.Set(entry, val)
 }
 
-func (entry *Entry) Delete(field Field) (interface{}, bool) {
+func (entry *Entry) Delete(field FieldInterface) (interface{}, bool) {
 	return field.Delete(entry)
 }
 
-func (entry *Entry) Read(field Field, dest interface{}) error {
+func (entry *Entry) Read(field FieldInterface, dest interface{}) error {
 	val, ok := entry.Get(field)
 	if !ok {
 		return fmt.Errorf("field does not exist")

entry/entry_test.go

@@ -35,84 +35,84 @@ func TestRead(t *testing.T) {
 
 	t.Run("field not exist error", func(t *testing.T) {
 		var s string
-		err := testEntry.Read(Field{[]string{"nonexistant_field"}}, &s)
+		err := testEntry.Read(NewRecordField("nonexistant_field"), &s)
 		require.Error(t, err)
 	})
 
 	t.Run("unsupported type error", func(t *testing.T) {
 		var s **string
-		err := testEntry.Read(Field{[]string{"string_field"}}, &s)
+		err := testEntry.Read(NewRecordField("string_field"), &s)
 		require.Error(t, err)
 	})
 
 	t.Run("string", func(t *testing.T) {
 		var s string
-		err := testEntry.Read(Field{[]string{"string_field"}}, &s)
+		err := testEntry.Read(NewRecordField("string_field"), &s)
 		require.NoError(t, err)
 		require.Equal(t, "string_val", s)
 	})
 
 	t.Run("string error", func(t *testing.T) {
 		var s string
-		err := testEntry.Read(Field{[]string{"map_string_interface_field"}}, &s)
+		err := testEntry.Read(NewRecordField("map_string_interface_field"), &s)
 		require.Error(t, err)
 	})
 
 	t.Run("map[string]interface{}", func(t *testing.T) {
 		var m map[string]interface{}
-		err := testEntry.Read(Field{[]string{"map_string_interface_field"}}, &m)
+		err := testEntry.Read(NewRecordField("map_string_interface_field"), &m)
 		require.NoError(t, err)
 		require.Equal(t, map[string]interface{}{"nested": "interface_val"}, m)
 	})
 
 	t.Run("map[string]interface{} error", func(t *testing.T) {
 		var m map[string]interface{}
-		err := testEntry.Read(Field{[]string{"string_field"}}, &m)
+		err := testEntry.Read(NewRecordField("string_field"), &m)
 		require.Error(t, err)
 	})
 
 	t.Run("map[string]string from map[string]interface{}", func(t *testing.T) {
 		var m map[string]string
-		err := testEntry.Read(Field{[]string{"map_string_interface_field"}}, &m)
+		err := testEntry.Read(NewRecordField("map_string_interface_field"), &m)
 		require.NoError(t, err)
 		require.Equal(t, map[string]string{"nested": "interface_val"}, m)
 	})
 
 	t.Run("map[string]string from map[string]interface{} err", func(t *testing.T) {
 		var m map[string]string
-		err := testEntry.Read(Field{[]string{"map_string_interface_nonstring_field"}}, &m)
+		err := testEntry.Read(NewRecordField("map_string_interface_nonstring_field"), &m)
 		require.Error(t, err)
 	})
 
 	t.Run("map[string]string from map[interface{}]interface{}", func(t *testing.T) {
 		var m map[string]string
-		err := testEntry.Read(Field{[]string{"map_interface_interface_field"}}, &m)
+		err := testEntry.Read(NewRecordField("map_interface_interface_field"), &m)
 		require.NoError(t, err)
 		require.Equal(t, map[string]string{"nested": "interface_val"}, m)
 	})
 
 	t.Run("map[string]string from map[interface{}]interface{} nonstring key error", func(t *testing.T) {
 		var m map[string]string
-		err := testEntry.Read(Field{[]string{"map_interface_interface_nonstring_key_field"}}, &m)
+		err := testEntry.Read(NewRecordField("map_interface_interface_nonstring_key_field"), &m)
 		require.Error(t, err)
 	})
 
 	t.Run("map[string]string from map[interface{}]interface{} nonstring value error", func(t *testing.T) {
 		var m map[string]string
-		err := testEntry.Read(Field{[]string{"map_interface_interface_nonstring_value_field"}}, &m)
+		err := testEntry.Read(NewRecordField("map_interface_interface_nonstring_value_field"), &m)
 		require.Error(t, err)
 	})
 
 	t.Run("interface{} from any", func(t *testing.T) {
 		var i interface{}
-		err := testEntry.Read(Field{[]string{"map_interface_interface_field"}}, &i)
+		err := testEntry.Read(NewRecordField("map_interface_interface_field"), &i)
 		require.NoError(t, err)
 		require.Equal(t, map[interface{}]interface{}{"nested": "interface_val"}, i)
 	})
 
 	t.Run("string from []byte", func(t *testing.T) {
 		var i string
-		err := testEntry.Read(NewField("byte_field"), &i)
+		err := testEntry.Read(NewRecordField("byte_field"), &i)
 		require.NoError(t, err)
 		require.Equal(t, "test", i)
 	})
@@ -139,3 +139,55 @@ func TestCopy(t *testing.T) {
 	require.Equal(t, map[string]string{"label": "value"}, copy.Labels)
 	require.Equal(t, "test", copy.Record)
 }
+
+func TestFieldFromString(t *testing.T) {
+	cases := []struct {
+		name          string
+		input         string
+		output        Field
+		expectedError bool
+	}{
+		{
+			"SimpleRecord",
+			"test",
+			Field{RecordField{[]string{"test"}}},
+			false,
+		},
+		{
+			"PrefixedRecord",
+			"$.test",
+			Field{RecordField{[]string{"test"}}},
+			false,
+		},
+		{
+			"FullPrefixedRecord",
+			"$record.test",
+			Field{RecordField{[]string{"test"}}},
+			false,
+		},
+		{
+			"SimpleLabel",
+			"$labels.test",
+			Field{LabelField{"test"}},
+			false,
+		},
+		{
+			"LabelsTooManyFields",
+			"$labels.test.bar",
+			Field{},
+			true,
+		},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			f, err := fieldFromString(tc.input)
+			if tc.expectedError {
+				require.Error(t, err)
+				return
+			}
+
+			require.Equal(t, tc.output, f)
+		})
+	}
+}