enrich-pubsub: add reference.conf and provide minimal config example (c…

…lose #505)
snowplow · Feb 1, 2022 · a89c2a0 · a89c2a0
1 parent 2a38964
commit a89c2a0
Show file tree

Hide file tree

Showing 16 changed files with 356 additions and 359 deletions.
diff --git a/config/config.pubsub.hocon.sample → config/config.file.extended.hocon b/config/config.pubsub.hocon.sample → config/config.file.extended.hocon
@@ -1,52 +1,35 @@
+# For testing purposes only
+# Can be used with either enrich-pubsub or enrich-kinesis
+
 {
-  # Collector input
   "input": {
-    "type": "PubSub"
-    "subscription": "projects/test-project/subscriptions/inputSub"
+    "type": "FileSystem"
 
-    # Local FS supported for testing purposes
-    # "type": "FileSystem"
-    # "dir": "/var/collector"
+    # Directory containing collector payloads that are Thrift encoded
+    "dir": "/var/collector-payloads"
   }
 
   "output": {
     # Enriched events output
     "good": {
-      "type": "PubSub"
-      "topic": "projects/test-project/topics/good-topic"
-
-      # Enriched event fields to add as PubSub message attributes.
-      "attributes": [ "app_id" ]
-
-      # Local FS supported for testing purposes
-      # "type": "FileSystem"
-      # "file": "/var/enriched"
-      # "maxBytes": 1000000
+      "type": "FileSystem"
+      "file": "/var/enriched"
+      "maxBytes": 1000000
     }
 
     # Pii events output
     "pii": {
-      "type": "PubSub"
-      "topic": "projects/test-project/topics/pii-topic"
-
-      # Enriched event fields to add as PubSub message attributes.
-      # "attributes": [ "app_id" ]
-
-      # Local FS supported for testing purposes
-      # "type": "FileSystem"
-      # "file": "/var/pii"
-      # "maxBytes": 1000000
+      "type": "FileSystem"
+      "file": "/var/pii"
+      "maxBytes": 1000000
     }
 
     # Bad rows output
     "bad": {
-      "type": "PubSub"
-      "topic": "projects/test-project/topics/bad-topic"
-
       # Local FS supported for testing purposes
-      # "type": "FileSystem"
-      # "file": "/var/bad"
-      # "maxBytes": 1000000
+      "type": "FileSystem"
+      "file": "/var/bad"
+      "maxBytes": 1000000
     }
   }
 
@@ -58,7 +41,7 @@
     "sink": 3
   }
 
-  # Optional, period after which enrich assets should be checked for updates
+  # Optional. period after which enrich assets should be checked for updates
   # no assets will be updated if the key is absent
   "assetsUpdatePeriod": "7 days"
 
@@ -72,9 +55,8 @@
     # Optional, configure how metrics are reported
     "metrics": {
 
-      # Send metrics to a StatsD server on localhost
+      # Optional. Send metrics to a StatsD server
       "statsd": {
-
         "hostname": "localhost"
         "port": 8125
 
@@ -90,16 +72,17 @@
         # "prefix": "snowplow.enrich."
       }
 
-      # Log to stdout using Slf4j
+      # Optional. Log to stdout using Slf4j
       "stdout": {
         "period": "10 seconds"
 
         # Optional, override the default metric prefix
         # "prefix": "snowplow.enrich."
       }
 
+      # Optional. Cloudwatch metrics for KPL
+      "cloudwatch": true
     }
-
   }
 
   # Optional, configure telemetry
@@ -141,5 +124,5 @@
 
     # Version of the terraform module that deployed the app
     moduleVersion = 1.0.0
-   }
+  }
 }
diff --git a/config/config.hocon.sample b/config/config.hocon.sample
diff --git a/config/config.kinesis.reference.hocon → config/config.kinesis.extended.hocon b/config/config.kinesis.reference.hocon → config/config.kinesis.extended.hocon
@@ -352,30 +352,41 @@
   # Optional, configure telemetry
   # All the fields are optional
   "telemetry": {
+
     # Set to true to disable telemetry
     "disable": false
+
     # Interval for the heartbeat event
     "interval": 15 minutes
+
     # HTTP method used to send the heartbeat event
-    "method": "POST"
+    "method": POST
+
     # URI of the collector receiving the heartbeat event
-    "collectorUri": "collector-g.snowplowanalytics.com"
+    "collectorUri": collector-g.snowplowanalytics.com
+
     # Port of the collector receiving the heartbeat event
     "collectorPort": 443
+
     # Whether to use https or not
     "secure": true
+
     # Identifier intended to tie events together across modules,
     # infrastructure and apps when used consistently
-    "userProvidedId": "my_pipeline"
+    "userProvidedId": my_pipeline
+
     # ID automatically generated upon running a modules deployment script
     # Intended to identify each independent module, and the infrastructure it controls
-    "autoGeneratedId": "hfy67e5ydhtrd"
+    "autoGeneratedId": hfy67e5ydhtrd
+
     # Unique identifier for the VM instance
     # Unique for each instance of the app running within a module
-    "instanceId": "665bhft5u6udjf"
+    instanceId = 665bhft5u6udjf
+
     # Name of the terraform module that deployed the app
-    "moduleName": "enrich-kinesis-ce"
+    moduleName = enrich-kinesis-ce
+
     # Version of the terraform module that deployed the app
-    "moduleVersion": "1.0.0"
-   }
+    moduleVersion = 1.0.0
+  }
 }