refactor: Consolidate function pipeline configuration

closes edgexfoundry#204 Signed-off-by: lenny <[email protected]>
lenny-goodell · Mar 2, 2021 · b714e75 · b714e75
1 parent ca2daa4
commit b714e75
Show file tree

Hide file tree

Showing 5 changed files with 144 additions and 210 deletions.
diff --git a/go.mod b/go.mod
@@ -3,3 +3,7 @@ module github.com/edgexfoundry/app-service-configurable
 go 1.15
 
 require github.com/edgexfoundry/app-functions-sdk-go/v2 v2.0.0-dev.23
+
+replace (
+	github.com/edgexfoundry/app-functions-sdk-go/v2 => ../app-functions-sdk-go
+)
diff --git a/res/functional-tests/configuration.toml b/res/functional-tests/configuration.toml
@@ -18,24 +18,16 @@ LogLevel = "INFO"
       FilterOut = "false"
     [Writable.Pipeline.Functions.AddTags]
       [Writable.Pipeline.Functions.AddTags.Parameters]
-      tags=""
-    [Writable.Pipeline.Functions.TransformToXML]
-      [Writable.Pipeline.Functions.TransformToXML.Parameters]
-      NoOp = "" # Must have at least one parameter for function to be populated in Consul
-    [Writable.Pipeline.Functions.TransformToJSON]
-      [Writable.Pipeline.Functions.TransformToJSON.Parameters]
-      NoOp = "" # Must have at least one parameter for function to be populated in Consul
-    [Writable.Pipeline.Functions.CompressWithGZIP]
-      [Writable.Pipeline.Functions.CompressWithGZIP.Parameters]
-      NoOp = "" # Must have at least one parameter for function to be populated in Consul
-    [Writable.Pipeline.Functions.CompressWithZLIB]
-      [Writable.Pipeline.Functions.CompressWithZLIB.Parameters]
-      NoOp = "" # Must have at least one parameter for function to be populated in Consul
-    [Writable.Pipeline.Functions.MarkAsPushed]
-      [Writable.Pipeline.Functions.MarkAsPushed.Parameters]
-      NoOp = "" # Must have at least one parameter for function to be populated in Consul
-    [Writable.Pipeline.Functions.EncryptWithAES]
-      [Writable.Pipeline.Functions.EncryptWithAES.Parameters]
+      Tags=""
+    [Writable.Pipeline.Functions.Transform]
+      [Writable.Pipeline.Functions.Transform.Parameters]
+      Type = "xml"
+    [Writable.Pipeline.Functions.Compress]
+      [Writable.Pipeline.Functions.Compress.Parameters]
+      Algorithm = "gzip"
+    [Writable.Pipeline.Functions.Encrypt]
+      [Writable.Pipeline.Functions.Encrypt.Parameters]
+      Algorithm = "aes"
       Key = "aquqweoruqwpeoruqwpoeruqwpoierupqoweiurpoqwiuerpqowieurqpowieurpoqiweuroipwqure"
       InitVector = "123456789012345678901234567890"
       SecretPath = ""
@@ -47,28 +39,23 @@ LogLevel = "INFO"
       [Writable.Pipeline.Functions.PushToCore.Parameters]
       DeviceName = ""
       ReadingName = ""
-    [Writable.Pipeline.Functions.HTTPPost]
-      [Writable.Pipeline.Functions.HTTPPost.Parameters]
-      url = "http://"
-      mimeType = "" #OPTIONAL - default application/json
-    [Writable.Pipeline.Functions.HTTPPostJSON]
-      [Writable.Pipeline.Functions.HTTPPostJSON.Parameters]
-      url = "http://"
-    [Writable.Pipeline.Functions.HTTPPostXML]
-      [Writable.Pipeline.Functions.HTTPPostXML.Parameters]
-      url = "http://"
-    [Writable.Pipeline.Functions.MQTTSecretSend]
-      [Writable.Pipeline.Functions.MQTTSecretSend.Parameters]
-      brokeraddress = "tcps://localhost:8883"
-      topic = "mytopic"
-      secretpath = "mqtt"
-      clientid = "myclientid"
-      qos="0"
-      autoreconnect="false"
-      retain="false"
-      skipverify = "false"
-      authmode = "none"
-      persistOnError = "false"
+    [Writable.Pipeline.Functions.HTTPExport]
+      [Writable.Pipeline.Functions.HTTPExport.Parameters]
+      Method = "post"
+      Url = "http://"
+      MimeType = "" #OPTIONAL - default application/json
+    [Writable.Pipeline.Functions.MQTTExport]
+      [Writable.Pipeline.Functions.MQTTExport.Parameters]
+      BrokerAddress = "tcps://localhost:8883"
+      Topic = "mytopic"
+      SecretPath = "mqtt"
+      ClientId = "myclientid"
+      QOS="0"
+      AutoReconnect="false"
+      Retain="false"
+      SkipVerify = "false"
+      AuthMode = "none"
+      PersistOnError = "false"
 
 [Service]
 BootTimeout = "30s"

diff --git a/res/http-export/configuration.toml b/res/http-export/configuration.toml
@@ -7,63 +7,50 @@
   MaxRetryCount = 10
 
   [Writable.Pipeline]
-    UseTargetTypeOfByteArray = false
-    ExecutionOrder = "FilterByDeviceName, TransformToJSON, HTTPPostJSON"
-      [Writable.Pipeline.Functions.TransformToXML]
-        [Writable.Pipeline.Functions.TransformToXML.Parameters]
-        NoOp = "" # Must have at least one parameter for function to be populated in Consul
-      [Writable.Pipeline.Functions.TransformToJSON]
-        [Writable.Pipeline.Functions.TransformToJSON.Parameters]
-        NoOp = "" # Must have at least one parameter for function to be populated in Consul
-      [Writable.Pipeline.Functions.CompressWithGZIP]
-        [Writable.Pipeline.Functions.CompressWithGZIP.Parameters]
-        NoOp = "" # Must have at least one parameter for function to be populated in Consul
-      [Writable.Pipeline.Functions.CompressWithZLIB]
-        [Writable.Pipeline.Functions.CompressWithZLIB.Parameters]
-        NoOp = "" # Must have at least one parameter for function to be populated in Consul
-      [Writable.Pipeline.Functions.FilterByProfileName]
-        [Writable.Pipeline.Functions.FilterByProfileName.Parameters]
-        ProfileNames = ""
-        FilterOut = "false"
-      [Writable.Pipeline.Functions.FilterByDeviceName]
-        [Writable.Pipeline.Functions.FilterByDeviceName.Parameters]
-        DeviceNames = ""
-        FilterOut = "false"
-      [Writable.Pipeline.Functions.FilterByResourceName]
-        [Writable.Pipeline.Functions.FilterByResourceName.Parameters]
-        ResourceNames = ""
-        FilterOut = "false"
-      [Writable.Pipeline.Functions.HTTPPostJSON]
-        [Writable.Pipeline.Functions.HTTPPostJSON.Parameters]
-        url = "https://f095cfcd-a3f6-4885-85ea-7157c6afd17e.mock.pstmn.io" # Postman mock service
-        persistOnError = "false"
-        headername = "" # Name of the header key to add to the HTTP header
-        secretname = "" # Name of the secret for the header value in the SecretStore
-        secretpath = "" # Path to the secret for the header value in the SecretStore
-      [Writable.Pipeline.Functions.HTTPPutJSON]
-        [Writable.Pipeline.Functions.HTTPPutJSON.Parameters]
-        url = "https://f095cfcd-a3f6-4885-85ea-7157c6afd17e.mock.pstmn.io" # Postman mock service
-        persistOnError = "false"
-        headername = "" # Name of the header key to add to the HTTP header
-        secretname = "" # Name of the secret for the header value in the SecretStore
-        secretpath = "" # Path to the secret for the header value in the SecretStore
-      [Writable.Pipeline.Functions.HTTPPostXML]
-        [Writable.Pipeline.Functions.HTTPPostXML.Parameters]
-        url = "https://f095cfcd-a3f6-4885-85ea-7157c6afd17e.mock.pstmn.io" # Postman mock service
-        persistOnError = "false"
-        headername = "" # Name of the header key to add to the HTTP header
-        secretname = "" # Name of the secret for the header value in the SecretStore
-        secretpath = "" # Path to the secret for the header value in the SecretStore
-      [Writable.Pipeline.Functions.HTTPPutXML]
-        [Writable.Pipeline.Functions.HTTPPutXML.Parameters]
-        url = "https://f095cfcd-a3f6-4885-85ea-7157c6afd17e.mock.pstmn.io" # Postman mock service
-        persistOnError = "false"
-        headername = "" # Name of the header key to add to the HTTP header
-        secretname = "" # Name of the secret for the header value in the SecretStore
-        secretpath = "" # Path to the secret for the header value in the SecretStore
-      [Writable.Pipeline.Functions.AddTags]
-        [Writable.Pipeline.Functions.AddTags.Parameters]
-        tags=""
+  UseTargetTypeOfByteArray = false
+  ExecutionOrder = "FilterByDeviceName, Transform, HTTPExport"
+    [Writable.Pipeline.Functions.FilterByProfileName]
+      [Writable.Pipeline.Functions.FilterByProfileName.Parameters]
+      ProfileNames = ""
+      FilterOut = "false"
+    [Writable.Pipeline.Functions.FilterByDeviceName]
+      [Writable.Pipeline.Functions.FilterByDeviceName.Parameters]
+      DeviceNames = ""
+      FilterOut = "false"
+    [Writable.Pipeline.Functions.FilterByResourceName]
+      [Writable.Pipeline.Functions.FilterByResourceName.Parameters]
+      ResourceNames = ""
+      FilterOut = "false"
+    [Writable.Pipeline.Functions.Transform]
+      [Writable.Pipeline.Functions.Transform.Parameters]
+      Type = "json"
+    [Writable.Pipeline.Functions.Compress]
+      [Writable.Pipeline.Functions.Compress.Parameters]
+      Algorithm = "gzip"
+    [Writable.Pipeline.Functions.Encrypt]
+      [Writable.Pipeline.Functions.Encrypt.Parameters]
+      Algorithm = "aes" # currently can be only be 'aes'
+      Key = "aquqweoruqwpeoruqwpoeruqwpoierupqoweiurpoqwiuerpqowieurqpowieurpoqiweuroipwqure"
+      InitVector = "123456789012345678901234567890"
+      SecretPath = ""
+      SecretName = ""
+    [Writable.Pipeline.Functions.Batch]
+      [Writable.Pipeline.Functions.Batch.Parameters]
+      Mode = "bytimecount" # can be 'bycount', 'bytime' or 'bytimecount'
+      BatchThreshold = "30"
+      TimeInterval = "60s"
+    [Writable.Pipeline.Functions.AddTags]
+      [Writable.Pipeline.Functions.AddTags.Parameters]
+      Tags=""
+    [Writable.Pipeline.Functions.HTTPExport]
+      [Writable.Pipeline.Functions.HTTPExport.Parameters]
+      Method = "post"
+      MimeType = "application/json"
+      Url = "https://f095cfcd-a3f6-4885-85ea-7157c6afd17e.mock.pstmn.io" # Postman mock service
+      PersistOnError = "false"
+      HeaderName = "" # Name of the header key to add to the HTTP header
+      SecretName = "" # Name of the secret for the header value in the SecretStore
+      SecretPath = "" # Path to the secret for the header value in the SecretStore
 
 # InsecureSecrets are required for Store and Forward DB access and for authenticated HTTP exports when not using
 # security services, i.e. Vault

diff --git a/res/mqtt-export/configuration.toml b/res/mqtt-export/configuration.toml
@@ -7,19 +7,7 @@ LogLevel = "INFO"
   MaxRetryCount = 10
 
   [Writable.Pipeline]
-  ExecutionOrder = "TransformToJSON, MQTTSecretSend"
-    [Writable.Pipeline.Functions.TransformToXML]
-      [Writable.Pipeline.Functions.TransformToXML.Parameters]
-      NoOp = "" # Must have at least one parameter for function to be populated in Consul
-    [Writable.Pipeline.Functions.TransformToJSON]
-      [Writable.Pipeline.Functions.TransformToJSON.Parameters]
-      NoOp = "" # Must have at least one parameter for function to be populated in Consul
-    [Writable.Pipeline.Functions.CompressWithGZIP]
-      [Writable.Pipeline.Functions.CompressWithGZIP.Parameters]
-      NoOp = "" # Must have at least one parameter for function to be populated in Consul
-    [Writable.Pipeline.Functions.CompressWithZLIB]
-      [Writable.Pipeline.Functions.CompressWithZLIB.Parameters]
-      NoOp = "" # Must have at least one parameter for function to be populated in Consul
+  ExecutionOrder = "Transform, MQTTExport"
     [Writable.Pipeline.Functions.FilterByProfileName]
       [Writable.Pipeline.Functions.FilterByProfileName.Parameters]
       ProfileNames = ""
@@ -32,21 +20,39 @@ LogLevel = "INFO"
       [Writable.Pipeline.Functions.FilterByResourceName.Parameters]
       ResourceNames = ""
       FilterOut = "false"
-    [Writable.Pipeline.Functions.MQTTSecretSend]
-      [Writable.Pipeline.Functions.MQTTSecretSend.Parameters]
-      brokeraddress = "tcp://broker.hivemq.com:1883" # Hive Public Test Broker
-      topic = "edgex-export"
-      secretpath = "mqtt"
-      clientid = "MQTT-Export"
-      qos="0"
-      autoreconnect="true"
-      retain="false"
-      skipverify = "false"
-      authmode = "none"
-      persistOnError = "false"
+    [Writable.Pipeline.Functions.Transform]
+      [Writable.Pipeline.Functions.Transform.Parameters]
+      Type = "json"
+    [Writable.Pipeline.Functions.Compress]
+      [Writable.Pipeline.Functions.Compress.Parameters]
+      Algorithm = "gzip"
+    [Writable.Pipeline.Functions.Encrypt]
+      [Writable.Pipeline.Functions.Encrypt.Parameters]
+      Algorithm = "aes" # currently can be only be 'aes'
+      Key = "aquqweoruqwpeoruqwpoeruqwpoierupqoweiurpoqwiuerpqowieurqpowieurpoqiweuroipwqure"
+      InitVector = "123456789012345678901234567890"
+      SecretPath = ""
+      SecretName = ""
+    [Writable.Pipeline.Functions.Batch]
+      [Writable.Pipeline.Functions.Batch.Parameters]
+      Mode = "bytimecount" # can be 'bycount', 'bytime' or 'bytimecount'
+      BatchThreshold = "30"
+      TimeInterval = "60s"
     [Writable.Pipeline.Functions.AddTags]
       [Writable.Pipeline.Functions.AddTags.Parameters]
-      tags=""
+      Tags=""
+    [Writable.Pipeline.Functions.MQTTExport]
+      [Writable.Pipeline.Functions.MQTTExport.Parameters]
+      BrokerAddress = "tcp://broker.hivemq.com:1883" # Hive Public Test Broker
+      Topic = "edgex-export"
+      SecretPath = "mqtt"
+      ClientId = "MQTT-Export"
+      QOS="0"
+      AutoReconnect="true"
+      Retain="false"
+      SkipVerify = "false"
+      AuthMode = "none"
+      PersistOnError = "false"
 
 # InsecureSecrets are required for Store and Forward DB access and for authenticated MQTT exports with
 # new MQTTSecretSend function when not using security services, i.e. Vault