diff --git a/Makefile b/Makefile index 9c770f01f..7fb7d45dd 100644 --- a/Makefile +++ b/Makefile @@ -21,6 +21,9 @@ GO=CGO_ENABLED=1 GO111MODULE=on go build: make -C ./app-service-template build +tidy: + go mod tidy + test-template: make -C ./app-service-template test diff --git a/app-service-template/Makefile b/app-service-template/Makefile index 52c6da635..643494ed3 100644 --- a/app-service-template/Makefile +++ b/app-service-template/Makefile @@ -36,13 +36,12 @@ GOFLAGS=-ldflags "-X github.com/edgexfoundry/app-functions-sdk-go/v2/internal.SD #GIT_SHA=$(shell git rev-parse HEAD) GIT_SHA=no-sha -# `go mod tidy` is needed for the Depend a Bot PRs which don't update the go.sum in this folder, causing tests to fail -# in the pipeline since the tests call this make build. -# TODO: Remove `go mod tidy` when creating your new app services using this template -build: - go mod tidy +build: tidy $(GO) build $(GOFLAGS) -o $(MICROSERVICE) +tidy: + go mod tidy + # TODO: Change the registries (edgexfoundry, nexus3.edgexfoundry.org:10004) below as needed. # Leave them as is if service is to be upstreamed to EdgeX Foundry # NOTE: This is only used for local development. Jenkins CI does not use this make target diff --git a/app-service-template/functions/sample.go b/app-service-template/functions/sample.go index be59f2101..e660d92ae 100644 --- a/app-service-template/functions/sample.go +++ b/app-service-template/functions/sample.go @@ -17,7 +17,7 @@ package functions import ( - "errors" + "fmt" "strings" "github.com/edgexfoundry/app-functions-sdk-go/v2/pkg/interfaces" @@ -28,32 +28,35 @@ import ( // TODO: Create your custom type and function(s) and remove these samples +// NewSample ... // TODO: Add parameters that the function(s) will need each time one is executed func NewSample() Sample { return Sample{} } +// Sample ... type Sample struct { // TODO: Add properties that the function(s) will need each time one is executed } -// LogEventDetails is example of processing an Event and passing the original Event to to next function in the pipeline +// LogEventDetails is example of processing an Event and passing the original Event to next function in the pipeline // For more details on the Context API got here: https://docs.edgexfoundry.org/1.3/microservices/application/ContextAPI/ func (s *Sample) LogEventDetails(ctx interfaces.AppFunctionContext, data interface{}) (bool, interface{}) { lc := ctx.LoggingClient() - lc.Debug("LogEventDetails called") + lc.Debugf("LogEventDetails called in pipeline '%s'", ctx.PipelineId()) if data == nil { // Go here for details on Error Handle: https://docs.edgexfoundry.org/1.3/microservices/application/ErrorHandling/ - return false, errors.New("no Event Received") + return false, fmt.Errorf("function LogEventDetails in pipeline '%s': No Data Received", ctx.PipelineId()) } event, ok := data.(dtos.Event) if !ok { - return false, errors.New("type received is not an Event") + return false, fmt.Errorf("function LogEventDetails in pipeline '%s', type received is not an Event", ctx.PipelineId()) } - lc.Infof("Event received: ID=%s, Device=%s, and ReadingCount=%d", + lc.Infof("Event received in pipeline '%s': ID=%s, Device=%s, and ReadingCount=%d", + ctx.PipelineId(), event.Id, event.DeviceName, len(event.Readings)) @@ -61,16 +64,18 @@ func (s *Sample) LogEventDetails(ctx interfaces.AppFunctionContext, data interfa switch strings.ToLower(reading.ValueType) { case strings.ToLower(common.ValueTypeBinary): lc.Infof( - "Reading #%d received with ID=%s, Resource=%s, ValueType=%s, MediaType=%s and BinaryValue of size=`%d`", + "Reading #%d received in pipeline '%s' with ID=%s, Resource=%s, ValueType=%s, MediaType=%s and BinaryValue of size=`%d`", index+1, + ctx.PipelineId(), reading.Id, reading.ResourceName, reading.ValueType, reading.MediaType, len(reading.BinaryValue)) default: - lc.Infof("Reading #%d received with ID=%s, Resource=%s, ValueType=%s, Value=`%s`", + lc.Infof("Reading #%d received in pipeline '%s' with ID=%s, Resource=%s, ValueType=%s, Value=`%s`", index+1, + ctx.PipelineId(), reading.Id, reading.ResourceName, reading.ValueType, @@ -83,29 +88,29 @@ func (s *Sample) LogEventDetails(ctx interfaces.AppFunctionContext, data interfa return true, event } -// ConvertEventToXML is example of transforming an Event and passing the transformed data to to next function in the pipeline +// ConvertEventToXML is example of transforming an Event and passing the transformed data to next function in the pipeline func (s *Sample) ConvertEventToXML(ctx interfaces.AppFunctionContext, data interface{}) (bool, interface{}) { lc := ctx.LoggingClient() - lc.Debug("ConvertEventToXML called") + lc.Debugf("ConvertEventToXML called in pipeline '%s'", ctx.PipelineId()) if data == nil { - return false, errors.New("no Event Received") + return false, fmt.Errorf("function ConvertEventToXML in pipeline '%s': No Data Received", ctx.PipelineId()) } event, ok := data.(dtos.Event) if !ok { - return false, errors.New("type received is not an Event") + return false, fmt.Errorf("function ConvertEventToXML in pipeline '%s': type received is not an Event", ctx.PipelineId()) } xml, err := event.ToXML() if err != nil { - return false, errors.New("failed to convert event to XML") + return false, fmt.Errorf("function ConvertEventToXML in pipeline '%s': failed to convert event to XML", ctx.PipelineId()) } // Example of DEBUG message which by default you don't want to be logged. // To see debug log messages, Set WRITABLE_LOGLEVEL=DEBUG environment variable or // change LogLevel in configuration.toml before running app service. - lc.Debug("Event converted to XML: " + xml) + lc.Debugf("Event converted to XML in pipeline '%s': %s", ctx.PipelineId(), xml) // Returning true indicates that the pipeline execution should continue with the next function // using the event passed as input in this case. @@ -115,18 +120,18 @@ func (s *Sample) ConvertEventToXML(ctx interfaces.AppFunctionContext, data inter // OutputXML is an example of processing transformed data func (s *Sample) OutputXML(ctx interfaces.AppFunctionContext, data interface{}) (bool, interface{}) { lc := ctx.LoggingClient() - lc.Debug("OutputXML called") + lc.Debugf("OutputXML called in pipeline '%s'", ctx.PipelineId()) if data == nil { - return false, errors.New("no XML Received") + return false, fmt.Errorf("function OutputXML in pipeline '%s': No Data Received", ctx.PipelineId()) } xml, ok := data.(string) if !ok { - return false, errors.New("type received is not an string") + return false, fmt.Errorf("function ConvertEventToXML in pipeline '%s': type received is not an string", ctx.PipelineId()) } - lc.Debugf("Outputting the following XML: %s", xml) + lc.Debugf("Outputting the following XML in pipeline '%s': %s", ctx.PipelineId(), xml) // This sends the XML as a response. i.e. publish for MessageBus/MQTT triggers as configured or // HTTP response to for the HTTP Trigger diff --git a/app-service-template/go.mod b/app-service-template/go.mod index b360254dd..658af05f0 100644 --- a/app-service-template/go.mod +++ b/app-service-template/go.mod @@ -7,7 +7,7 @@ go 1.15 require ( github.com/edgexfoundry/app-functions-sdk-go/v2 v2.0.1 github.com/edgexfoundry/go-mod-core-contracts/v2 v2.0.0 - github.com/google/uuid v1.2.0 + github.com/google/uuid v1.3.0 github.com/stretchr/testify v1.7.0 ) diff --git a/app-service-template/go.sum b/app-service-template/go.sum index 5c79f5800..e62d184cd 100644 --- a/app-service-template/go.sum +++ b/app-service-template/go.sum @@ -32,8 +32,9 @@ github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fxamacker/cbor/v2 v2.2.0 h1:6eXqdDDe588rSYAi1HfZKbx6YYQO4mxQ9eC6xYpU/JQ= github.com/fxamacker/cbor/v2 v2.2.0/go.mod h1:TA1xS00nchWmaBnEIxPSE5oHLuJBAVvqrtAnWBwBCVo= +github.com/fxamacker/cbor/v2 v2.3.0 h1:aM45YGMctNakddNNAezPxDUpv38j44Abh+hifNuqXik= +github.com/fxamacker/cbor/v2 v2.3.0/go.mod h1:TA1xS00nchWmaBnEIxPSE5oHLuJBAVvqrtAnWBwBCVo= github.com/go-kit/kit v0.9.0 h1:wDJmvq38kDhkVxi50ni9ykkdUr1PKgqKOoi01fa0Mdk= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.4.0 h1:MP4Eh7ZCb31lleYCFuwm0oe4/YGak+5l1vA2NOE80nA= @@ -57,8 +58,9 @@ github.com/gomodule/redigo v2.0.0+incompatible h1:K/R+8tc58AaqLkqG2Ol3Qk+DR/TlNu github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c h1:964Od4U6p2jUkFxvCydnIczKteheJEzHRToSGK3Bnlw= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs= github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= diff --git a/app-service-template/main.go b/app-service-template/main.go index 7d1748f17..4d327eb9b 100644 --- a/app-service-template/main.go +++ b/app-service-template/main.go @@ -97,9 +97,11 @@ func (app *myApp) CreateAndRunAppService(serviceKey string, newServiceFactory fu return -1 } - // TODO: Replace below functions with built in and/or your custom functions for your use case. - // See https://docs.edgexfoundry.org/2.0/microservices/application/BuiltIn/ for list of built-in functions sample := functions.NewSample() + + // TODO: Replace below functions with built in and/or your custom functions for your use case + // or remove is using Pipeline By Topic below. + // See https://docs.edgexfoundry.org/2.0/microservices/application/BuiltIn/ for list of built-in functions err = app.service.SetFunctionsPipeline( transforms.NewFilterFor(deviceNames).FilterByDeviceName, sample.LogEventDetails, @@ -110,6 +112,36 @@ func (app *myApp) CreateAndRunAppService(serviceKey string, newServiceFactory fu return -1 } + // TODO: Remove adding functions pipelines by topic if default pipeline above is all your Use Case needs. + // Or remove default above if your use case needs multiple pipelines by topic. + // Example of adding functions pipelines by topic. + // These pipelines will only execute if the specified topic match the incoming topic. + // Note: Device services publish to the 'edgex/events/device///' topic + // Core Data publishes to the 'edgex/events/core///' topic + // Note: This example with default above causes Events from Random-Float-Device device to be processed twice + // resulting in the XML to be published back to the MessageBus twice. + // See for more details. + err = app.service.AddFunctionsPipelineForTopic("Floats", "edgex/events/#/#/Random-Float-Device/#", + transforms.NewFilterFor(deviceNames).FilterByDeviceName, + sample.LogEventDetails, + sample.ConvertEventToXML, + sample.OutputXML) + if err != nil { + app.lc.Errorf("AddFunctionsPipelineForTopic returned error: %s", err.Error()) + return -1 + } + // Note: This example with default above causes Events from Int32 source to be processed twice + // resulting in the XML to be published back to the MessageBus twice. + err = app.service.AddFunctionsPipelineForTopic("Int32s", "edgex/events/#/#/#/Int32", + transforms.NewFilterFor(deviceNames).FilterByDeviceName, + sample.LogEventDetails, + sample.ConvertEventToXML, + sample.OutputXML) + if err != nil { + app.lc.Errorf("AddFunctionsPipelineForTopic returned error: %s", err.Error()) + return -1 + } + if err := app.service.MakeItRun(); err != nil { app.lc.Errorf("MakeItRun returned error: %s", err.Error()) return -1 @@ -120,10 +152,10 @@ func (app *myApp) CreateAndRunAppService(serviceKey string, newServiceFactory fu return 0 } -// TODO: Update using your Custom configuration 'writeable' type or remove if not using ListenForCustomConfigChanges // ProcessConfigUpdates processes the updated configuration for the service's writable configuration. // At a minimum it must copy the updated configuration into the service's current configuration. Then it can // do any special processing for changes that require more. +// TODO: Update using your Custom configuration 'writeable' type or remove if not using ListenForCustomConfigChanges func (app *myApp) ProcessConfigUpdates(rawWritableConfig interface{}) { updated, ok := rawWritableConfig.(*config.AppCustomConfig) if !ok { diff --git a/app-service-template/main_test.go b/app-service-template/main_test.go index 96397f384..933da76f8 100644 --- a/app-service-template/main_test.go +++ b/app-service-template/main_test.go @@ -44,6 +44,8 @@ func TestCreateAndRunService_Success(t *testing.T) { Return([]string{"Random-Boolean-Device, Random-Integer-Device"}, nil) mockAppService.On("SetFunctionsPipeline", mock.Anything, mock.Anything, mock.Anything, mock.Anything). Return(nil) + mockAppService.On("AddFunctionsPipelineForTopic", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything). + Return(nil) mockAppService.On("LoadCustomConfig", mock.Anything, mock.Anything, mock.Anything). Return(nil).Run(func(args mock.Arguments) { // set the required configuration so validation passes @@ -148,6 +150,8 @@ func TestCreateAndRunService_MakeItRun_Failed(t *testing.T) { Return(nil) mockAppService.On("SetFunctionsPipeline", mock.Anything, mock.Anything, mock.Anything, mock.Anything). Return(nil) + mockAppService.On("AddFunctionsPipelineForTopic", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything). + Return(nil) mockAppService.On("MakeItRun").Return(fmt.Errorf("Failed")).Run(func(args mock.Arguments) { makeItRunCalled = true }) diff --git a/internal/app/configupdates.go b/internal/app/configupdates.go index e342e32ee..79aed35c7 100644 --- a/internal/app/configupdates.go +++ b/internal/app/configupdates.go @@ -132,8 +132,6 @@ func (processor *ConfigUpdateProcessor) processConfigChangedStoreForwardEnabled( return storeClient }, }) - - sdk.runtime.Initialize(sdk.dic) } sdk.startStoreForward() @@ -149,9 +147,8 @@ func (processor *ConfigUpdateProcessor) processConfigChangedPipeline() { transforms, err := sdk.LoadConfigurablePipeline() if err != nil { sdk.LoggingClient().Error("unable to reload Configurable Pipeline from new configuration: " + err.Error()) - // Reset the transforms so error occurs when attempting to execute the pipeline. - sdk.transforms = nil - sdk.runtime.SetTransforms(nil) + // Reset the default pipeline transforms to nil so error occurs when attempting to execute the pipeline. + _ = sdk.runtime.SetDefaultFunctionsPipeline(nil) return } diff --git a/internal/app/service.go b/internal/app/service.go index f058a3974..1352a034f 100644 --- a/internal/app/service.go +++ b/internal/app/service.go @@ -20,7 +20,6 @@ import ( "context" "errors" "fmt" - "github.com/edgexfoundry/app-functions-sdk-go/v2/internal/appfunction" nethttp "net/http" "os" "os/signal" @@ -30,6 +29,7 @@ import ( "syscall" "github.com/edgexfoundry/app-functions-sdk-go/v2/internal" + "github.com/edgexfoundry/app-functions-sdk-go/v2/internal/appfunction" "github.com/edgexfoundry/app-functions-sdk-go/v2/internal/bootstrap/container" "github.com/edgexfoundry/app-functions-sdk-go/v2/internal/bootstrap/handlers" "github.com/edgexfoundry/app-functions-sdk-go/v2/internal/common" @@ -77,7 +77,6 @@ type Service struct { targetType interface{} config *common.ConfigurationStruct lc logger.LoggingClient - transforms []interfaces.AppFunction usingConfigurablePipeline bool runtime *runtime.GolangRuntime webserver *webserver.WebServer @@ -135,11 +134,11 @@ func (svc *Service) AddBackgroundPublisherWithTopic(capacity int, topic string) // for custom triggers we don't know if background publishing available or not // but probably makes sense to trust the caller. if svc.config.Trigger.Type == TriggerTypeHTTP || svc.config.Trigger.Type == TriggerTypeMQTT { - return nil, fmt.Errorf("Background publishing not supported for %s trigger.", svc.config.Trigger.Type) + return nil, fmt.Errorf("background publishing not supported for %s trigger", svc.config.Trigger.Type) } - bgchan, pub := newBackgroundPublisher(topic, capacity) - svc.backgroundPublishChannel = bgchan + bgChan, pub := newBackgroundPublisher(topic, capacity) + svc.backgroundPublishChannel = bgChan return pub, nil } @@ -160,28 +159,20 @@ func (svc *Service) MakeItRun() error { svc.ctx.stop = stop - svc.runtime = &runtime.GolangRuntime{ - TargetType: svc.targetType, - ServiceKey: svc.serviceKey, - } - - svc.runtime.Initialize(svc.dic) - svc.runtime.SetTransforms(svc.transforms) - // determine input type and create trigger for it t := svc.setupTrigger(svc.config, svc.runtime) if t == nil { - return errors.New("Failed to create Trigger") + return errors.New("failed to create Trigger") } // Initialize the trigger (i.e. start a web server, or connect to message bus) deferred, err := t.Initialize(svc.ctx.appWg, svc.ctx.appCtx, svc.backgroundPublishChannel) if err != nil { svc.lc.Error(err.Error()) - return errors.New("Failed to initialize Trigger") + return errors.New("failed to initialize Trigger") } - // deferred is a a function that needs to be called when services exits. + // deferred is a function that needs to be called when services exits. svc.addDeferred(deferred) if svc.config.Writable.StoreAndForward.Enabled { @@ -219,7 +210,7 @@ func (svc *Service) MakeItRun() error { svc.ctx.storeForwardWg.Wait() } - svc.ctx.appCancelCtx() // Cancel all long running go funcs + svc.ctx.appCancelCtx() // Cancel all long-running go funcs svc.ctx.appWg.Wait() // Call all the deferred funcs that need to happen when exiting. // These are things like un-register from the Registry, disconnect from the Message Bus, etc @@ -231,8 +222,25 @@ func (svc *Service) MakeItRun() error { } // LoadConfigurablePipeline sets the function pipeline from configuration +// Note this API has been deprecated, replaced by LoadConfigurableFunctionPipelines and will be removed in a future release +// TODO: Remove this API in 3.0 release func (svc *Service) LoadConfigurablePipeline() ([]interfaces.AppFunction, error) { - var pipeline []interfaces.AppFunction + pipelines, err := svc.LoadConfigurableFunctionPipelines() + if err != nil { + return nil, err + } + + defaultPipeline, found := pipelines[interfaces.DefaultPipelineId] + if !found { + return nil, fmt.Errorf("default functions pipeline not configured") + } + + return defaultPipeline.Transforms, nil +} + +// LoadConfigurableFunctionPipelines return the configured function pipelines (default and per topic) from configuration. +func (svc *Service) LoadConfigurableFunctionPipelines() (map[string]interfaces.FunctionPipeline, error) { + pipelines := make(map[string]interfaces.FunctionPipeline) svc.usingConfigurablePipeline = true @@ -244,25 +252,70 @@ func (svc *Service) LoadConfigurablePipeline() ([]interfaces.AppFunction, error) configurable := reflect.ValueOf(NewConfigurable(svc.lc)) pipelineConfig := svc.config.Writable.Pipeline - executionOrder := util.DeleteEmptyAndTrim(strings.FieldsFunc(pipelineConfig.ExecutionOrder, util.SplitComma)) - if len(executionOrder) <= 0 { - return nil, errors.New( - "execution Order has 0 functions specified. You must have a least one function in the pipeline") + defaultExecutionOrder := strings.TrimSpace(pipelineConfig.ExecutionOrder) + + if len(defaultExecutionOrder) == 0 && len(pipelineConfig.PerTopicPipelines) == 0 { + return nil, errors.New("default ExecutionOrder has 0 functions specified and PerTopicPipelines is empty") + } + + if len(defaultExecutionOrder) > 0 { + svc.lc.Debugf("Default Function Pipeline Execution Order: [%s]", pipelineConfig.ExecutionOrder) + functionNames := util.DeleteEmptyAndTrim(strings.FieldsFunc(defaultExecutionOrder, util.SplitComma)) + + transforms, err := svc.loadConfigurablePipelineTransforms(interfaces.DefaultPipelineId, functionNames, pipelineConfig.Functions, configurable) + if err != nil { + return nil, err + } + pipeline := interfaces.FunctionPipeline{ + Id: interfaces.DefaultPipelineId, + Transforms: transforms, + Topic: runtime.TopicWildCard, + } + pipelines[pipeline.Id] = pipeline + } + + if len(pipelineConfig.PerTopicPipelines) > 0 { + for _, perTopicPipeline := range pipelineConfig.PerTopicPipelines { + svc.lc.Debugf("'%s' Function Pipeline Execution Order: [%s]", perTopicPipeline.Id, perTopicPipeline.ExecutionOrder) + + functionNames := util.DeleteEmptyAndTrim(strings.FieldsFunc(perTopicPipeline.ExecutionOrder, util.SplitComma)) + + transforms, err := svc.loadConfigurablePipelineTransforms(perTopicPipeline.Id, functionNames, pipelineConfig.Functions, configurable) + if err != nil { + return nil, err + } + + pipeline := interfaces.FunctionPipeline{ + Id: perTopicPipeline.Id, + Transforms: transforms, + Topic: perTopicPipeline.Topic, + } + + pipelines[pipeline.Id] = pipeline + } } - svc.lc.Debugf("Function Pipeline Execution Order: [%s]", pipelineConfig.ExecutionOrder) + return pipelines, nil +} + +func (svc *Service) loadConfigurablePipelineTransforms( + pipelineId string, + executionOrder []string, + functions map[string]common.PipelineFunction, + configurable reflect.Value) ([]interfaces.AppFunction, error) { + var transforms []interfaces.AppFunction for _, functionName := range executionOrder { functionName = strings.TrimSpace(functionName) - configuration, ok := pipelineConfig.Functions[functionName] + configuration, ok := functions[functionName] if !ok { - return nil, fmt.Errorf("function '%s' configuration not found in Pipeline.Functions section", functionName) + return nil, fmt.Errorf("function '%s' configuration not found in Pipeline.Functions section for pipeline '%s'", functionName, pipelineId) } functionValue, functionType, err := svc.findMatchingFunction(configurable, functionName) if err != nil { - return nil, err + return nil, fmt.Errorf("%s for pipeline '%s'", err.Error(), pipelineId) } // determine number of parameters required for function call @@ -282,7 +335,8 @@ func (svc *Service) LoadConfigurablePipeline() ([]interfaces.AppFunction, error) default: return nil, fmt.Errorf( - "function %s has an unsupported parameter type: %s", + "function %s for pipeline '%s' has an unsupported parameter type: %s", + pipelineId, functionName, parameter.String(), ) @@ -291,36 +345,68 @@ func (svc *Service) LoadConfigurablePipeline() ([]interfaces.AppFunction, error) function, ok := functionValue.Call(inputParameters)[0].Interface().(interfaces.AppFunction) if !ok { - return nil, fmt.Errorf("failed to cast function %s as AppFunction type", functionName) + return nil, fmt.Errorf("failed to cast function %s as AppFunction type for pipeline '%s'", functionName, pipelineId) } if function == nil { - return nil, fmt.Errorf("%s from configuration failed", functionName) + return nil, fmt.Errorf("%s from configuration failed for pipeline '%s'", functionName, pipelineId) } - pipeline = append(pipeline, function) - svc.lc.Debugf( - "%s function added to configurable pipeline with parameters: [%s]", + transforms = append(transforms, function) + svc.lc.Debugf("%s function added to '%s' configurable pipeline with parameters: [%s]", functionName, + pipelineId, listParameters(configuration.Parameters)) } - return pipeline, nil + return transforms, nil } -// SetFunctionsPipeline sets the function pipeline to the list of specified functions in the order provided. +// SetFunctionsPipeline has been deprecated and replaced by SetDefaultFunctionsPipeline. func (svc *Service) SetFunctionsPipeline(transforms ...interfaces.AppFunction) error { + return svc.SetDefaultFunctionsPipeline(transforms...) +} + +// SetDefaultFunctionsPipeline sets the default functions pipeline to the list of specified functions in the order provided. +func (svc *Service) SetDefaultFunctionsPipeline(transforms ...interfaces.AppFunction) error { if len(transforms) == 0 { return errors.New("no transforms provided to pipeline") } - svc.transforms = transforms + svc.runtime.TargetType = svc.targetType + err := svc.runtime.SetDefaultFunctionsPipeline(transforms) + if err != nil { + return err + } + + svc.lc.Debugf("Default pipeline added with %d transform(s)", len(transforms)) - if svc.runtime != nil { - svc.runtime.SetTransforms(transforms) - svc.runtime.TargetType = svc.targetType + return nil +} + +// AddFunctionsPipelineForTopic adds a functions pipeline for the specified for the specified id and topic +func (svc *Service) AddFunctionsPipelineForTopic(id string, topic string, transforms ...interfaces.AppFunction) error { + switch strings.ToUpper(svc.config.Trigger.Type) { + case TriggerTypeMessageBus: + case TriggerTypeMQTT: + default: + return errors.New("pipeline per topic only valid with EdgeX MessageBus and External MQTT") + } + + if len(transforms) == 0 { + return errors.New("no transforms provided to pipeline") } + if len(strings.TrimSpace(topic)) == 0 { + return errors.New("topic for pipeline can not be blank") + } + + err := svc.runtime.AddFunctionsPipeline(id, topic, transforms) + if err != nil { + return err + } + + svc.lc.Debugf("Pipeline '%s' added for topic '%s' with %d transform(s)", id, topic, len(transforms)) return nil } @@ -417,13 +503,15 @@ func (svc *Service) Initialize() error { }, ) - // deferred is a a function that needs to be called when services exits. + // deferred is a function that needs to be called when services exits. svc.addDeferred(deferred) if !successful { return fmt.Errorf("boostrapping failed") } + svc.runtime = runtime.NewGolangRuntime(svc.serviceKey, svc.targetType, svc.dic) + // Bootstrapping is complete, so now need to retrieve the needed objects from the containers. svc.lc = bootstrapContainer.LoggingClientFrom(svc.dic.Get) @@ -569,7 +657,7 @@ func (svc *Service) setServiceKey(profile string) { return } - // Have to handle environment override here before common bootstrap is used so it is passed the proper service key + // Have to handle environment override here before common bootstrap is used, so it is passed the proper service key profileOverride := os.Getenv(envProfile) if len(profileOverride) > 0 { profile = profileOverride @@ -584,7 +672,7 @@ func (svc *Service) setServiceKey(profile string) { svc.serviceKey = strings.Replace(svc.serviceKey, svc.profileSuffixPlaceholder, "", 1) } -// BuildContext allows external callers that may need a context (eg background publishers) +// BuildContext allows external callers that may need a context (e.g. background publishers) // to easily create one around the service's dic func (svc *Service) BuildContext(correlationId string, contentType string) interfaces.AppFunctionContext { return appfunction.NewContext(correlationId, svc.dic, contentType) diff --git a/internal/app/service_test.go b/internal/app/service_test.go index 5378a969e..89fa854f8 100644 --- a/internal/app/service_test.go +++ b/internal/app/service_test.go @@ -18,13 +18,16 @@ package app import ( "fmt" - "github.com/edgexfoundry/app-functions-sdk-go/v2/internal/appfunction" - "github.com/google/uuid" "net/http" "os" "reflect" "testing" + "github.com/google/uuid" + + "github.com/edgexfoundry/app-functions-sdk-go/v2/internal/appfunction" + builtin "github.com/edgexfoundry/app-functions-sdk-go/v2/pkg/transforms" + "github.com/edgexfoundry/app-functions-sdk-go/v2/internal/bootstrap/container" "github.com/edgexfoundry/app-functions-sdk-go/v2/internal/common" "github.com/edgexfoundry/app-functions-sdk-go/v2/internal/runtime" @@ -237,9 +240,10 @@ func TestSetupHTTPTrigger(t *testing.T) { }, }, } - testRuntime := &runtime.GolangRuntime{} - testRuntime.Initialize(dic) - testRuntime.SetTransforms(sdk.transforms) + + testRuntime := runtime.NewGolangRuntime("", nil, dic) + err := testRuntime.SetDefaultFunctionsPipeline(nil) + require.NoError(t, err) trigger := sdk.setupTrigger(sdk.config, testRuntime) result := IsInstanceOf(trigger, (*triggerHttp.Trigger)(nil)) assert.True(t, result, "Expected Instance of HTTP Trigger") @@ -254,15 +258,15 @@ func TestSetupMessageBusTrigger(t *testing.T) { }, }, } - testRuntime := &runtime.GolangRuntime{} - testRuntime.Initialize(dic) - testRuntime.SetTransforms(sdk.transforms) + testRuntime := runtime.NewGolangRuntime("", nil, dic) + err := testRuntime.SetDefaultFunctionsPipeline(nil) + require.NoError(t, err) trigger := sdk.setupTrigger(sdk.config, testRuntime) result := IsInstanceOf(trigger, (*messagebus.Trigger)(nil)) assert.True(t, result, "Expected Instance of Message Bus Trigger") } -func TestSetFunctionsPipelineNoTransforms(t *testing.T) { +func TestSetDefaultFunctionsPipelineNoTransforms(t *testing.T) { sdk := Service{ lc: lc, config: &common.ConfigurationStruct{ @@ -271,15 +275,15 @@ func TestSetFunctionsPipelineNoTransforms(t *testing.T) { }, }, } - err := sdk.SetFunctionsPipeline() + err := sdk.SetDefaultFunctionsPipeline() require.Error(t, err, "There should be an error") assert.Equal(t, "no transforms provided to pipeline", err.Error()) } -func TestSetFunctionsPipelineOneTransform(t *testing.T) { - sdk := Service{ +func TestSetDefaultFunctionsPipelineOneTransform(t *testing.T) { + service := Service{ lc: lc, - runtime: &runtime.GolangRuntime{}, + runtime: runtime.NewGolangRuntime("", nil, dic), config: &common.ConfigurationStruct{ Trigger: common.TriggerInfo{ Type: TriggerTypeMessageBus, @@ -290,12 +294,60 @@ func TestSetFunctionsPipelineOneTransform(t *testing.T) { return true, nil } - sdk.runtime.Initialize(dic) - err := sdk.SetFunctionsPipeline(function) + err := service.SetDefaultFunctionsPipeline(function) require.NoError(t, err) - assert.Equal(t, 1, len(sdk.transforms)) } +func TestService_AddFunctionsPipelineForTopic(t *testing.T) { + service := Service{ + lc: lc, + runtime: runtime.NewGolangRuntime("", nil, nil), + config: &common.ConfigurationStruct{ + Trigger: common.TriggerInfo{ + Type: TriggerTypeMessageBus, + }, + }, + } + + tags := builtin.NewTags(nil) + + transforms := []interfaces.AppFunction{tags.AddTags} + + // This sets the Default Pipeline allowing to test for duplicate iD. + err := service.SetDefaultFunctionsPipeline(transforms...) + require.NoError(t, err) + + tests := []struct { + name string + id string + trigger string + topic string + transforms []interfaces.AppFunction + expectError bool + }{ + {"Happy Path", "123", TriggerTypeMessageBus, "#", transforms, false}, + {"Empty Topic", "123", TriggerTypeMessageBus, " ", transforms, true}, + {"No Transforms", "123", TriggerTypeMessageBus, "#", nil, true}, + {"Duplicate Id", interfaces.DefaultPipelineId, TriggerTypeMessageBus, "#", transforms, true}, + {"Wrong Trigger Type", "123", TriggerTypeHTTP, "#", transforms, true}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + service.config.Trigger.Type = test.trigger + + err := service.AddFunctionsPipelineForTopic(test.id, test.topic, test.transforms...) + if test.expectError { + require.Error(t, err) + return + } + + require.NoError(t, err) + actual := service.runtime.GetPipelineById(test.id) + assert.Equal(t, transforms, actual.Transforms) + }) + } +} func TestApplicationSettings(t *testing.T) { expectedSettingKey := "ApplicationName" expectedSettingValue := "simple-filter-xml" @@ -397,26 +449,50 @@ func TestGetAppSettingStringsNoAppSettings(t *testing.T) { assert.Contains(t, err.Error(), expected, "Error not as expected") } -func TestLoadConfigurablePipelineFunctionNotFound(t *testing.T) { - sdk := Service{ +func TestLoadConfigurableFunctionPipelinesDefaultNotFound(t *testing.T) { + service := Service{ lc: lc, config: &common.ConfigurationStruct{ Writable: common.WritableInfo{ Pipeline: common.PipelineInfo{ - ExecutionOrder: "Bogus", - Functions: make(map[string]common.PipelineFunction), + ExecutionOrder: "Bogus", + PerTopicPipelines: make(map[string]common.TopicPipeline), + Functions: make(map[string]common.PipelineFunction), }, }, }, } - appFunctions, err := sdk.LoadConfigurablePipeline() - require.Error(t, err, "expected error for function not found in config") - assert.Equal(t, "function 'Bogus' configuration not found in Pipeline.Functions section", err.Error()) - assert.Nil(t, appFunctions, "expected app functions list to be nil") + tests := []struct { + name string + defaultExecutionOrder string + perTopicExecutionOrder string + }{ + {"Default Not Found", "Bogus", ""}, + {"PerTopicNotFound", "", "Bogus"}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + service.config.Writable.Pipeline.ExecutionOrder = test.defaultExecutionOrder + if len(test.perTopicExecutionOrder) > 0 { + service.config.Writable.Pipeline.PerTopicPipelines["bogus"] = common.TopicPipeline{ + Id: "bogus", + Topic: "#", + ExecutionOrder: test.perTopicExecutionOrder, + } + } + + appFunctions, err := service.LoadConfigurableFunctionPipelines() + require.Error(t, err, "expected error for function not found in config") + assert.Contains(t, err.Error(), "function 'Bogus' configuration not found in Pipeline.Functions section") + assert.Nil(t, appFunctions, "expected app functions list to be nil") + + }) + } } -func TestLoadConfigurablePipelineNotABuiltInSdkFunction(t *testing.T) { +func TestLoadConfigurableFunctionPipelinesNotABuiltInSdkFunction(t *testing.T) { functions := make(map[string]common.PipelineFunction) functions["Bogus"] = common.PipelineFunction{} @@ -432,21 +508,25 @@ func TestLoadConfigurablePipelineNotABuiltInSdkFunction(t *testing.T) { }, } - appFunctions, err := sdk.LoadConfigurablePipeline() + appFunctions, err := sdk.LoadConfigurableFunctionPipelines() require.Error(t, err, "expected error") - assert.Equal(t, "function Bogus is not a built in SDK function", err.Error()) + assert.Contains(t, err.Error(), "function Bogus is not a built in SDK function") assert.Nil(t, appFunctions, "expected app functions list to be nil") } -func TestLoadConfigurablePipelineNumFunctions(t *testing.T) { - functions := make(map[string]common.PipelineFunction) - functions["FilterByDeviceName"] = common.PipelineFunction{ +func TestLoadConfigurableFunctionPipelinesNumFunctions(t *testing.T) { + expectedPipelinesCount := 2 + expectedTransformsCount := 3 + perTopicPipelineId := "pre-topic" + + transforms := make(map[string]common.PipelineFunction) + transforms["FilterByDeviceName"] = common.PipelineFunction{ Parameters: map[string]string{"DeviceNames": "Random-Float-Device, Random-Integer-Device"}, } - functions["Transform"] = common.PipelineFunction{ + transforms["Transform"] = common.PipelineFunction{ Parameters: map[string]string{TransformType: TransformXml}, } - functions["SetResponseData"] = common.PipelineFunction{} + transforms["SetResponseData"] = common.PipelineFunction{} sdk := Service{ lc: lc, @@ -454,16 +534,31 @@ func TestLoadConfigurablePipelineNumFunctions(t *testing.T) { Writable: common.WritableInfo{ Pipeline: common.PipelineInfo{ ExecutionOrder: "FilterByDeviceName, Transform, SetResponseData", - Functions: functions, + PerTopicPipelines: map[string]common.TopicPipeline{ + perTopicPipelineId: { + Id: perTopicPipelineId, + Topic: "#", + ExecutionOrder: "FilterByDeviceName, Transform, SetResponseData", + }, + }, + Functions: transforms, }, }, }, } - appFunctions, err := sdk.LoadConfigurablePipeline() + pipelines, err := sdk.LoadConfigurableFunctionPipelines() require.NoError(t, err) - require.NotNil(t, appFunctions, "expected app functions list to be set") - assert.Equal(t, 3, len(appFunctions)) + require.NotNil(t, pipelines, "expected app pipelines list to be set") + assert.Equal(t, expectedPipelinesCount, len(pipelines)) + + pipeline, found := pipelines[interfaces.DefaultPipelineId] + require.True(t, found) + assert.Equal(t, expectedTransformsCount, len(pipeline.Transforms)) + + pipeline, found = pipelines[perTopicPipelineId] + require.True(t, found) + assert.Equal(t, expectedTransformsCount, len(pipeline.Transforms)) } func TestUseTargetTypeOfByteArrayTrue(t *testing.T) { @@ -798,15 +893,15 @@ func TestService_BuildContext(t *testing.T) { contentType := uuid.NewString() - appctx := sdk.BuildContext(correlationId, contentType) + appCtx := sdk.BuildContext(correlationId, contentType) - require.NotNil(t, appctx) + require.NotNil(t, appCtx) - require.Equal(t, correlationId, appctx.CorrelationID()) - require.Equal(t, contentType, appctx.InputContentType()) + require.Equal(t, correlationId, appCtx.CorrelationID()) + require.Equal(t, contentType, appCtx.InputContentType()) - castctx := appctx.(*appfunction.Context) + castCtx := appCtx.(*appfunction.Context) - require.NotNil(t, castctx) - require.Equal(t, dic, castctx.Dic) + require.NotNil(t, castCtx) + require.Equal(t, dic, castCtx.Dic) } diff --git a/internal/app/triggerfactory.go b/internal/app/triggerfactory.go index 843f74319..e55da423f 100644 --- a/internal/app/triggerfactory.go +++ b/internal/app/triggerfactory.go @@ -74,7 +74,8 @@ func (svc *Service) defaultTriggerMessageProcessor(appContext interfaces.AppFunc return errors.New("App Context must be an instance of internal appfunction.Context. Use NewAppContext to create instance.") } - messageError := svc.runtime.ProcessMessage(context, envelope) + defaultPipeline := svc.runtime.GetDefaultPipeline() + messageError := svc.runtime.ProcessMessage(context, envelope, defaultPipeline) if messageError != nil { // ProcessMessage logs the error, so no need to log it here. return messageError.Err diff --git a/internal/appfunction/context.go b/internal/appfunction/context.go index 5c9e587ab..fb6398cd8 100644 --- a/internal/appfunction/context.go +++ b/internal/appfunction/context.go @@ -24,11 +24,13 @@ import ( "strings" "time" - "github.com/edgexfoundry/app-functions-sdk-go/v2/internal/bootstrap/container" bootstrapContainer "github.com/edgexfoundry/go-mod-bootstrap/v2/bootstrap/container" + "github.com/edgexfoundry/app-functions-sdk-go/v2/internal/bootstrap/container" + "github.com/edgexfoundry/app-functions-sdk-go/v2/pkg/interfaces" + "github.com/edgexfoundry/go-mod-bootstrap/v2/di" - "github.com/edgexfoundry/go-mod-core-contracts/v2/clients/interfaces" + clients "github.com/edgexfoundry/go-mod-core-contracts/v2/clients/interfaces" "github.com/edgexfoundry/go-mod-core-contracts/v2/clients/logger" "github.com/edgexfoundry/go-mod-core-contracts/v2/dtos" "github.com/edgexfoundry/go-mod-core-contracts/v2/dtos/common" @@ -39,7 +41,7 @@ import ( func NewContext(correlationID string, dic *di.Container, inputContentType string) *Context { return &Context{ correlationID: correlationID, - // Dic is public so we can confirm it is set correctly + // Dic is public, so we can confirm it is set correctly Dic: dic, inputContentType: inputContentType, contextData: make(map[string]string, 0), @@ -49,7 +51,7 @@ func NewContext(correlationID string, dic *di.Container, inputContentType string // Context contains the data functions that implement the interfaces.AppFunctionContext type Context struct { - // Dic is public so we can confirm it is set correctly + // Dic is public, so we can confirm it is set correctly Dic *di.Container correlationID string inputContentType string @@ -134,37 +136,37 @@ func (appContext *Context) LoggingClient() logger.LoggingClient { } // EventClient returns the Event client, which may be nil, from the dependency injection container -func (appContext *Context) EventClient() interfaces.EventClient { +func (appContext *Context) EventClient() clients.EventClient { return container.EventClientFrom(appContext.Dic.Get) } // CommandClient returns the Command client, which may be nil, from the dependency injection container -func (appContext *Context) CommandClient() interfaces.CommandClient { +func (appContext *Context) CommandClient() clients.CommandClient { return container.CommandClientFrom(appContext.Dic.Get) } // DeviceServiceClient returns the DeviceService client, which may be nil, from the dependency injection container -func (appContext *Context) DeviceServiceClient() interfaces.DeviceServiceClient { +func (appContext *Context) DeviceServiceClient() clients.DeviceServiceClient { return container.DeviceServiceClientFrom(appContext.Dic.Get) } // DeviceProfileClient returns the DeviceProfile client, which may be nil, from the dependency injection container -func (appContext *Context) DeviceProfileClient() interfaces.DeviceProfileClient { +func (appContext *Context) DeviceProfileClient() clients.DeviceProfileClient { return container.DeviceProfileClientFrom(appContext.Dic.Get) } // DeviceClient returns the Device client, which may be nil, from the dependency injection container -func (appContext *Context) DeviceClient() interfaces.DeviceClient { +func (appContext *Context) DeviceClient() clients.DeviceClient { return container.DeviceClientFrom(appContext.Dic.Get) } // NotificationClient returns the Notification client, which may be nil, from the dependency injection container -func (appContext *Context) NotificationClient() interfaces.NotificationClient { +func (appContext *Context) NotificationClient() clients.NotificationClient { return container.NotificationClientFrom(appContext.Dic.Get) } // SubscriptionClient returns the Subscription client, which may be nil, from the dependency injection container -func (appContext *Context) SubscriptionClient() interfaces.SubscriptionClient { +func (appContext *Context) SubscriptionClient() clients.SubscriptionClient { return container.SubscriptionClientFrom(appContext.Dic.Get) } @@ -253,3 +255,9 @@ func (appContext *Context) ApplyValues(format string) (string, error) { return result, nil } + +// PipelineId returns the ID of the pipeline that is executing +func (appContext *Context) PipelineId() string { + id, _ := appContext.GetValue(interfaces.PIPELINEID) + return id +} diff --git a/internal/common/config.go b/internal/common/config.go index 19662716f..3d1062645 100644 --- a/internal/common/config.go +++ b/internal/common/config.go @@ -17,8 +17,9 @@ package common import ( - "github.com/edgexfoundry/app-functions-sdk-go/v2/internal/store/db" bootstrapConfig "github.com/edgexfoundry/go-mod-bootstrap/v2/config" + + "github.com/edgexfoundry/app-functions-sdk-go/v2/internal/store/db" ) // WritableInfo is used to hold configuration information that is considered "live" or can be changed on the fly without a restart of the service. @@ -148,14 +149,33 @@ type ExternalMqttConfig struct { AuthMode string } +// PipelineInfo defines the top level data for configurable pipelines type PipelineInfo struct { - ExecutionOrder string + // ExecutionOrder is a list of functions, in execution order, for the default configurable pipeline + ExecutionOrder string + // PerTopicPipelines is a collection of pipelines that only execute if the incoming topic matched the pipelines configured topic + PerTopicPipelines map[string]TopicPipeline + // UseTargetTypeOfByteArray indicates if raw []byte type is to be used for the TargetType UseTargetTypeOfByteArray bool - Functions map[string]PipelineFunction + // Functions is a collection of pipeline functions with configured parameters to be used in the ExecutionOrder of one + // of the configured pipelines (default or pre topic) + Functions map[string]PipelineFunction +} + +// TopicPipeline define the data to a Pre Topic functions pipeline +type TopicPipeline struct { + // Id is the unique ID of the pipeline instance + Id string + // Topic is the topic which must match the incoming topic inorder for the pipeline to execute + Topic string + // ExecutionOrder is a list of functions, in execution order, for the pipeline instance + ExecutionOrder string } +// PipelineFunction is a collection of built-in pipeline functions configurations. +// The map key must be unique start with the name of one of the built-in configurable functions type PipelineFunction struct { - // Name string + // Parameters is the collection of configurable parameters specific to the built-in configurable function specified by the map key. Parameters map[string]string } diff --git a/internal/runtime/runtime.go b/internal/runtime/runtime.go index 5ae74b826..8e13e4896 100644 --- a/internal/runtime/runtime.go +++ b/internal/runtime/runtime.go @@ -23,6 +23,7 @@ import ( "fmt" "net/http" "reflect" + "runtime" "strings" "sync" @@ -41,11 +42,27 @@ import ( "github.com/fxamacker/cbor/v2" ) +const ( + TopicWildCard = "#" + TopicLevelSeparator = "/" +) + +func NewFunctionPipeline(id string, topic string, transforms []interfaces.AppFunction) interfaces.FunctionPipeline { + pipeline := interfaces.FunctionPipeline{ + Id: id, + Transforms: transforms, + Topic: topic, + Hash: calculatePipelineHash(transforms), + } + + return pipeline +} + // GolangRuntime represents the golang runtime environment type GolangRuntime struct { TargetType interface{} ServiceKey string - transforms []interfaces.AppFunction + pipelines map[string]*interfaces.FunctionPipeline isBusyCopying sync.Mutex storeForward storeForwardInfo dic *di.Container @@ -56,34 +73,67 @@ type MessageError struct { ErrorCode int } -// Initialize sets the internal reference to the StoreClient for use when Store and Forward is enabled -func (gr *GolangRuntime) Initialize(dic *di.Container) { - gr.dic = dic - gr.storeForward.runtime = gr +// NewGolangRuntime creates and initializes the GolangRuntime instance +func NewGolangRuntime(serviceKey string, targetType interface{}, dic *di.Container) *GolangRuntime { + gr := &GolangRuntime{ + ServiceKey: serviceKey, + TargetType: targetType, + dic: dic, + pipelines: make(map[string]*interfaces.FunctionPipeline), + } + gr.storeForward.dic = dic + gr.storeForward.runtime = gr + + return gr +} + +// SetDefaultFunctionsPipeline sets the default function pipeline +func (gr *GolangRuntime) SetDefaultFunctionsPipeline(transforms []interfaces.AppFunction) error { + pipeline := gr.GetDefaultPipeline() + if pipeline.Transforms != nil { + gr.isBusyCopying.Lock() + pipeline.Transforms = transforms + pipeline.Hash = calculatePipelineHash(transforms) + gr.isBusyCopying.Unlock() + return nil + } + + return gr.AddFunctionsPipeline(interfaces.DefaultPipelineId, TopicWildCard, transforms) } -// SetTransforms is thread safe to set transforms -func (gr *GolangRuntime) SetTransforms(transforms []interfaces.AppFunction) { +// AddFunctionsPipeline is thread safe to set transforms +func (gr *GolangRuntime) AddFunctionsPipeline(id string, topic string, transforms []interfaces.AppFunction) error { + _, exists := gr.pipelines[id] + if exists { + return fmt.Errorf("pipeline with Id='%s' already exists", id) + } + + pipeline := NewFunctionPipeline(id, topic, transforms) gr.isBusyCopying.Lock() - gr.transforms = transforms - gr.storeForward.pipelineHash = gr.storeForward.calculatePipelineHash() // Only need to calculate hash when the pipeline changes. + gr.pipelines[id] = &pipeline gr.isBusyCopying.Unlock() + + return nil } -// ProcessMessage sends the contents of the message thru the functions pipeline -func (gr *GolangRuntime) ProcessMessage(appContext *appfunction.Context, envelope types.MessageEnvelope) *MessageError { +// ProcessMessage sends the contents of the message through the functions pipeline +func (gr *GolangRuntime) ProcessMessage( + appContext *appfunction.Context, + envelope types.MessageEnvelope, + pipeline *interfaces.FunctionPipeline) *MessageError { lc := appContext.LoggingClient() - if len(gr.transforms) == 0 { - err := errors.New("No transforms configured. Please check log for errors loading pipeline") + if len(pipeline.Transforms) == 0 { + err := fmt.Errorf("no transforms configured for pipleline Id='%s'. Please check log for earlier errors loading pipeline", pipeline.Id) logError(lc, err, envelope.CorrelationID) return &MessageError{Err: err, ErrorCode: http.StatusInternalServerError} } appContext.AddValue(interfaces.RECEIVEDTOPIC, envelope.ReceivedTopic) + appContext.AddValue(interfaces.PIPELINEID, pipeline.Id) - lc.Debugf("Processing message %d Transforms", len(gr.transforms)) + lc.Debugf("Pipeline '%s' processing message %d Transforms", pipeline.Id, len(pipeline.Transforms)) // Default Target Type for the function pipeline is an Event DTO. // The Event DTO can be wrapped in an AddEventRequest DTO or just be the un-wrapped Event DTO, @@ -103,11 +153,11 @@ func (gr *GolangRuntime) ProcessMessage(appContext *appfunction.Context, envelop switch target.(type) { case *[]byte: - lc.Debug("Pipeline is expecting raw byte data") + lc.Debug("Expecting raw byte data") target = &envelope.Payload case *dtos.Event: - lc.Debug("Pipeline is expecting an AddEventRequest or Event DTO") + lc.Debug("Expecting an AddEventRequest or Event DTO") // Dynamically process either AddEventRequest or Event DTO event, err := gr.processEventPayload(envelope, lc) @@ -135,7 +185,7 @@ func (gr *GolangRuntime) ProcessMessage(appContext *appfunction.Context, envelop default: customTypeName := di.TypeInstanceToName(target) - lc.Debugf("Pipeline is expecting a custom type of %s", customTypeName) + lc.Debugf("Expecting a custom type of %s", customTypeName) // Expecting a custom type so just unmarshal into the target type. if err := gr.unmarshalPayload(envelope, target); err != nil { @@ -153,25 +203,30 @@ func (gr *GolangRuntime) ProcessMessage(appContext *appfunction.Context, envelop // Make copy of transform functions to avoid disruption of pipeline when updating the pipeline from registry gr.isBusyCopying.Lock() - transforms := make([]interfaces.AppFunction, len(gr.transforms)) - copy(transforms, gr.transforms) + execPipeline := &interfaces.FunctionPipeline{ + Id: pipeline.Id, + Transforms: make([]interfaces.AppFunction, len(pipeline.Transforms)), + Topic: pipeline.Topic, + Hash: pipeline.Hash, + } + copy(execPipeline.Transforms, pipeline.Transforms) gr.isBusyCopying.Unlock() - return gr.ExecutePipeline(target, envelope.ContentType, appContext, transforms, 0, false) + return gr.ExecutePipeline(target, envelope.ContentType, appContext, execPipeline, 0, false) } func (gr *GolangRuntime) ExecutePipeline( target interface{}, contentType string, appContext *appfunction.Context, - transforms []interfaces.AppFunction, + pipeline *interfaces.FunctionPipeline, startPosition int, isRetry bool) *MessageError { var result interface{} var continuePipeline bool - for functionIndex, trxFunc := range transforms { + for functionIndex, trxFunc := range pipeline.Transforms { if functionIndex < startPosition { continue } @@ -188,11 +243,15 @@ func (gr *GolangRuntime) ExecutePipeline( if continuePipeline != true { if result != nil { if err, ok := result.(error); ok { - appContext.LoggingClient().Error( - fmt.Sprintf("Pipeline function #%d resulted in error", functionIndex), - "error", err.Error(), common.CorrelationHeader, appContext.CorrelationID()) + appContext.LoggingClient().Errorf( + "Pipeline (%s) function #%d resulted in error: %s (%s=%s)", + pipeline.Id, + functionIndex, + err.Error(), + common.CorrelationHeader, + appContext.CorrelationID()) if appContext.RetryData() != nil && !isRetry { - gr.storeForward.storeForLaterRetry(appContext.RetryData(), appContext, functionIndex) + gr.storeForward.storeForLaterRetry(appContext.RetryData(), appContext, pipeline, functionIndex) } return &MessageError{Err: err, ErrorCode: http.StatusUnprocessableEntity} @@ -304,6 +363,74 @@ func (gr *GolangRuntime) debugLogEvent(lc logger.LoggingClient, event *dtos.Even } } +func (gr *GolangRuntime) GetDefaultPipeline() *interfaces.FunctionPipeline { + pipeline := gr.pipelines[interfaces.DefaultPipelineId] + if pipeline == nil { + pipeline = &interfaces.FunctionPipeline{ + Id: interfaces.DefaultPipelineId, + } + } + return pipeline +} + +func (gr *GolangRuntime) GetMatchingPipelines(incomingTopic string) []*interfaces.FunctionPipeline { + var matches []*interfaces.FunctionPipeline + + if len(gr.pipelines) == 0 { + return matches + } + + for _, pipeline := range gr.pipelines { + if topicMatches(incomingTopic, pipeline.Topic) { + matches = append(matches, pipeline) + } + } + + return matches +} + +func (gr *GolangRuntime) GetPipelineById(id string) *interfaces.FunctionPipeline { + return gr.pipelines[id] +} + +func topicMatches(incomingTopic string, pipelineTopic string) bool { + if pipelineTopic == TopicWildCard { + return true + } + + wildcardCount := strings.Count(pipelineTopic, TopicWildCard) + switch wildcardCount { + case 0: + return incomingTopic == pipelineTopic + default: + pipelineLevels := strings.Split(pipelineTopic, TopicLevelSeparator) + incomingLevels := strings.Split(incomingTopic, TopicLevelSeparator) + + if len(pipelineLevels) > len(incomingLevels) { + return false + } + + for index, level := range pipelineLevels { + if level == TopicWildCard { + incomingLevels[index] = TopicWildCard + } + } + + incomingWithWildCards := strings.Join(incomingLevels, "/") + return strings.Index(incomingWithWildCards, pipelineTopic) == 0 + } +} + +func calculatePipelineHash(transforms []interfaces.AppFunction) string { + hash := "Pipeline-functions: " + for _, item := range transforms { + name := runtime.FuncForPC(reflect.ValueOf(item).Pointer()).Name() + hash = hash + " " + name + } + + return hash +} + func logError(lc logger.LoggingClient, err error, correlationID string) { lc.Errorf("%s. %s=%s", err.Error(), common.CorrelationHeader, correlationID) } diff --git a/internal/runtime/runtime_test.go b/internal/runtime/runtime_test.go index 525eb1460..bd76707c7 100644 --- a/internal/runtime/runtime_test.go +++ b/internal/runtime/runtime_test.go @@ -73,10 +73,10 @@ func TestProcessMessageBusRequest(t *testing.T) { return true, "Hello" } - runtime := GolangRuntime{} - runtime.Initialize(nil) - runtime.SetTransforms([]interfaces.AppFunction{dummyTransform}) - result := runtime.ProcessMessage(context, envelope) + runtime := NewGolangRuntime("", nil, nil) + err = runtime.SetDefaultFunctionsPipeline([]interfaces.AppFunction{dummyTransform}) + require.NoError(t, err) + result := runtime.ProcessMessage(context, envelope, runtime.GetDefaultPipeline()) require.NotNil(t, result) assert.Equal(t, expected, result.ErrorCode) } @@ -93,10 +93,9 @@ func TestProcessMessageNoTransforms(t *testing.T) { } context := appfunction.NewContext("testId", dic, "") - runtime := GolangRuntime{} - runtime.Initialize(nil) + runtime := NewGolangRuntime("", nil, nil) - result := runtime.ProcessMessage(context, envelope) + result := runtime.ProcessMessage(context, envelope, runtime.GetDefaultPipeline()) require.NotNil(t, result) assert.Equal(t, expected, result.ErrorCode) } @@ -123,10 +122,10 @@ func TestProcessMessageOneCustomTransform(t *testing.T) { transform1WasCalled = true return true, "Hello" } - runtime := GolangRuntime{} - runtime.Initialize(nil) - runtime.SetTransforms([]interfaces.AppFunction{transform1}) - result := runtime.ProcessMessage(context, envelope) + runtime := NewGolangRuntime("", nil, nil) + err = runtime.SetDefaultFunctionsPipeline([]interfaces.AppFunction{transform1}) + require.NoError(t, err) + result := runtime.ProcessMessage(context, envelope, runtime.GetDefaultPipeline()) require.Nil(t, result) require.True(t, transform1WasCalled, "transform1 should have been called") @@ -164,11 +163,11 @@ func TestProcessMessageTwoCustomTransforms(t *testing.T) { return true, "Hello" } - runtime := GolangRuntime{} - runtime.Initialize(nil) - runtime.SetTransforms([]interfaces.AppFunction{transform1, transform2}) + runtime := NewGolangRuntime("", nil, nil) + err = runtime.SetDefaultFunctionsPipeline([]interfaces.AppFunction{transform1, transform2}) + require.NoError(t, err) - result := runtime.ProcessMessage(context, envelope) + result := runtime.ProcessMessage(context, envelope, runtime.GetDefaultPipeline()) require.Nil(t, result) assert.True(t, transform1WasCalled, "transform1 should have been called") assert.True(t, transform2WasCalled, "transform2 should have been called") @@ -213,11 +212,11 @@ func TestProcessMessageThreeCustomTransformsOneFail(t *testing.T) { require.Equal(t, "Transform1Result", data, "Did not receive result from previous transform") return true, "Hello" } - runtime := GolangRuntime{} - runtime.Initialize(nil) - runtime.SetTransforms([]interfaces.AppFunction{transform1, transform2, transform3}) + runtime := NewGolangRuntime("", nil, nil) + err = runtime.SetDefaultFunctionsPipeline([]interfaces.AppFunction{transform1, transform2, transform3}) + require.NoError(t, err) - result := runtime.ProcessMessage(context, envelope) + result := runtime.ProcessMessage(context, envelope, runtime.GetDefaultPipeline()) require.Nil(t, result) assert.True(t, transform1WasCalled, "transform1 should have been called") assert.False(t, transform2WasCalled, "transform2 should NOT have been called") @@ -244,17 +243,17 @@ func TestProcessMessageTransformError(t *testing.T) { } context := appfunction.NewContext("testId", dic, "") - // Let the Runtime know we are sending a RegistryInfo so it passes it to the first function - runtime := GolangRuntime{TargetType: &config.RegistryInfo{}} - runtime.Initialize(nil) + // Let the Runtime know we are sending a RegistryInfo, so it passes it to the first function + runtime := NewGolangRuntime("", &config.RegistryInfo{}, nil) // FilterByDeviceName with return an error if it doesn't receive and Event - runtime.SetTransforms([]interfaces.AppFunction{transforms.NewFilterFor([]string{"SomeDevice"}).FilterByDeviceName}) - err := runtime.ProcessMessage(context, envelope) + err := runtime.SetDefaultFunctionsPipeline([]interfaces.AppFunction{transforms.NewFilterFor([]string{"SomeDevice"}).FilterByDeviceName}) + require.NoError(t, err) + msgErr := runtime.ProcessMessage(context, envelope, runtime.GetDefaultPipeline()) - require.NotNil(t, err, "Expected an error") - require.Error(t, err.Err, "Expected an error") - assert.Equal(t, expectedError, err.Err.Error()) - assert.Equal(t, expectedErrorCode, err.ErrorCode) + require.NotNil(t, msgErr, "Expected an error") + require.Error(t, msgErr.Err, "Expected an error") + assert.Contains(t, msgErr.Err.Error(), expectedError) + assert.Equal(t, expectedErrorCode, msgErr.ErrorCode) assertReceivedTopicSet(t, context, envelope) } @@ -311,11 +310,12 @@ func TestProcessMessageJSON(t *testing.T) { return false, nil } - runtime := GolangRuntime{} - runtime.Initialize(nil) - runtime.SetTransforms([]interfaces.AppFunction{transform1}) + runtime := NewGolangRuntime("", nil, nil) + + err = runtime.SetDefaultFunctionsPipeline([]interfaces.AppFunction{transform1}) + require.NoError(t, err) - result := runtime.ProcessMessage(context, envelope) + result := runtime.ProcessMessage(context, envelope, runtime.GetDefaultPipeline()) assert.Nilf(t, result, "result should be null. Got %v", result) assert.True(t, transform1WasCalled, "transform1 should have been called") } @@ -350,11 +350,12 @@ func TestProcessMessageCBOR(t *testing.T) { return false, nil } - runtime := GolangRuntime{} - runtime.Initialize(nil) - runtime.SetTransforms([]interfaces.AppFunction{transform1}) + runtime := NewGolangRuntime("", nil, nil) - result := runtime.ProcessMessage(context, envelope) + err = runtime.SetDefaultFunctionsPipeline([]interfaces.AppFunction{transform1}) + require.NoError(t, err) + + result := runtime.ProcessMessage(context, envelope, runtime.GetDefaultPipeline()) assert.Nil(t, result, "result should be null") assert.True(t, transform1WasCalled, "transform1 should have been called") } @@ -420,11 +421,12 @@ func TestProcessMessageTargetType(t *testing.T) { context := appfunction.NewContext("testing", dic, "") - runtime := GolangRuntime{TargetType: currentTest.TargetType} - runtime.Initialize(nil) - runtime.SetTransforms([]interfaces.AppFunction{transforms.NewResponseData().SetResponseData}) + runtime := NewGolangRuntime("", currentTest.TargetType, nil) + + err = runtime.SetDefaultFunctionsPipeline([]interfaces.AppFunction{transforms.NewResponseData().SetResponseData}) + require.NoError(t, err) - err := runtime.ProcessMessage(context, envelope) + err := runtime.ProcessMessage(context, envelope, runtime.GetDefaultPipeline()) if currentTest.ErrorExpected { assert.NotNil(t, err, fmt.Sprintf("expected an error for test '%s'", currentTest.Name)) assert.Error(t, err.Err, fmt.Sprintf("expected an error for test '%s'", currentTest.Name)) @@ -451,19 +453,21 @@ func TestExecutePipelinePersist(t *testing.T) { expectedItemCount := 1 context := appfunction.NewContext("testing", dic, "") - transformPassthru := func(appContext interfaces.AppFunctionContext, data interface{}) (bool, interface{}) { + transformPassThru := func(appContext interfaces.AppFunctionContext, data interface{}) (bool, interface{}) { return true, data } - runtime := GolangRuntime{ServiceKey: serviceKey} - runtime.Initialize(updateDicWithMockStoreClient()) + runtime := NewGolangRuntime(serviceKey, nil, updateDicWithMockStoreClient()) httpPost := transforms.NewHTTPSender("http://nowhere", "", true).HTTPPost - runtime.SetTransforms([]interfaces.AppFunction{transformPassthru, httpPost}) + err := runtime.SetDefaultFunctionsPipeline([]interfaces.AppFunction{transformPassThru, httpPost}) + require.NoError(t, err) + payload := []byte("My Payload") + pipeline := runtime.GetDefaultPipeline() // Target of this test - actual := runtime.ExecutePipeline(payload, "", context, runtime.transforms, 0, false) + actual := runtime.ExecutePipeline(payload, "", context, pipeline, 0, false) require.NotNil(t, actual) require.Error(t, actual.Err, "Error expected from export function") @@ -522,3 +526,141 @@ func TestGolangRuntime_processEventPayload(t *testing.T) { }) } } + +func TestTopicMatches(t *testing.T) { + incomingTopic := "edgex/events/P/D/S" + + tests := []struct { + name string + incomingTopic string + pipelineTopic string + expected bool + }{ + {"Match - Default all", incomingTopic, TopicWildCard, true}, + {"Match - Exact", incomingTopic, incomingTopic, true}, + {"Match - Any Profile for Device and Source", incomingTopic, "edgex/events/#/D/S", true}, + {"Match - Any Device for Profile and Source", incomingTopic, "edgex/events/P/#/S", true}, + {"Match - Any Source for Profile and Device", incomingTopic, "edgex/events/P/D/#", true}, + {"Match - All Events ", incomingTopic, "edgex/events/#", true}, + {"Match - All Devices and Sources for Profile ", incomingTopic, "edgex/events/P/#", true}, + {"Match - All Sources for Profile and Device ", incomingTopic, "edgex/events/P/D/#", true}, + {"Match - All Sources for a Device for any Profile ", incomingTopic, "edgex/events/#/D/#", true}, + {"Match - Source for any Profile and any Device ", incomingTopic, "edgex/events/#/#/S", true}, + {"NoMatch - SourceX for any Profile and any Device ", incomingTopic, "edgex/events/#/#/Sx", false}, + {"NoMatch - All Sources for DeviceX and any Profile ", incomingTopic, "edgex/events/#/Dx/#", false}, + {"NoMatch - All Sources for ProfileX and Device ", incomingTopic, "edgex/events/Px/D/#", false}, + {"NoMatch - All Sources for Profile and DeviceX ", incomingTopic, "edgex/events/P/Dx/#", false}, + {"NoMatch - All Sources for ProfileX and DeviceX ", incomingTopic, "edgex/events/Px/Dx/#", false}, + {"NoMatch - All Devices and Sources for ProfileX ", incomingTopic, "edgex/events/Px/#", false}, + {"NoMatch - Any Profile for DeviceX and Source", incomingTopic, "edgex/events/#/Dx/S", false}, + {"NoMatch - Any Profile for DeviceX and Source", incomingTopic, "edgex/events/#/Dx/S", false}, + {"NoMatch - Any Profile for Device and SourceX", incomingTopic, "edgex/events/#/D/Sx", false}, + {"NoMatch - Any Profile for DeviceX and SourceX", incomingTopic, "edgex/events/#/Dx/Sx", false}, + {"NoMatch - Any Device for Profile and SourceX", incomingTopic, "edgex/events/P/#/Sx", false}, + {"NoMatch - Any Device for ProfileX and Source", incomingTopic, "edgex/events/Px/#/S", false}, + {"NoMatch - Any Device for ProfileX and SourceX", incomingTopic, "edgex/events/Px/#/Sx", false}, + {"NoMatch - Any Source for ProfileX and Device", incomingTopic, "edgex/events/Px/D/#", false}, + {"NoMatch - Any Source for Profile and DeviceX", incomingTopic, "edgex/events/P/Dx/#", false}, + {"NoMatch - Any Source for ProfileX and DeviceX", incomingTopic, "edgex/events/Px/Dx/#", false}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + actual := topicMatches(test.incomingTopic, test.pipelineTopic) + assert.Equal(t, test.expected, actual) + }) + } +} + +func TestGetPipelineById(t *testing.T) { + target := NewGolangRuntime(serviceKey, nil, nil) + + expectedId := "my-pipeline" + expectedTopic := "edgex/events/#" + expectedTransforms := []interfaces.AppFunction{ + transforms.NewResponseData().SetResponseData, + } + badId := "bogus" + + err := target.SetDefaultFunctionsPipeline(expectedTransforms) + require.NoError(t, err) + + err = target.AddFunctionsPipeline(expectedId, expectedTopic, expectedTransforms) + require.NoError(t, err) + + actual := target.GetPipelineById(interfaces.DefaultPipelineId) + require.NotNil(t, actual) + assert.Equal(t, interfaces.DefaultPipelineId, actual.Id) + assert.Equal(t, TopicWildCard, actual.Topic) + assert.Equal(t, expectedTransforms, actual.Transforms) + assert.NotEmpty(t, actual.Hash) + + actual = target.GetPipelineById(expectedId) + require.NotNil(t, actual) + assert.Equal(t, expectedId, actual.Id) + assert.Equal(t, expectedTopic, actual.Topic) + assert.Equal(t, expectedTransforms, actual.Transforms) + assert.NotEmpty(t, actual.Hash) + + actual = target.GetPipelineById(badId) + require.Nil(t, actual) +} + +func TestGetMatchingPipelines(t *testing.T) { + target := NewGolangRuntime(serviceKey, nil, nil) + + expectedTransforms := []interfaces.AppFunction{ + transforms.NewResponseData().SetResponseData, + } + + err := target.AddFunctionsPipeline("one", "edgex/events/#/D1/#", expectedTransforms) + require.NoError(t, err) + err = target.AddFunctionsPipeline("two", "edgex/events/P1/#", expectedTransforms) + require.NoError(t, err) + err = target.AddFunctionsPipeline("three", "edgex/events/P1/D1/S1", expectedTransforms) + require.NoError(t, err) + + tests := []struct { + name string + incomingTopic string + expected int + }{ + {"Match 3", "edgex/events/P1/D1/S1", 3}, + {"Match 2", "edgex/events/P1/D1/S2", 2}, + {"Match 1", "edgex/events/P2/D1/S2", 1}, + {"Match 0", "edgex/events/P2/D2/S2", 0}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + actual := target.GetMatchingPipelines(test.incomingTopic) + assert.Equal(t, test.expected, len(actual)) + }) + } +} + +func TestGolangRuntime_GetDefaultPipeline(t *testing.T) { + target := NewGolangRuntime(serviceKey, nil, nil) + + expectedTransforms := []interfaces.AppFunction{ + transforms.NewResponseData().SetResponseData, + } + + // Returns dummy default pipeline with nil transforms if default never set. + actual := target.GetDefaultPipeline() + require.NotNil(t, actual) + assert.Equal(t, interfaces.DefaultPipelineId, actual.Id) + assert.Empty(t, actual.Topic) + assert.Nil(t, actual.Transforms) + assert.Empty(t, actual.Hash) + + err := target.SetDefaultFunctionsPipeline(expectedTransforms) + require.NoError(t, err) + + actual = target.GetDefaultPipeline() + require.NotNil(t, actual) + assert.Equal(t, interfaces.DefaultPipelineId, actual.Id) + assert.Equal(t, TopicWildCard, actual.Topic) + assert.Equal(t, expectedTransforms, actual.Transforms) + assert.NotEmpty(t, actual.Hash) +} diff --git a/internal/runtime/storeforward.go b/internal/runtime/storeforward.go index 004186b5e..3aba7e702 100644 --- a/internal/runtime/storeforward.go +++ b/internal/runtime/storeforward.go @@ -19,8 +19,6 @@ package runtime import ( "context" "fmt" - "reflect" - "runtime" "strings" "sync" "time" @@ -40,9 +38,8 @@ const ( ) type storeForwardInfo struct { - runtime *GolangRuntime - dic *di.Container - pipelineHash string + runtime *GolangRuntime + dic *di.Container } func (sf *storeForwardInfo) startStoreAndForwardRetryLoop( @@ -109,28 +106,27 @@ func (sf *storeForwardInfo) startStoreAndForwardRetryLoop( func (sf *storeForwardInfo) storeForLaterRetry( payload []byte, appContext interfaces.AppFunctionContext, + pipeline *interfaces.FunctionPipeline, pipelinePosition int) { - item := contracts.NewStoredObject(sf.runtime.ServiceKey, payload, pipelinePosition, sf.pipelineHash, appContext.GetAllValues()) + item := contracts.NewStoredObject(sf.runtime.ServiceKey, payload, pipeline.Id, pipelinePosition, pipeline.Hash, appContext.GetAllValues()) item.CorrelationID = appContext.CorrelationID() - appContext.LoggingClient().Trace("Storing data for later retry", - common.CorrelationHeader, appContext.CorrelationID()) + appContext.LoggingClient().Tracef("Storing data for later retry for pipeline '%s' (%s=%s)", + pipeline.Id, + common.CorrelationHeader, + appContext.CorrelationID()) config := container.ConfigurationFrom(sf.dic.Get) if !config.Writable.StoreAndForward.Enabled { - appContext.LoggingClient().Error( - "Failed to store item for later retry", "error", "StoreAndForward not enabled", - common.CorrelationHeader, item.CorrelationID) + appContext.LoggingClient().Errorf("Failed to store item for later retry for pipeline '%s': StoreAndForward not enabled", pipeline.Id) return } storeClient := container.StoreClientFrom(sf.dic.Get) if _, err := storeClient.Store(item); err != nil { - appContext.LoggingClient().Error("Failed to store item for later retry", - "error", err, - common.CorrelationHeader, item.CorrelationID) + appContext.LoggingClient().Errorf("Failed to store item for later retry for pipeline '%s': %s", pipeline.Id, err.Error()) } } @@ -141,36 +137,33 @@ func (sf *storeForwardInfo) retryStoredData(serviceKey string) { items, err := storeClient.RetrieveFromStore(serviceKey) if err != nil { - lc.Error("Unable to load store and forward items from DB", "error", err) + lc.Errorf("Unable to load store and forward items from DB: %s", err.Error()) return } - lc.Debugf(" %d stored data items found for retrying", len(items)) + lc.Debugf("%d stored data items found for retrying", len(items)) if len(items) > 0 { itemsToRemove, itemsToUpdate := sf.processRetryItems(items) - lc.Debug( - fmt.Sprintf(" %d stored data items will be removed post retry", len(itemsToRemove))) - lc.Debug( - fmt.Sprintf(" %d stored data items will be update post retry", len(itemsToUpdate))) + lc.Debugf(" %d stored data items will be removed post retry", len(itemsToRemove)) + lc.Debugf(" %d stored data items will be update post retry", len(itemsToUpdate)) for _, item := range itemsToRemove { if err := storeClient.RemoveFromStore(item); err != nil { - lc.Error( - "Unable to remove stored data item from DB", - "error", err, - "objectID", item.ID, - common.CorrelationHeader, item.CorrelationID) + lc.Errorf("Unable to remove stored data item for pipeline '%s' from DB, objectID=%s: %s", + item.PipelineId, + err.Error(), + item.ID) } } for _, item := range itemsToUpdate { if err := storeClient.Update(item); err != nil { - lc.Error("Unable to update stored data item in DB", - "error", err, - "objectID", item.ID, - common.CorrelationHeader, item.CorrelationID) + lc.Errorf("Unable to update stored data item for pipeline '%s' from DB, objectID=%s: %s", + item.PipelineId, + err.Error(), + item.ID) } } } @@ -183,75 +176,76 @@ func (sf *storeForwardInfo) processRetryItems(items []contracts.StoredObject) ([ var itemsToRemove []contracts.StoredObject var itemsToUpdate []contracts.StoredObject + // Item will be removed from store if: + // - successfully retried + // - max retries exceeded + // - version no longer matches current Pipeline + // Item will not be removed if retry failed and more retries available (hit 'continue' above) for _, item := range items { - if item.Version == sf.calculatePipelineHash() { - if !sf.retryExportFunction(item) { - item.RetryCount++ - if config.Writable.StoreAndForward.MaxRetryCount == 0 || - item.RetryCount < config.Writable.StoreAndForward.MaxRetryCount { - lc.Trace("Export retry failed. Incrementing retry count", - "retries", - item.RetryCount, - common.CorrelationHeader, - item.CorrelationID) - itemsToUpdate = append(itemsToUpdate, item) - continue - } - - lc.Trace( - "Max retries exceeded. Removing item from DB", "retries", + pipeline := sf.runtime.GetPipelineById(item.PipelineId) + + if pipeline == nil { + lc.Errorf("Stored data item's pipeline '%s' no longer exists. Removing item from DB", item.PipelineId) + itemsToRemove = append(itemsToRemove, item) + continue + } + + if item.Version != pipeline.Hash { + lc.Error("Stored data item's pipeline Version doesn't match '%s' pipeline's Version. Removing item from DB", item.PipelineId) + itemsToRemove = append(itemsToRemove, item) + continue + } + + if !sf.retryExportFunction(item, pipeline) { + item.RetryCount++ + if config.Writable.StoreAndForward.MaxRetryCount == 0 || + item.RetryCount < config.Writable.StoreAndForward.MaxRetryCount { + lc.Tracef("Export retry failed for pipeline '%s'. retries=%d, Incrementing retry count (%s=%s)", + item.PipelineId, item.RetryCount, common.CorrelationHeader, item.CorrelationID) - // Note that item will be removed for DB below. - } else { - lc.Trace( - "Export retry successful. Removing item from DB", - common.CorrelationHeader, - item.CorrelationID) + itemsToUpdate = append(itemsToUpdate, item) + continue } + + lc.Tracef("Max retries exceeded for pipeline '%s'. retries=%d, Removing item from DB (%s=%s)", + item.PipelineId, + item.RetryCount, + common.CorrelationHeader, + item.CorrelationID) + itemsToRemove = append(itemsToRemove, item) + + // Note that item will be removed for DB below. } else { - lc.Error( - "Stored data item's Function Pipeline Version doesn't match current Function Pipeline Version. Removing item from DB", + lc.Tracef("Retry successful for pipeline '%s'. Removing item from DB (%s=%s)", + item.PipelineId, common.CorrelationHeader, item.CorrelationID) + itemsToRemove = append(itemsToRemove, item) } - - // Item will be remove from store if: - // - successfully retried - // - max retries exceeded - // - version no longer matches current Pipeline - // Item will not be removed if retry failed and more retries available (hit 'continue' above) - itemsToRemove = append(itemsToRemove, item) } return itemsToRemove, itemsToUpdate } -func (sf *storeForwardInfo) retryExportFunction(item contracts.StoredObject) bool { +func (sf *storeForwardInfo) retryExportFunction(item contracts.StoredObject, pipeline *interfaces.FunctionPipeline) bool { appContext := appfunction.NewContext(item.CorrelationID, sf.dic, "") for k, v := range item.ContextData { appContext.AddValue(strings.ToLower(k), v) } - appContext.LoggingClient().Trace("Retrying stored data", common.CorrelationHeader, appContext.CorrelationID()) + appContext.LoggingClient().Tracef("Retrying stored data for pipeline '%s' (%s=%s)", + item.PipelineId, + common.CorrelationHeader, + appContext.CorrelationID()) return sf.runtime.ExecutePipeline( item.Payload, "", appContext, - sf.runtime.transforms, + pipeline, item.PipelinePosition, true) == nil } - -func (sf *storeForwardInfo) calculatePipelineHash() string { - hash := "Pipeline-functions: " - for _, item := range sf.runtime.transforms { - name := runtime.FuncForPC(reflect.ValueOf(item).Pointer()).Name() - hash = hash + " " + name - } - - return hash -} diff --git a/internal/runtime/storeforward_test.go b/internal/runtime/storeforward_test.go index 0f96d22b6..477aabbe5 100644 --- a/internal/runtime/storeforward_test.go +++ b/internal/runtime/storeforward_test.go @@ -86,7 +86,6 @@ func TestProcessRetryItems(t *testing.T) { require.Equal(t, contextData, appContext.GetAllValues()) return false, errors.New("I failed") } - runtime := GolangRuntime{} tests := []struct { Name string @@ -98,25 +97,42 @@ func TestProcessRetryItems(t *testing.T) { RemoveCount int BadVersion bool ContextData map[string]string + UsePerTopic bool }{ - {"Happy Path", successTransform, true, expectedPayload, 0, 0, 1, false, contextData}, - {"RetryCount Increased", failureTransform, true, expectedPayload, 4, 5, 0, false, contextData}, - {"Max Retries", failureTransform, true, expectedPayload, 9, 9, 1, false, contextData}, - {"Bad Version", successTransform, false, expectedPayload, 0, 0, 1, true, contextData}, + {"Happy Path - Default", successTransform, true, expectedPayload, 0, 0, 1, false, contextData, false}, + {"RetryCount Increased - Default", failureTransform, true, expectedPayload, 4, 5, 0, false, contextData, false}, + {"Max Retries - Default", failureTransform, true, expectedPayload, 9, 9, 1, false, contextData, false}, + {"Bad Version - Default", successTransform, false, expectedPayload, 0, 0, 1, true, contextData, false}, + {"Happy Path - Per Topic", successTransform, true, expectedPayload, 0, 0, 1, false, contextData, true}, + {"RetryCount Increased - Per Topic", failureTransform, true, expectedPayload, 4, 5, 0, false, contextData, true}, + {"Max Retries - Per Topic", failureTransform, true, expectedPayload, 9, 9, 1, false, contextData, true}, + {"Bad Version - Per Topic", successTransform, false, expectedPayload, 0, 0, 1, true, contextData, true}, } for _, test := range tests { t.Run(test.Name, func(t *testing.T) { targetTransformWasCalled = false + runtime := NewGolangRuntime(serviceKey, nil, dic) + + var pipeline *interfaces.FunctionPipeline + + if test.UsePerTopic { + err := runtime.AddFunctionsPipeline("per-topic", "#", []interfaces.AppFunction{transformPassthru, transformPassthru, test.TargetTransform}) + require.NoError(t, err) + pipeline = runtime.GetPipelineById("per-topic") + require.NotNil(t, pipeline) + } else { + err := runtime.SetDefaultFunctionsPipeline([]interfaces.AppFunction{transformPassthru, transformPassthru, test.TargetTransform}) + require.NoError(t, err) + pipeline = runtime.GetDefaultPipeline() + require.NotNil(t, pipeline) + } - runtime.Initialize(dic) - runtime.SetTransforms([]interfaces.AppFunction{transformPassthru, transformPassthru, test.TargetTransform}) - - version := runtime.storeForward.pipelineHash + version := pipeline.Hash if test.BadVersion { version = "some bad version" } - storedObject := contracts.NewStoredObject("dummy", []byte(test.ExpectedPayload), 2, version, contextData) + storedObject := contracts.NewStoredObject("dummy", []byte(test.ExpectedPayload), pipeline.Id, 2, version, contextData) storedObject.RetryCount = test.RetryCount removes, updates := runtime.storeForward.processRetryItems([]contracts.StoredObject{storedObject}) @@ -132,7 +148,6 @@ func TestProcessRetryItems(t *testing.T) { } func TestDoStoreAndForwardRetry(t *testing.T) { - serviceKey := "AppService-UnitTest" payload := []byte("My Payload") httpPost := transforms.NewHTTPSender("http://nowhere", "", true).HTTPPost @@ -149,19 +164,35 @@ func TestDoStoreAndForwardRetry(t *testing.T) { RetryCount int ExpectedRetryCount int ExpectedObjectCount int + UsePerTopic bool }{ - {"RetryCount Increased", httpPost, 1, 2, 1}, - {"Max Retries", httpPost, 9, 0, 0}, - {"Retry Success", successTransform, 1, 0, 0}, + {"RetryCount Increased - Default", httpPost, 1, 2, 1, false}, + {"Max Retries - Default", httpPost, 9, 0, 0, false}, + {"Retry Success - Default", successTransform, 1, 0, 0, false}, + {"RetryCount Increased - Per Topic", httpPost, 1, 2, 1, true}, + {"Max Retries - Per Topic", httpPost, 9, 0, 0, true}, + {"Retry Success - Per Topic", successTransform, 1, 0, 0, true}, } for _, test := range tests { t.Run(test.Name, func(t *testing.T) { - runtime := GolangRuntime{ServiceKey: serviceKey} - runtime.Initialize(updateDicWithMockStoreClient()) - runtime.SetTransforms([]interfaces.AppFunction{transformPassthru, test.TargetTransform}) + runtime := NewGolangRuntime(serviceKey, nil, updateDicWithMockStoreClient()) + + var pipeline *interfaces.FunctionPipeline + + if test.UsePerTopic { + err := runtime.AddFunctionsPipeline("per-topic", "#", []interfaces.AppFunction{transformPassthru, test.TargetTransform}) + require.NoError(t, err) + pipeline = runtime.GetPipelineById("per-topic") + require.NotNil(t, pipeline) + } else { + err := runtime.SetDefaultFunctionsPipeline([]interfaces.AppFunction{transformPassthru, test.TargetTransform}) + require.NoError(t, err) + pipeline = runtime.GetDefaultPipeline() + require.NotNil(t, pipeline) + } - object := contracts.NewStoredObject(serviceKey, payload, 1, runtime.storeForward.calculatePipelineHash(), nil) + object := contracts.NewStoredObject(serviceKey, payload, pipeline.Id, 1, pipeline.Hash, nil) object.CorrelationID = "CorrelationID" object.RetryCount = test.RetryCount diff --git a/internal/store/contracts/storedobject.go b/internal/store/contracts/storedobject.go index f4c2c11da..92b7f737c 100644 --- a/internal/store/contracts/storedobject.go +++ b/internal/store/contracts/storedobject.go @@ -25,36 +25,32 @@ import ( type StoredObject struct { // ID uniquely identifies this StoredObject ID string - // AppServiceKey identifies the app to which this data belongs. AppServiceKey string - // Payload is the data to be exported Payload []byte - // RetryCount is how many times this has tried to be exported RetryCount int - + // PipelineId is the ID of the pipeline that needs to be restarted. + PipelineId string // PipelinePosition is where to pickup in the pipeline PipelinePosition int - // Version is a hash of the functions to know if the pipeline has changed. Version string - // CorrelationID is an identifier provided by EdgeX to track this record as it moves CorrelationID string - // ContextData is a snapshot of data used by the pipeline at runtime ContextData map[string]string } // NewStoredObject creates a new instance of StoredObject and is the preferred way to create one. -func NewStoredObject(appServiceKey string, payload []byte, pipelinePosition int, +func NewStoredObject(appServiceKey string, payload []byte, pipelineId string, pipelinePosition int, version string, contextData map[string]string) StoredObject { return StoredObject{ AppServiceKey: appServiceKey, Payload: payload, RetryCount: 0, + PipelineId: pipelineId, PipelinePosition: pipelinePosition, Version: version, ContextData: contextData, diff --git a/internal/store/db/interfaces/mocks/StoreClient.go b/internal/store/db/interfaces/mocks/StoreClient.go index f31715220..0feb49f5d 100644 --- a/internal/store/db/interfaces/mocks/StoreClient.go +++ b/internal/store/db/interfaces/mocks/StoreClient.go @@ -2,9 +2,11 @@ package mocks -import contracts "github.com/edgexfoundry/app-functions-sdk-go/v2/internal/store/contracts" +import ( + contracts "github.com/edgexfoundry/app-functions-sdk-go/v2/internal/store/contracts" -import mock "github.com/stretchr/testify/mock" + mock "github.com/stretchr/testify/mock" +) // StoreClient is an autogenerated mock type for the StoreClient type type StoreClient struct { diff --git a/internal/store/db/redis/models/storedobject.go b/internal/store/db/redis/models/storedobject.go index 0c2e4abab..fb9989b88 100644 --- a/internal/store/db/redis/models/storedobject.go +++ b/internal/store/db/redis/models/storedobject.go @@ -25,25 +25,20 @@ import ( type StoredObject struct { // ID uniquely identifies this StoredObject ID string `json:"id"` - // AppServiceKey identifies the app to which this data belongs. AppServiceKey string `json:"appServiceKey"` - // Payload is the data to be exported Payload []byte `json:"payload"` - // RetryCount is how many times this has tried to be exported RetryCount int `json:"retryCount"` - + // PipelineId is the ID of the pipeline that needs to be restarted. + PipelineId string `json:"pipelineId"` // PipelinePosition is where to pickup in the pipeline PipelinePosition int `json:"pipelinePosition"` - // Version is a hash of the functions to know if the pipeline has changed. Version string `json:"version"` - // CorrelationID is an identifier provided by EdgeX to track this record as it moves CorrelationID string `json:"correlationID"` - // ContextData is a snapshot of data used by the pipeline at runtime ContextData map[string]string } @@ -55,6 +50,7 @@ func (o StoredObject) ToContract() contracts.StoredObject { AppServiceKey: o.AppServiceKey, Payload: o.Payload, RetryCount: o.RetryCount, + PipelineId: o.PipelineId, PipelinePosition: o.PipelinePosition, Version: o.Version, CorrelationID: o.CorrelationID, @@ -68,6 +64,7 @@ func (o *StoredObject) FromContract(c contracts.StoredObject) { o.AppServiceKey = c.AppServiceKey o.Payload = c.Payload o.RetryCount = c.RetryCount + o.PipelineId = c.PipelineId o.PipelinePosition = c.PipelinePosition o.Version = c.Version o.CorrelationID = c.CorrelationID @@ -81,6 +78,7 @@ func (o StoredObject) MarshalJSON() ([]byte, error) { AppServiceKey *string `json:"appServiceKey,omitempty"` Payload []byte `json:"payload,omitempty"` RetryCount int `json:"retryCount,omitempty"` + PipelineId string `json:"pipelineId,omitempty"` PipelinePosition int `json:"pipelinePosition,omitempty"` Version *string `json:"version,omitempty"` CorrelationID *string `json:"correlationID,omitempty"` @@ -90,6 +88,7 @@ func (o StoredObject) MarshalJSON() ([]byte, error) { }{ Payload: o.Payload, RetryCount: o.RetryCount, + PipelineId: o.PipelineId, PipelinePosition: o.PipelinePosition, ContextData: o.ContextData, } @@ -118,6 +117,7 @@ func (o *StoredObject) UnmarshalJSON(data []byte) error { AppServiceKey *string `json:"appServiceKey"` Payload []byte `json:"payload"` RetryCount int `json:"retryCount"` + PipelineId string `json:"pipelineId"` PipelinePosition int `json:"pipelinePosition"` Version *string `json:"version"` CorrelationID *string `json:"correlationID"` @@ -147,6 +147,7 @@ func (o *StoredObject) UnmarshalJSON(data []byte) error { o.Payload = alias.Payload o.RetryCount = alias.RetryCount + o.PipelineId = alias.PipelineId o.PipelinePosition = alias.PipelinePosition o.ContextData = alias.ContextData diff --git a/internal/store/db/redis/store_test.go b/internal/store/db/redis/store_test.go index aceefa6aa..5374bf1fc 100644 --- a/internal/store/db/redis/store_test.go +++ b/internal/store/db/redis/store_test.go @@ -23,9 +23,11 @@ package redis import ( "testing" + bootstrapConfig "github.com/edgexfoundry/go-mod-bootstrap/v2/config" + "github.com/stretchr/testify/require" + "github.com/edgexfoundry/app-functions-sdk-go/v2/internal/store/contracts" "github.com/edgexfoundry/app-functions-sdk-go/v2/internal/store/db" - "github.com/stretchr/testify/require" "github.com/google/uuid" ) @@ -33,15 +35,15 @@ import ( const ( TestHost = "localhost" TestPort = 6379 - TestTimeout = 5000 + TestTimeout = "5s" + TestMaxIdle = 5000 TestBatchSize = 1337 TestRetryCount = 100 TestPipelinePosition = 1337 TestVersion = "your" TestCorrelationID = "test" - TestEventID = "probably" - TestEventChecksum = "failed :(" + TestPipelineId = "test-pipeline" ) var TestPayload = []byte("brandon was here") @@ -51,18 +53,17 @@ var TestValidNoAuthConfig = db.DatabaseInfo{ Host: TestHost, Port: TestPort, Timeout: TestTimeout, - MaxIdle: TestTimeout, + MaxIdle: TestMaxIdle, BatchSize: TestBatchSize, } var TestContractBase = contracts.StoredObject{ Payload: TestPayload, RetryCount: TestRetryCount, + PipelineId: TestPipelineId, PipelinePosition: TestPipelinePosition, Version: TestVersion, CorrelationID: TestCorrelationID, - EventID: TestEventID, - EventChecksum: TestEventChecksum, } var TestContractBadID = contracts.StoredObject{ @@ -70,42 +71,38 @@ var TestContractBadID = contracts.StoredObject{ AppServiceKey: "brandon!", Payload: TestPayload, RetryCount: TestRetryCount, + PipelineId: TestPipelineId, PipelinePosition: TestPipelinePosition, Version: TestVersion, CorrelationID: TestCorrelationID, - EventID: TestEventID, - EventChecksum: TestEventChecksum, } var TestContractNoAppServiceKey = contracts.StoredObject{ ID: uuid.New().String(), Payload: TestPayload, RetryCount: TestRetryCount, + PipelineId: TestPipelineId, PipelinePosition: TestPipelinePosition, Version: TestVersion, CorrelationID: TestCorrelationID, - EventID: TestEventID, - EventChecksum: TestEventChecksum, } var TestContractNoPayload = contracts.StoredObject{ AppServiceKey: uuid.New().String(), RetryCount: TestRetryCount, + PipelineId: TestPipelineId, PipelinePosition: TestPipelinePosition, Version: TestVersion, CorrelationID: TestCorrelationID, - EventID: TestEventID, - EventChecksum: TestEventChecksum, } var TestContractNoVersion = contracts.StoredObject{ AppServiceKey: uuid.New().String(), Payload: TestPayload, RetryCount: TestRetryCount, + PipelineId: TestPipelineId, PipelinePosition: TestPipelinePosition, CorrelationID: TestCorrelationID, - EventID: TestEventID, - EventChecksum: TestEventChecksum, } func TestClient_NewClient(t *testing.T) { @@ -118,7 +115,7 @@ func TestClient_NewClient(t *testing.T) { } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - _, err := NewClient(test.config) + _, err := NewClient(test.config, bootstrapConfig.Credentials{}) if test.expectedError { require.Error(t, err) @@ -140,7 +137,7 @@ func TestClient_Store(t *testing.T) { TestContractNoAppServiceKey := TestContractBase TestContractUUID.ID = uuid.New().String() - client, _ := NewClient(TestValidNoAuthConfig) + client, _ := NewClient(TestValidNoAuthConfig, bootstrapConfig.Credentials{}) tests := []struct { name string @@ -220,7 +217,7 @@ func TestClient_RetrieveFromStore(t *testing.T) { UUIDContract1.ID = uuid.New().String() UUIDContract1.AppServiceKey = UUIDAppServiceKey - client, _ := NewClient(TestValidNoAuthConfig) + client, _ := NewClient(TestValidNoAuthConfig, bootstrapConfig.Credentials{}) tests := []struct { name string @@ -262,7 +259,7 @@ func TestClient_RetrieveFromStore(t *testing.T) { require.NoError(t, err) } - require.NotEqual(t, len(actual), len(test.toStore), "Returned slice length doesn't match expected") + require.Equal(t, len(actual), len(test.toStore), "Returned slice length doesn't match expected") }) } } @@ -272,7 +269,7 @@ func TestClient_Update(t *testing.T) { TestContractValid.AppServiceKey = uuid.New().String() TestContractValid.Version = uuid.New().String() - client, _ := NewClient(TestValidNoAuthConfig) + client, _ := NewClient(TestValidNoAuthConfig, bootstrapConfig.Credentials{}) // add the objects we're going to update in the database now so we have a known state TestContractValid.ID, _ = client.Store(TestContractValid) @@ -331,7 +328,7 @@ func TestClient_RemoveFromStore(t *testing.T) { TestContractValid := TestContractBase TestContractValid.AppServiceKey = uuid.New().String() - client, _ := NewClient(TestValidNoAuthConfig) + client, _ := NewClient(TestValidNoAuthConfig, bootstrapConfig.Credentials{}) // add the objects we're going to update in the database now so we have a known state TestContractValid.ID, _ = client.Store(TestContractValid) diff --git a/internal/trigger/http/rest.go b/internal/trigger/http/rest.go index 9bf36ec5c..0fbf017f6 100644 --- a/internal/trigger/http/rest.go +++ b/internal/trigger/http/rest.go @@ -20,11 +20,12 @@ import ( "context" "errors" "fmt" - "github.com/edgexfoundry/app-functions-sdk-go/v2/pkg/interfaces" "io" "net/http" "sync" + "github.com/edgexfoundry/app-functions-sdk-go/v2/pkg/interfaces" + "github.com/edgexfoundry/app-functions-sdk-go/v2/internal" "github.com/edgexfoundry/app-functions-sdk-go/v2/internal/appfunction" "github.com/edgexfoundry/app-functions-sdk-go/v2/internal/runtime" @@ -97,7 +98,7 @@ func (trigger *Trigger) requestHandler(writer http.ResponseWriter, r *http.Reque Payload: data, } - messageError := trigger.Runtime.ProcessMessage(appContext, envelope) + messageError := trigger.Runtime.ProcessMessage(appContext, envelope, trigger.Runtime.GetDefaultPipeline()) if messageError != nil { // ProcessMessage logs the error, so no need to log it here. writer.WriteHeader(messageError.ErrorCode) diff --git a/internal/trigger/http/rest_test.go b/internal/trigger/http/rest_test.go index a2f99e2d8..171150462 100644 --- a/internal/trigger/http/rest_test.go +++ b/internal/trigger/http/rest_test.go @@ -18,9 +18,10 @@ package http import ( - "github.com/edgexfoundry/app-functions-sdk-go/v2/pkg/interfaces" "testing" + "github.com/edgexfoundry/app-functions-sdk-go/v2/pkg/interfaces" + bootstrapContainer "github.com/edgexfoundry/go-mod-bootstrap/v2/bootstrap/container" "github.com/edgexfoundry/go-mod-bootstrap/v2/di" "github.com/edgexfoundry/go-mod-core-contracts/v2/clients/logger" diff --git a/internal/trigger/messagebus/messaging.go b/internal/trigger/messagebus/messaging.go index a96685e02..0bbdd4ed8 100644 --- a/internal/trigger/messagebus/messaging.go +++ b/internal/trigger/messagebus/messaging.go @@ -20,10 +20,11 @@ import ( "context" "errors" "fmt" - "github.com/edgexfoundry/app-functions-sdk-go/v2/pkg/interfaces" "strings" "sync" + "github.com/edgexfoundry/app-functions-sdk-go/v2/pkg/interfaces" + "github.com/edgexfoundry/app-functions-sdk-go/v2/internal/appfunction" "github.com/edgexfoundry/app-functions-sdk-go/v2/internal/bootstrap/container" sdkCommon "github.com/edgexfoundry/app-functions-sdk-go/v2/internal/common" @@ -109,7 +110,7 @@ func (trigger *Trigger) Initialize(appWg *sync.WaitGroup, appCtx context.Context config.Trigger.EdgexMessageBus.PublishHost.Port) } - // Need to have a go func for each subscription so we know with topic the data was received for. + // Need to have a go func for each subscription, so we know with topic the data was received for. for _, topic := range trigger.topics { appWg.Add(1) go func(triggerTopic types.TopicChannel) { @@ -121,8 +122,8 @@ func (trigger *Trigger) Initialize(appWg *sync.WaitGroup, appCtx context.Context case <-appCtx.Done(): lc.Infof("Exiting waiting for MessageBus '%s' topic messages", triggerTopic.Topic) return - case msgs := <-triggerTopic.Messages: - go trigger.processMessage(lc, triggerTopic, msgs) + case message := <-triggerTopic.Messages: + trigger.messageHandler(lc, triggerTopic, message) } } }(topic) @@ -173,13 +174,24 @@ func (trigger *Trigger) Initialize(appWg *sync.WaitGroup, appCtx context.Context return deferred, nil } -func (trigger *Trigger) processMessage(logger logger.LoggingClient, triggerTopic types.TopicChannel, message types.MessageEnvelope) { - logger.Debugf("Received message from MessageBus on topic '%s'. Content-Type=%s", triggerTopic.Topic, message.ContentType) - logger.Tracef("%s=%s", common.CorrelationHeader, message.CorrelationID) +func (trigger *Trigger) messageHandler(logger logger.LoggingClient, _ types.TopicChannel, message types.MessageEnvelope) { + logger.Debugf("MessageBus Trigger: Received message with %d bytes on topic '%s'. Content-Type=%s", + len(message.Payload), + message.ReceivedTopic, + message.ContentType) + logger.Tracef("MessageBus Trigger: Received message with %s=%s", common.CorrelationHeader, message.CorrelationID) + + pipelines := trigger.runtime.GetMatchingPipelines(message.ReceivedTopic) + logger.Debugf("MessageBus Trigger found %d pipeline(s) that match the incoming topic '%s'", len(pipelines), message.ReceivedTopic) + for _, pipeline := range pipelines { + go trigger.processMessageWithPipeline(logger, message, pipeline) + } +} +func (trigger *Trigger) processMessageWithPipeline(logger logger.LoggingClient, message types.MessageEnvelope, pipeline *interfaces.FunctionPipeline) { appContext := appfunction.NewContext(message.CorrelationID, trigger.dic, message.ContentType) - messageError := trigger.runtime.ProcessMessage(appContext, message) + messageError := trigger.runtime.ProcessMessage(appContext, message, pipeline) if messageError != nil { // ProcessMessage logs the error, so no need to log it here. return @@ -207,18 +219,27 @@ func (trigger *Trigger) processMessage(logger logger.LoggingClient, triggerTopic publishTopic, err := appContext.ApplyValues(config.Trigger.EdgexMessageBus.PublishHost.PublishTopic) if err != nil { - logger.Errorf("Unable to format output topic '%s': %s", config.Trigger.EdgexMessageBus.PublishHost.PublishTopic, err.Error()) + logger.Errorf("MessageBus Trigger: Unable to format output topic '%s' for pipeline '%s': %s", + config.Trigger.EdgexMessageBus.PublishHost.PublishTopic, + pipeline.Id, + err.Error()) return } err = trigger.client.Publish(outputEnvelope, publishTopic) if err != nil { - logger.Errorf("Failed to publish Message to bus, %v", err) + logger.Errorf("MessageBus trigger: Could not publish to topic '%s' for pipeline '%s': %s", + publishTopic, + pipeline.Id, + err.Error()) return } - logger.Debugf("Published message to bus on '%s' topic", publishTopic) - logger.Tracef("%s=%s", common.CorrelationHeader, message.CorrelationID) + logger.Debugf("MessageBus Trigger: Published response message for pipeline '%s' on topic '%s' with %d bytes", + pipeline.Id, + publishTopic, + len(appContext.ResponseData())) + logger.Tracef("MessageBus Trigger published message: %s=%s", common.CorrelationHeader, message.CorrelationID) } } @@ -258,11 +279,11 @@ func (trigger *Trigger) setOptionalAuthData(messageBusConfig *types.MessageBusCo secretData, err := bootstrapMessaging.GetSecretData(authMode, secretName, secretProvider) if err != nil { - return fmt.Errorf("Unable to get Secret Data for secure message bus: %w", err) + return fmt.Errorf("unable to get Secret Data for secure message bus: %w", err) } if err := bootstrapMessaging.ValidateSecretData(authMode, secretName, secretData); err != nil { - return fmt.Errorf("Secret Data for secure message bus invalid: %w", err) + return fmt.Errorf("secret Data for secure message bus invalid: %w", err) } if messageBusConfig.Optional == nil { diff --git a/internal/trigger/messagebus/messaging_test.go b/internal/trigger/messagebus/messaging_test.go index b079ff6e4..53ce483e2 100644 --- a/internal/trigger/messagebus/messaging_test.go +++ b/internal/trigger/messagebus/messaging_test.go @@ -204,6 +204,110 @@ func TestInitializeBadConfiguration(t *testing.T) { assert.Error(t, err) } +func TestPipelinePerTopic(t *testing.T) { + testClientConfig := types.MessageBusConfig{ + PublishHost: types.HostInfo{ + Host: "*", + Port: 6664, + Protocol: "tcp", + }, + Type: "zero", + } + + testClient, err := messaging.NewMessageClient(testClientConfig) + require.NoError(t, err, "Unable to create to publisher") + + config := sdkCommon.ConfigurationStruct{ + Trigger: sdkCommon.TriggerInfo{ + Type: TriggerTypeMessageBus, + EdgexMessageBus: sdkCommon.MessageBusConfig{ + Type: "zero", + PublishHost: sdkCommon.PublishHostInfo{ + Host: "*", + Port: 6666, + Protocol: "tcp", + PublishTopic: "", + }, + SubscribeHost: sdkCommon.SubscribeHostInfo{ + Host: "localhost", + Port: 6664, + Protocol: "tcp", + SubscribeTopics: "edgex/events/device", + }, + }, + }, + } + + dic.Update(di.ServiceConstructorMap{ + container.ConfigurationName: func(get di.Get) interface{} { + return &config + }, + }) + + expectedCorrelationID := "123" + + transform1WasCalled := make(chan bool, 1) + transform2WasCalled := make(chan bool, 1) + + transform1 := func(appContext interfaces.AppFunctionContext, data interface{}) (bool, interface{}) { + assert.Equal(t, expectedEvent, data) + transform1WasCalled <- true + return false, nil + } + + transform2 := func(appContext interfaces.AppFunctionContext, data interface{}) (bool, interface{}) { + assert.Equal(t, expectedEvent, data) + transform2WasCalled <- true + return false, nil + } + + goRuntime := runtime.NewGolangRuntime("", nil, dic) + + err = goRuntime.AddFunctionsPipeline("P1", "edgex/events/device/P1/#", []interfaces.AppFunction{transform1}) + require.NoError(t, err) + err = goRuntime.AddFunctionsPipeline("P2", "edgex/events/device/P2/#", []interfaces.AppFunction{transform2}) + require.NoError(t, err) + + trigger := NewTrigger(dic, goRuntime) + _, err = trigger.Initialize(&sync.WaitGroup{}, context.Background(), nil) + require.NoError(t, err) + + payload, err := json.Marshal(addEventRequest) + require.NoError(t, err) + + message := types.MessageEnvelope{ + CorrelationID: expectedCorrelationID, + Payload: payload, + ContentType: common.ContentTypeJSON, + } + + //transform1 in P1 pipeline should be called after this executes + err = testClient.Publish(message, "edgex/events/device/P1/LivingRoomThermostat/temperature") + require.NoError(t, err, "Failed to publish message") + + select { + case <-transform1WasCalled: + // do nothing, just need to fall out. + case <-transform2WasCalled: + t.Fail() // should not have happened + case <-time.After(3 * time.Second): + require.Fail(t, "Transform never called") + } + + //transform2 in P2 pipeline should be called after this executes + err = testClient.Publish(message, "edgex/events/device/P2/LivingRoomThermostat/temperature") + require.NoError(t, err, "Failed to publish message") + + select { + case <-transform1WasCalled: + t.Fail() // should not have happened + case <-transform2WasCalled: + // do nothing, just need to fall out. + case <-time.After(3 * time.Second): + require.Fail(t, "Transform never called") + } +} + func TestInitializeAndProcessEventWithNoOutput(t *testing.T) { config := sdkCommon.ConfigurationStruct{ @@ -243,11 +347,12 @@ func TestInitializeAndProcessEventWithNoOutput(t *testing.T) { return false, nil } - goRuntime := &runtime.GolangRuntime{} - goRuntime.Initialize(dic) - goRuntime.SetTransforms([]interfaces.AppFunction{transform1}) + goRuntime := runtime.NewGolangRuntime("", nil, dic) + + err := goRuntime.SetDefaultFunctionsPipeline([]interfaces.AppFunction{transform1}) + require.NoError(t, err) trigger := NewTrigger(dic, goRuntime) - _, err := trigger.Initialize(&sync.WaitGroup{}, context.Background(), nil) + _, err = trigger.Initialize(&sync.WaitGroup{}, context.Background(), nil) require.NoError(t, err) payload, err := json.Marshal(addEventRequest) @@ -326,9 +431,10 @@ func TestInitializeAndProcessEventWithOutput(t *testing.T) { } - goRuntime := &runtime.GolangRuntime{} - goRuntime.Initialize(dic) - goRuntime.SetTransforms([]interfaces.AppFunction{transform1}) + goRuntime := runtime.NewGolangRuntime("", nil, dic) + + err := goRuntime.SetDefaultFunctionsPipeline([]interfaces.AppFunction{transform1}) + require.NoError(t, err) trigger := NewTrigger(dic, goRuntime) testClientConfig := types.MessageBusConfig{ @@ -429,9 +535,11 @@ func TestInitializeAndProcessEventWithOutput_InferJSON(t *testing.T) { } - goRuntime := &runtime.GolangRuntime{} - goRuntime.Initialize(dic) - goRuntime.SetTransforms([]interfaces.AppFunction{transform1}) + goRuntime := runtime.NewGolangRuntime("", nil, dic) + + err := goRuntime.SetDefaultFunctionsPipeline([]interfaces.AppFunction{transform1}) + require.NoError(t, err) + trigger := NewTrigger(dic, goRuntime) testClientConfig := types.MessageBusConfig{ @@ -532,9 +640,11 @@ func TestInitializeAndProcessEventWithOutput_AssumeCBOR(t *testing.T) { return false, nil } - goRuntime := &runtime.GolangRuntime{} - goRuntime.Initialize(dic) - goRuntime.SetTransforms([]interfaces.AppFunction{transform1}) + goRuntime := runtime.NewGolangRuntime("", nil, dic) + + err := goRuntime.SetDefaultFunctionsPipeline([]interfaces.AppFunction{transform1}) + require.NoError(t, err) + trigger := NewTrigger(dic, goRuntime) testClientConfig := types.MessageBusConfig{ SubscribeHost: types.HostInfo{ @@ -626,8 +736,8 @@ func TestInitializeAndProcessBackgroundMessage(t *testing.T) { expectedPayload := []byte(`{"id":"5888dea1bd36573f4681d6f9","origin":1471806386919,"pushed":0,"device":"livingroomthermostat","readings":[{"id":"5888dea0bd36573f4681d6f8","created":1485364896983,"modified":1485364896983,"origin":1471806386919,"pushed":0,"name":"temperature","value":"38","device":"livingroomthermostat"}]}`) - goRuntime := &runtime.GolangRuntime{} - goRuntime.Initialize(dic) + goRuntime := runtime.NewGolangRuntime("", nil, dic) + trigger := NewTrigger(dic, goRuntime) testClientConfig := types.MessageBusConfig{ @@ -719,11 +829,13 @@ func TestInitializeAndProcessEventMultipleTopics(t *testing.T) { return false, nil } - goRuntime := &runtime.GolangRuntime{} - goRuntime.Initialize(dic) - goRuntime.SetTransforms([]interfaces.AppFunction{transform1}) + goRuntime := runtime.NewGolangRuntime("", nil, dic) + + err := goRuntime.SetDefaultFunctionsPipeline([]interfaces.AppFunction{transform1}) + require.NoError(t, err) + trigger := NewTrigger(dic, goRuntime) - _, err := trigger.Initialize(&sync.WaitGroup{}, context.Background(), nil) + _, err = trigger.Initialize(&sync.WaitGroup{}, context.Background(), nil) require.NoError(t, err) payload, _ := json.Marshal(addEventRequest) diff --git a/internal/trigger/mqtt/mqtt.go b/internal/trigger/mqtt/mqtt.go index c0df13ecf..88d10cc93 100644 --- a/internal/trigger/mqtt/mqtt.go +++ b/internal/trigger/mqtt/mqtt.go @@ -46,10 +46,13 @@ import ( // Trigger implements Trigger to support Triggers type Trigger struct { - dic *di.Container - lc logger.LoggingClient - mqttClient pahoMqtt.Client - runtime *runtime.GolangRuntime + dic *di.Container + lc logger.LoggingClient + mqttClient pahoMqtt.Client + runtime *runtime.GolangRuntime + qos byte + retain bool + publishTopic string } func NewTrigger(dic *di.Container, runtime *runtime.GolangRuntime) *Trigger { @@ -68,6 +71,10 @@ func (trigger *Trigger) Initialize(_ *sync.WaitGroup, _ context.Context, backgro brokerConfig := config.Trigger.ExternalMqtt topics := config.Trigger.ExternalMqtt.SubscribeTopics + trigger.qos = brokerConfig.QoS + trigger.retain = brokerConfig.Retain + trigger.publishTopic = config.Trigger.ExternalMqtt.PublishTopic + lc.Info("Initializing MQTT Trigger") if background != nil { @@ -75,7 +82,7 @@ func (trigger *Trigger) Initialize(_ *sync.WaitGroup, _ context.Context, backgro } if len(strings.TrimSpace(topics)) == 0 { - return nil, fmt.Errorf("missing SubscribeTopics for MQTT Trigger. Must be present in [Trigger.ExternalMqtt] section.") + return nil, fmt.Errorf("missing SubscribeTopics for MQTT Trigger. Must be present in [Trigger.ExternalMqtt] section") } brokerUrl, err := url.Parse(brokerConfig.Url) @@ -148,14 +155,11 @@ func (trigger *Trigger) onConnectHandler(mqttClient pahoMqtt.Client) { lc.Infof("Subscribed to topic(s) '%s' for MQTT trigger", config.Trigger.ExternalMqtt.SubscribeTopics) } -func (trigger *Trigger) messageHandler(client pahoMqtt.Client, message pahoMqtt.Message) { +func (trigger *Trigger) messageHandler(_ pahoMqtt.Client, mqttMessage pahoMqtt.Message) { // Convenience short cuts lc := trigger.lc - config := container.ConfigurationFrom(trigger.dic.Get) - brokerConfig := config.Trigger.ExternalMqtt - topic := config.Trigger.ExternalMqtt.PublishTopic - data := message.Payload() + data := mqttMessage.Payload() contentType := common.ContentTypeJSON if data[0] != byte('{') && data[0] != byte('[') { // If not JSON then assume it is CBOR @@ -164,37 +168,57 @@ func (trigger *Trigger) messageHandler(client pahoMqtt.Client, message pahoMqtt. correlationID := uuid.New().String() - appContext := appfunction.NewContext(correlationID, trigger.dic, contentType) - - lc.Debugf("Received message from MQTT Trigger with %d bytes from topic '%s'. Content-Type=%s", len(data), message.Topic(), contentType) - lc.Tracef("%s=%s", common.CorrelationHeader, correlationID) - - envelope := types.MessageEnvelope{ + message := types.MessageEnvelope{ CorrelationID: correlationID, ContentType: contentType, Payload: data, - ReceivedTopic: message.Topic(), + ReceivedTopic: mqttMessage.Topic(), } - messageError := trigger.runtime.ProcessMessage(appContext, envelope) + lc.Debugf("MQTT Trigger: Received message with %d bytes on topic '%s'. Content-Type=%s", + len(message.Payload), + message.ReceivedTopic, + message.ContentType) + lc.Tracef("%s=%s", common.CorrelationHeader, correlationID) + + pipelines := trigger.runtime.GetMatchingPipelines(message.ReceivedTopic) + lc.Debugf("MQTT Trigger found %d pipeline(s) that match the incoming topic '%s'", len(pipelines), message.ReceivedTopic) + for _, pipeline := range pipelines { + go trigger.processMessageWithPipeline(message, pipeline) + } +} + +func (trigger *Trigger) processMessageWithPipeline(envelope types.MessageEnvelope, pipeline *interfaces.FunctionPipeline) { + appContext := appfunction.NewContext(envelope.CorrelationID, trigger.dic, envelope.ContentType) + + messageError := trigger.runtime.ProcessMessage(appContext, envelope, pipeline) if messageError != nil { // ProcessMessage logs the error, so no need to log it here. // ToDo: Do we want to publish the error back to the Broker? return } - if len(appContext.ResponseData()) > 0 && len(topic) > 0 { - formattedTopic, err := appContext.ApplyValues(topic) + if len(appContext.ResponseData()) > 0 && len(trigger.publishTopic) > 0 { + formattedTopic, err := appContext.ApplyValues(trigger.publishTopic) if err != nil { - lc.Errorf("could not format topic '%s' for MQTT trigger output: %s", topic, err.Error()) + trigger.lc.Errorf("MQTT trigger: Unable to format topic '%s' for pipeline '%s': %s", + trigger.publishTopic, + pipeline.Id, + err.Error()) } - if token := client.Publish(formattedTopic, brokerConfig.QoS, brokerConfig.Retain, appContext.ResponseData()); token.Wait() && token.Error() != nil { - lc.Errorf("could not publish to topic '%s' for MQTT trigger: %s", topic, token.Error().Error()) + if token := trigger.mqttClient.Publish(formattedTopic, trigger.qos, trigger.retain, appContext.ResponseData()); token.Wait() && token.Error() != nil { + trigger.lc.Errorf("MQTT trigger: Could not publish to topic '%s' for pipeline '%s': %s", + formattedTopic, + pipeline.Id, + token.Error().Error()) } else { - lc.Trace("Sent MQTT Trigger response message", common.CorrelationHeader, correlationID) - lc.Debugf("Sent MQTT Trigger response message on topic '%s' with %d bytes", topic, len(appContext.ResponseData())) + trigger.lc.Debugf("MQTT Trigger: Published response message for pipeline '%s' on topic '%s' with %d bytes", + pipeline.Id, + formattedTopic, + len(appContext.ResponseData())) + trigger.lc.Tracef("MQTT Trigger published message: %s=%s", common.CorrelationHeader, envelope.CorrelationID) } } } diff --git a/pkg/interfaces/context.go b/pkg/interfaces/context.go index 01d13e5c5..039000349 100644 --- a/pkg/interfaces/context.go +++ b/pkg/interfaces/context.go @@ -24,10 +24,13 @@ import ( "github.com/edgexfoundry/go-mod-core-contracts/v2/dtos/common" ) -const DEVICENAME = "devicename" -const PROFILENAME = "profilename" -const SOURCENAME = "sourcename" -const RECEIVEDTOPIC = "receivedtopic" +const ( + DEVICENAME = "devicename" + PROFILENAME = "profilename" + SOURCENAME = "sourcename" + RECEIVEDTOPIC = "receivedtopic" + PIPELINEID = "pipelineid" +) // AppFunction is a type alias for a application pipeline function. // appCtx is a reference to the AppFunctionContext below. @@ -107,4 +110,6 @@ type AppFunctionContext interface { // the key in context storage. An error will be returned if any placeholders // are not matched to a value in the context. ApplyValues(format string) (string, error) + // PipelineId returns the ID of the pipeline that is executing + PipelineId() string } diff --git a/pkg/interfaces/mocks/AppFunctionContext.go b/pkg/interfaces/mocks/AppFunctionContext.go index 35b93d02f..2de80910f 100644 --- a/pkg/interfaces/mocks/AppFunctionContext.go +++ b/pkg/interfaces/mocks/AppFunctionContext.go @@ -274,6 +274,20 @@ func (_m *AppFunctionContext) NotificationClient() clientsinterfaces.Notificatio return r0 } +// PipelineId provides a mock function with given fields: +func (_m *AppFunctionContext) PipelineId() string { + ret := _m.Called() + + var r0 string + if rf, ok := ret.Get(0).(func() string); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(string) + } + + return r0 +} + // PushToCore provides a mock function with given fields: event func (_m *AppFunctionContext) PushToCore(event dtos.Event) (common.BaseWithIdResponse, error) { ret := _m.Called(event) diff --git a/pkg/interfaces/mocks/ApplicationService.go b/pkg/interfaces/mocks/ApplicationService.go index cefd1d621..ddd32a022 100644 --- a/pkg/interfaces/mocks/ApplicationService.go +++ b/pkg/interfaces/mocks/ApplicationService.go @@ -36,17 +36,15 @@ func (_m *ApplicationService) AddBackgroundPublisher(capacity int) (interfaces.B var r1 error if rf, ok := ret.Get(1).(func(int) error); ok { - r1 = rf(1) + r1 = rf(capacity) } else { - if ret.Get(0) != nil { - r1 = ret.Get(1).(error) - } + r1 = ret.Error(1) } return r0, r1 } -// AddBackgroundPublisher provides a mock function with given fields: capacity, topic +// AddBackgroundPublisherWithTopic provides a mock function with given fields: capacity, topic func (_m *ApplicationService) AddBackgroundPublisherWithTopic(capacity int, topic string) (interfaces.BackgroundPublisher, error) { ret := _m.Called(capacity, topic) @@ -63,25 +61,28 @@ func (_m *ApplicationService) AddBackgroundPublisherWithTopic(capacity int, topi if rf, ok := ret.Get(1).(func(int, string) error); ok { r1 = rf(capacity, topic) } else { - if ret.Get(0) != nil { - r1 = ret.Get(1).(error) - } + r1 = ret.Error(1) } return r0, r1 } -// AddBackgroundPublisher provides a mock function with given fields: correlationId, contentType -func (_m *ApplicationService) BuildContext(correlationId string, contentType string) interfaces.AppFunctionContext { - ret := _m.Called(correlationId, contentType) +// AddFunctionsPipelineForTopic provides a mock function with given fields: id, topic, transforms +func (_m *ApplicationService) AddFunctionsPipelineForTopic(id string, topic string, transforms ...func(interfaces.AppFunctionContext, interface{}) (bool, interface{})) error { + _va := make([]interface{}, len(transforms)) + for _i := range transforms { + _va[_i] = transforms[_i] + } + var _ca []interface{} + _ca = append(_ca, id, topic) + _ca = append(_ca, _va...) + ret := _m.Called(_ca...) - var r0 interfaces.AppFunctionContext - if rf, ok := ret.Get(0).(func(string, string) interfaces.AppFunctionContext); ok { - r0 = rf(correlationId, contentType) + var r0 error + if rf, ok := ret.Get(0).(func(string, string, ...func(interfaces.AppFunctionContext, interface{}) (bool, interface{})) error); ok { + r0 = rf(id, topic, transforms...) } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(interfaces.AppFunctionContext) - } + r0 = ret.Error(0) } return r0 @@ -124,6 +125,22 @@ func (_m *ApplicationService) ApplicationSettings() map[string]string { return r0 } +// BuildContext provides a mock function with given fields: correlationId, contentType +func (_m *ApplicationService) BuildContext(correlationId string, contentType string) interfaces.AppFunctionContext { + ret := _m.Called(correlationId, contentType) + + var r0 interfaces.AppFunctionContext + if rf, ok := ret.Get(0).(func(string, string) interfaces.AppFunctionContext); ok { + r0 = rf(correlationId, contentType) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(interfaces.AppFunctionContext) + } + } + + return r0 +} + // CommandClient provides a mock function with given fields: func (_m *ApplicationService) CommandClient() clientsinterfaces.CommandClient { ret := _m.Called() @@ -292,6 +309,29 @@ func (_m *ApplicationService) ListenForCustomConfigChanges(configToWatch interfa return r0 } +// LoadConfigurableFunctionPipelines provides a mock function with given fields: +func (_m *ApplicationService) LoadConfigurableFunctionPipelines() (map[string]interfaces.FunctionPipeline, error) { + ret := _m.Called() + + var r0 map[string]interfaces.FunctionPipeline + if rf, ok := ret.Get(0).(func() map[string]interfaces.FunctionPipeline); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(map[string]interfaces.FunctionPipeline) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func() error); ok { + r1 = rf() + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // LoadConfigurablePipeline provides a mock function with given fields: func (_m *ApplicationService) LoadConfigurablePipeline() ([]func(interfaces.AppFunctionContext, interface{}) (bool, interface{}), error) { ret := _m.Called() @@ -410,6 +450,26 @@ func (_m *ApplicationService) RegistryClient() registry.Client { return r0 } +// SetDefaultFunctionsPipeline provides a mock function with given fields: transforms +func (_m *ApplicationService) SetDefaultFunctionsPipeline(transforms ...func(interfaces.AppFunctionContext, interface{}) (bool, interface{})) error { + _va := make([]interface{}, len(transforms)) + for _i := range transforms { + _va[_i] = transforms[_i] + } + var _ca []interface{} + _ca = append(_ca, _va...) + ret := _m.Called(_ca...) + + var r0 error + if rf, ok := ret.Get(0).(func(...func(interfaces.AppFunctionContext, interface{}) (bool, interface{})) error); ok { + r0 = rf(transforms...) + } else { + r0 = ret.Error(0) + } + + return r0 +} + // SetFunctionsPipeline provides a mock function with given fields: transforms func (_m *ApplicationService) SetFunctionsPipeline(transforms ...func(interfaces.AppFunctionContext, interface{}) (bool, interface{})) error { _va := make([]interface{}, len(transforms)) diff --git a/pkg/interfaces/mocks/BackgroundPublisher.go b/pkg/interfaces/mocks/BackgroundPublisher.go index f99bb27e9..b12d3a9b8 100644 --- a/pkg/interfaces/mocks/BackgroundPublisher.go +++ b/pkg/interfaces/mocks/BackgroundPublisher.go @@ -2,14 +2,26 @@ package mocks -import mock "github.com/stretchr/testify/mock" +import ( + interfaces "github.com/edgexfoundry/app-functions-sdk-go/v2/pkg/interfaces" + mock "github.com/stretchr/testify/mock" +) // BackgroundPublisher is an autogenerated mock type for the BackgroundPublisher type type BackgroundPublisher struct { mock.Mock } -// Publish provides a mock function with given fields: payload, correlationID, contentType -func (_m *BackgroundPublisher) Publish(payload []byte, correlationID string, contentType string) { - _m.Called(payload, correlationID, contentType) +// Publish provides a mock function with given fields: payload, context +func (_m *BackgroundPublisher) Publish(payload []byte, context interfaces.AppFunctionContext) error { + ret := _m.Called(payload, context) + + var r0 error + if rf, ok := ret.Get(0).(func([]byte, interfaces.AppFunctionContext) error); ok { + r0 = rf(payload, context) + } else { + r0 = ret.Error(0) + } + + return r0 } diff --git a/pkg/interfaces/mocks/Trigger.go b/pkg/interfaces/mocks/Trigger.go index 362b82569..86c7545d2 100644 --- a/pkg/interfaces/mocks/Trigger.go +++ b/pkg/interfaces/mocks/Trigger.go @@ -7,11 +7,11 @@ import ( bootstrap "github.com/edgexfoundry/go-mod-bootstrap/v2/bootstrap" + interfaces "github.com/edgexfoundry/app-functions-sdk-go/v2/pkg/interfaces" + mock "github.com/stretchr/testify/mock" sync "sync" - - types "github.com/edgexfoundry/go-mod-messaging/v2/pkg/types" ) // Trigger is an autogenerated mock type for the Trigger type @@ -20,11 +20,11 @@ type Trigger struct { } // Initialize provides a mock function with given fields: wg, ctx, background -func (_m *Trigger) Initialize(wg *sync.WaitGroup, ctx context.Context, background <-chan types.MessageEnvelope) (bootstrap.Deferred, error) { +func (_m *Trigger) Initialize(wg *sync.WaitGroup, ctx context.Context, background <-chan interfaces.BackgroundMessage) (bootstrap.Deferred, error) { ret := _m.Called(wg, ctx, background) var r0 bootstrap.Deferred - if rf, ok := ret.Get(0).(func(*sync.WaitGroup, context.Context, <-chan types.MessageEnvelope) bootstrap.Deferred); ok { + if rf, ok := ret.Get(0).(func(*sync.WaitGroup, context.Context, <-chan interfaces.BackgroundMessage) bootstrap.Deferred); ok { r0 = rf(wg, ctx, background) } else { if ret.Get(0) != nil { @@ -33,7 +33,7 @@ func (_m *Trigger) Initialize(wg *sync.WaitGroup, ctx context.Context, backgroun } var r1 error - if rf, ok := ret.Get(1).(func(*sync.WaitGroup, context.Context, <-chan types.MessageEnvelope) error); ok { + if rf, ok := ret.Get(1).(func(*sync.WaitGroup, context.Context, <-chan interfaces.BackgroundMessage) error); ok { r1 = rf(wg, ctx, background) } else { r1 = ret.Error(1) diff --git a/pkg/interfaces/service.go b/pkg/interfaces/service.go index 872929a4e..625c61b75 100644 --- a/pkg/interfaces/service.go +++ b/pkg/interfaces/service.go @@ -37,8 +37,23 @@ const ( // serviceKey = "MyServiceName-" + interfaces.ProfileSuffixPlaceholder // ) ProfileSuffixPlaceholder = "" + + // DefaultPipelineId is the ID used for the default pipeline create by SetFunctionsPipeline + DefaultPipelineId = "default-pipeline" ) +// FunctionPipeline defines an instance of a Functions Pipeline +type FunctionPipeline struct { + // Unique identifier for the pipeline. + Id string + // Collection of App Functions to execute + Transforms []AppFunction + // Topic to match against the incoming topic to determine if the pipeline will execute on the incoming message + Topic string + // Hash of the list of transforms set and used internally for Store and Forward + Hash string +} + // UpdatableConfig interface allows services to have custom configuration populated from configuration stored // in the Configuration Provider (aka Consul). Services using custom configuration must implement this interface // on their custom configuration, even if they do not use Configuration Provider. If they do not use the @@ -65,10 +80,19 @@ type ApplicationService interface { // application setting as a comma separated list. It returns the list of strings. // An error is returned if the specified setting is not found. GetAppSettingStrings(setting string) ([]string, error) - // SetFunctionsPipeline set the functions pipeline with the specified list of Application Functions. + // SetFunctionsPipeline has been deprecated (Replaced by SetDefaultFunctionsPipeline) and will be removed in a future release + // Functions the same as SetDefaultFunctionsPipeline. + SetFunctionsPipeline(transforms ...AppFunction) error + // SetDefaultFunctionsPipeline sets the default functions pipeline with the specified list of Application Functions. + // This pipeline is executed for all message received from the configured trigger. // Note that the functions are executed in the order provided in the list. // An error is returned if the list is empty. - SetFunctionsPipeline(transforms ...AppFunction) error + SetDefaultFunctionsPipeline(transforms ...AppFunction) error + // AddFunctionsPipelineForTopic adds a functions pipeline with the specified unique id and list of Application Functions + // to be executed when the incoming topic matches the specified topic. The specified topic may contain the '#' wildcard + // so that it matches multiple incoming topics. If just "#" is used for the specified topic it will match all incoming + // topics and the specified functions pipeline will execute on every message received. + AddFunctionsPipelineForTopic(id string, topic string, transforms ...AppFunction) error // MakeItRun starts the configured trigger to allow the functions pipeline to execute when the trigger // receives data and starts the internal webserver. This is a long running function which does not return until // the service is stopped or MakeItStop() is called. @@ -124,11 +148,17 @@ type ApplicationService interface { // RegistryClient returns the Registry client. Note the registry must been enable, otherwise this will return nil. // Useful if service needs to add additional health checks or needs to get endpoint of another registered service RegistryClient() registry.Client - // LoadConfigurablePipeline loads the function pipeline from configuration. + // LoadConfigurablePipeline loads the default function pipeline from configuration. // An error is returned if the configuration is not valid, i.e. missing required function parameters, // invalid function name, etc. // Only useful if pipeline from configuration is always defined in configuration as in App Service Configurable. + // Note this API is deprecated, replaced by LoadConfigurableFunctionPipelines and will be removed in a future release LoadConfigurablePipeline() ([]AppFunction, error) + // LoadConfigurableFunctionPipelines loads the function pipelines (default and per topic) from configuration. + // An error is returned if the configuration is not valid, i.e. missing required function parameters, + // invalid function name, etc. + // Only useful if pipeline is always defined in configuration as is with App Service Configurable. + LoadConfigurableFunctionPipelines() (map[string]FunctionPipeline, error) // LoadCustomConfig loads the service's custom configuration from local file or the Configuration Provider (if enabled) // Configuration Provider will also be seeded with the custom configuration if service is using the Configuration Provider. // UpdateFromRaw interface will be called on the custom configuration when the configuration is loaded from the diff --git a/pkg/transforms/batch.go b/pkg/transforms/batch.go index e2dd4cf19..95f8262d7 100644 --- a/pkg/transforms/batch.go +++ b/pkg/transforms/batch.go @@ -17,7 +17,7 @@ package transforms import ( - "errors" + "fmt" "sync" "time" @@ -127,10 +127,10 @@ func NewBatchByTimeAndCount(timeInterval string, batchThreshold int) (*BatchConf func (batch *BatchConfig) Batch(ctx interfaces.AppFunctionContext, data interface{}) (bool, interface{}) { if data == nil { // We didn't receive a result - return false, errors.New("No Data Received") + return false, fmt.Errorf("function Batch in pipeline '%s': No Data Received", ctx.PipelineId()) } - ctx.LoggingClient().Debug("Batching Data") + ctx.LoggingClient().Debugf("Batching Data in pipeline '%s'", ctx.PipelineId()) byteData, err := util.CoerceType(data) if err != nil { return false, err @@ -144,9 +144,9 @@ func (batch *BatchConfig) Batch(ctx interfaces.AppFunctionContext, data interfac batch.timerActive.Set(true) select { case <-batch.done: - ctx.LoggingClient().Debug("Batch count has been reached") + ctx.LoggingClient().Debugf("Batch count has been reached in pipeline '%s'", ctx.PipelineId()) case <-time.After(batch.parsedDuration): - ctx.LoggingClient().Debug("Timer has elapsed") + ctx.LoggingClient().Debugf("Timer has elapsed in pipeline '%s'", ctx.PipelineId()) } batch.timerActive.Set(false) } else { @@ -171,7 +171,7 @@ func (batch *BatchConfig) Batch(ctx interfaces.AppFunctionContext, data interfac } } - ctx.LoggingClient().Debug("Forwarding Batched Data...") + ctx.LoggingClient().Debugf("Forwarding Batched Data in pipeline '%s'", ctx.PipelineId()) // we've met the threshold, lets clear out the buffer and send it forward in the pipeline if batch.batchData.length() > 0 { copyOfData := batch.batchData.all() diff --git a/pkg/transforms/batch_test.go b/pkg/transforms/batch_test.go index 862df93d5..07a47ef97 100644 --- a/pkg/transforms/batch_test.go +++ b/pkg/transforms/batch_test.go @@ -31,7 +31,7 @@ func TestBatchNoData(t *testing.T) { bs, _ := NewBatchByCount(1) continuePipeline, err := bs.Batch(ctx, nil) assert.False(t, continuePipeline) - assert.Equal(t, "No Data Received", err.(error).Error()) + assert.Contains(t, err.(error).Error(), "No Data Received") } func TestBatchInCountMode(t *testing.T) { diff --git a/pkg/transforms/compression.go b/pkg/transforms/compression.go index 9273492fd..85a2bd94b 100644 --- a/pkg/transforms/compression.go +++ b/pkg/transforms/compression.go @@ -22,9 +22,10 @@ import ( "compress/gzip" "compress/zlib" "encoding/base64" - "errors" "fmt" + "github.com/edgexfoundry/go-mod-core-contracts/v2/clients/logger" + "github.com/edgexfoundry/app-functions-sdk-go/v2/pkg/interfaces" "github.com/edgexfoundry/app-functions-sdk-go/v2/pkg/util" @@ -46,9 +47,9 @@ func NewCompression() Compression { func (compression *Compression) CompressWithGZIP(ctx interfaces.AppFunctionContext, data interface{}) (bool, interface{}) { if data == nil { // We didn't receive a result - return false, errors.New("No Data Received") + return false, fmt.Errorf("function CompressWithGZIP in pipeline '%s': No Data Received", ctx.PipelineId()) } - ctx.LoggingClient().Debug("Compression with GZIP") + ctx.LoggingClient().Debugf("Compression with GZIP in pipeline '%s'", ctx.PipelineId()) rawData, err := util.CoerceType(data) if err != nil { return false, err @@ -63,18 +64,18 @@ func (compression *Compression) CompressWithGZIP(ctx interfaces.AppFunctionConte _, err = compression.gzipWriter.Write(rawData) if err != nil { - return false, fmt.Errorf("unable to write GZIP data: %s", err.Error()) + return false, fmt.Errorf("unable to write GZIP data in pipeline '%s': %s", ctx.PipelineId(), err.Error()) } err = compression.gzipWriter.Close() if err != nil { - return false, fmt.Errorf("unable to close GZIP data: %s", err.Error()) + return false, fmt.Errorf("unable to close GZIP data in pipeline '%s': %s", ctx.PipelineId(), err.Error()) } // Set response "content-type" header to "text/plain" ctx.SetResponseContentType(common.ContentTypeText) - return true, bytesBufferToBase64(buf) + return true, bytesBufferToBase64(ctx.LoggingClient(), buf) } @@ -83,9 +84,9 @@ func (compression *Compression) CompressWithGZIP(ctx interfaces.AppFunctionConte func (compression *Compression) CompressWithZLIB(ctx interfaces.AppFunctionContext, data interface{}) (bool, interface{}) { if data == nil { // We didn't receive a result - return false, errors.New("No Data Received") + return false, fmt.Errorf("function CompressWithZLIB in pipeline '%s': No Data Received", ctx.PipelineId()) } - ctx.LoggingClient().Debug("Compression with ZLIB") + ctx.LoggingClient().Debugf("Compression with ZLIB in pipeline '%s'", ctx.PipelineId()) byteData, err := util.CoerceType(data) if err != nil { return false, err @@ -100,23 +101,26 @@ func (compression *Compression) CompressWithZLIB(ctx interfaces.AppFunctionConte _, err = compression.zlibWriter.Write(byteData) if err != nil { - return false, fmt.Errorf("unable to write ZLIB data: %s", err.Error()) + return false, fmt.Errorf("unable to write ZLIB data in pipeline '%s': %s", ctx.PipelineId(), err.Error()) } err = compression.zlibWriter.Close() if err != nil { - return false, fmt.Errorf("unable to close ZLIB data: %s", err.Error()) + return false, fmt.Errorf("unable to close ZLIB data in pipeline '%s': %s", ctx.PipelineId(), err.Error()) } // Set response "content-type" header to "text/plain" ctx.SetResponseContentType(common.ContentTypeText) - return true, bytesBufferToBase64(buf) + return true, bytesBufferToBase64(ctx.LoggingClient(), buf) } -func bytesBufferToBase64(buf bytes.Buffer) []byte { +func bytesBufferToBase64(lc logger.LoggingClient, buf bytes.Buffer) []byte { + lc.Debugf("Encoding compressed bytes of length %d vs %d", len(buf.Bytes()), buf.Len()) dst := make([]byte, base64.StdEncoding.EncodedLen(buf.Len())) base64.StdEncoding.Encode(dst, buf.Bytes()) + lc.Debugf("Encoding compressed bytes complete. New length is %d", len(dst)) + return dst } diff --git a/pkg/transforms/conversion.go b/pkg/transforms/conversion.go index 39916cbd5..41d911184 100755 --- a/pkg/transforms/conversion.go +++ b/pkg/transforms/conversion.go @@ -18,7 +18,6 @@ package transforms import ( "encoding/json" - "errors" "fmt" "github.com/edgexfoundry/app-functions-sdk-go/v2/pkg/interfaces" @@ -40,39 +39,43 @@ func NewConversion() Conversion { // It will return an error and stop the pipeline if a non-edgex event is received or if no data is received. func (f Conversion) TransformToXML(ctx interfaces.AppFunctionContext, data interface{}) (continuePipeline bool, stringType interface{}) { if data == nil { - return false, errors.New("No Event Received") + return false, fmt.Errorf("function TransformToXML in pipeline '%s': No Data Received", ctx.PipelineId()) } - ctx.LoggingClient().Debug("Transforming to XML") + ctx.LoggingClient().Debugf("Transforming to XML in pipeline '%s'", ctx.PipelineId()) + if event, ok := data.(dtos.Event); ok { xml, err := event.ToXML() if err != nil { - return false, fmt.Errorf("unable to marshal Event to XML: %s", err.Error()) + return false, fmt.Errorf("unable to marshal Event to XML in pipeline '%s': %s", ctx.PipelineId(), err.Error()) } ctx.SetResponseContentType(common.ContentTypeXML) return true, xml } - return false, errors.New("Unexpected type received") + return false, fmt.Errorf("function TransformToXML in pipeline '%s': unexpected type received", ctx.PipelineId()) } // TransformToJSON transforms an EdgeX event to JSON. // It will return an error and stop the pipeline if a non-edgex event is received or if no data is received. func (f Conversion) TransformToJSON(ctx interfaces.AppFunctionContext, data interface{}) (continuePipeline bool, stringType interface{}) { if data == nil { - return false, errors.New("No Event Received") + return false, fmt.Errorf("function TransformToJSON in pipeline '%s': No Data Received", ctx.PipelineId()) } - ctx.LoggingClient().Debug("Transforming to JSON") + + ctx.LoggingClient().Debugf("Transforming to JSON in pipeline '%s'", ctx.PipelineId()) + if result, ok := data.(dtos.Event); ok { b, err := json.Marshal(result) if err != nil { - return false, errors.New("Error marshalling JSON") + return false, fmt.Errorf("unable to marshal Event to JSON in pipeline '%s': %s", ctx.PipelineId(), err.Error()) } ctx.SetResponseContentType(common.ContentTypeJSON) // should we return a byte[] or string? // return b return true, string(b) } - return false, errors.New("Unexpected type received") + + return false, fmt.Errorf("function TransformToJSON in pipeline '%s': unexpected type received", ctx.PipelineId()) } diff --git a/pkg/transforms/conversion_test.go b/pkg/transforms/conversion_test.go index 5f87ca57d..05bcf53b6 100644 --- a/pkg/transforms/conversion_test.go +++ b/pkg/transforms/conversion_test.go @@ -45,7 +45,7 @@ func TestTransformToXMLNoData(t *testing.T) { conv := NewConversion() continuePipeline, result := conv.TransformToXML(ctx, nil) - assert.Equal(t, "No Event Received", result.(error).Error()) + assert.Contains(t, result.(error).Error(), "No Data Received") assert.False(t, continuePipeline) } @@ -53,7 +53,7 @@ func TestTransformToXMLNotAnEvent(t *testing.T) { conv := NewConversion() continuePipeline, result := conv.TransformToXML(ctx, "") - assert.Equal(t, "Unexpected type received", result.(error).Error()) + assert.Contains(t, result.(error).Error(), "unexpected type received") assert.False(t, continuePipeline) } @@ -77,7 +77,7 @@ func TestTransformToJSONNoEvent(t *testing.T) { conv := NewConversion() continuePipeline, result := conv.TransformToJSON(ctx, nil) - assert.Equal(t, "No Event Received", result.(error).Error()) + assert.Contains(t, result.(error).Error(), "No Data Received") assert.False(t, continuePipeline) } @@ -85,6 +85,6 @@ func TestTransformToJSONNoEvent(t *testing.T) { func TestTransformToJSONNotAnEvent(t *testing.T) { conv := NewConversion() continuePipeline, result := conv.TransformToJSON(ctx, "") - require.EqualError(t, result.(error), "Unexpected type received") + require.Contains(t, result.(error).Error(), "unexpected type received") assert.False(t, continuePipeline) } diff --git a/pkg/transforms/coredata.go b/pkg/transforms/coredata.go index a5ad0d6a8..d0cd10c0f 100644 --- a/pkg/transforms/coredata.go +++ b/pkg/transforms/coredata.go @@ -18,7 +18,7 @@ package transforms import ( "context" - "errors" + "fmt" "github.com/edgexfoundry/app-functions-sdk-go/v2/pkg/interfaces" "github.com/edgexfoundry/app-functions-sdk-go/v2/pkg/util" @@ -65,12 +65,12 @@ func (cdc *CoreData) PushToCoreData(ctx interfaces.AppFunctionContext, data inte ctx.LoggingClient().Info("Pushing To CoreData...") if data == nil { - return false, errors.New("PushToCoreData - No Data Received") + return false, fmt.Errorf("function PushToCoreData in pipeline '%s': No Data Received", ctx.PipelineId()) } client := ctx.EventClient() if client == nil { - return false, errors.New("EventClient not initialized. Core Data is missing from clients configuration") + return false, fmt.Errorf("function PushToCoreData in pipeline '%s': EventClient not initialized. Core Data is missing from clients configuration", ctx.PipelineId()) } event := dtos.NewEvent(cdc.profileName, cdc.deviceName, cdc.resourceName) @@ -85,9 +85,15 @@ func (cdc *CoreData) PushToCoreData(ctx interfaces.AppFunctionContext, data inte if err != nil { return false, err } - event.AddSimpleReading(cdc.resourceName, cdc.valueType, string(reading)) + err = event.AddSimpleReading(cdc.resourceName, cdc.valueType, string(reading)) + if err != nil { + return false, fmt.Errorf("error adding Reading in pipeline '%s': %s", ctx.PipelineId(), err.Error()) + } } else { - event.AddSimpleReading(cdc.resourceName, cdc.valueType, data) + err := event.AddSimpleReading(cdc.resourceName, cdc.valueType, data) + if err != nil { + return false, fmt.Errorf("error adding Reading in pipeline '%s': %s", ctx.PipelineId(), err.Error()) + } } request := requests.NewAddEventRequest(event) diff --git a/pkg/transforms/coredata_test.go b/pkg/transforms/coredata_test.go index fe248f1e7..ef07bb1fa 100644 --- a/pkg/transforms/coredata_test.go +++ b/pkg/transforms/coredata_test.go @@ -36,6 +36,6 @@ func TestPushToCore_NoData(t *testing.T) { continuePipeline, result := coreData.PushToCoreData(ctx, nil) assert.NotNil(t, result) - assert.Equal(t, "PushToCoreData - No Data Received", result.(error).Error()) + assert.Contains(t, result.(error).Error(), "No Data Received") assert.False(t, continuePipeline) } diff --git a/pkg/transforms/encryption.go b/pkg/transforms/encryption.go index 0ee7be4ea..5e08b7454 100644 --- a/pkg/transforms/encryption.go +++ b/pkg/transforms/encryption.go @@ -23,7 +23,6 @@ import ( "crypto/cipher" "crypto/sha1" "encoding/base64" - "errors" "fmt" "github.com/edgexfoundry/app-functions-sdk-go/v2/pkg/interfaces" @@ -62,18 +61,18 @@ const blockSize = 16 func pkcs5Padding(ciphertext []byte, blockSize int) []byte { padding := blockSize - len(ciphertext)%blockSize - padtext := bytes.Repeat([]byte{byte(padding)}, padding) - return append(ciphertext, padtext...) + padText := bytes.Repeat([]byte{byte(padding)}, padding) + return append(ciphertext, padText...) } // EncryptWithAES encrypts a string, []byte, or json.Marshaller type using AES encryption. // It will return a Base64 encode []byte of the encrypted data. func (aesData Encryption) EncryptWithAES(ctx interfaces.AppFunctionContext, data interface{}) (bool, interface{}) { if data == nil { - return false, errors.New("no data received to encrypt") + return false, fmt.Errorf("function EncryptWithAES in pipeline '%s': No Data Received", ctx.PipelineId()) } - ctx.LoggingClient().Debug("Encrypting with AES") + ctx.LoggingClient().Debugf("Encrypting with AES in pipeline '%s'", ctx.PipelineId()) byteData, err := util.CoerceType(data) if err != nil { @@ -92,26 +91,31 @@ func (aesData Encryption) EncryptWithAES(ctx interfaces.AppFunctionContext, data secretData, err := ctx.GetSecret(aesData.SecretPath, aesData.SecretName) if err != nil { return false, fmt.Errorf( - "unable to retieve encryption key at secret path=%s and name=%s", + "unable to retieve encryption key at secret path=%s and name=%s in pipeline '%s'", aesData.SecretPath, - aesData.SecretName) + aesData.SecretName, + ctx.PipelineId()) } key, ok := secretData[aesData.SecretName] if !ok { - return false, fmt.Errorf("unable find encryption key in secret data for name=%s", aesData.SecretName) + return false, fmt.Errorf( + "unable find encryption key in secret data for name=%s in pipeline '%s'", + aesData.SecretName, + ctx.PipelineId()) } ctx.LoggingClient().Debugf( - "Using encryption key from Secret Store at path=%s & name=%s", + "Using encryption key from Secret Store at path=%s & name=%s in pipeline '%s'", aesData.SecretPath, - aesData.SecretName) + aesData.SecretName, + ctx.PipelineId()) aesData.EncryptionKey = key } if len(aesData.EncryptionKey) == 0 { - return false, fmt.Errorf("AES encryption key not set") + return false, fmt.Errorf("AES encryption key not set in pipeline '%s'", ctx.PipelineId()) } hash.Write([]byte((aesData.EncryptionKey))) @@ -120,7 +124,7 @@ func (aesData Encryption) EncryptWithAES(ctx interfaces.AppFunctionContext, data block, err := aes.NewCipher(key) if err != nil { - return false, err + return false, fmt.Errorf("failed to create new AES Cipher in pipeline '%s': %s", ctx.PipelineId(), err) } ecb := cipher.NewCBCEncrypter(block, iv) diff --git a/pkg/transforms/filter.go b/pkg/transforms/filter.go index 51a01c96c..fc47358a5 100755 --- a/pkg/transforms/filter.go +++ b/pkg/transforms/filter.go @@ -29,16 +29,17 @@ import ( type Filter struct { FilterValues []string FilterOut bool + ctx interfaces.AppFunctionContext } // NewFilterFor creates, initializes and returns a new instance of Filter -// that defaults FilterOut to false so it is filtering for specified values +// that defaults FilterOut to false, so it is filtering for specified values func NewFilterFor(filterValues []string) Filter { return Filter{FilterValues: filterValues, FilterOut: false} } // NewFilterOut creates, initializes and returns a new instance of Filter -// that defaults FilterOut to ture so it is filtering out specified values +// that defaults FilterOut to true, so it is filtering out specified values func NewFilterOut(filterValues []string) Filter { return Filter{FilterValues: filterValues, FilterOut: true} } @@ -47,6 +48,7 @@ func NewFilterOut(filterValues []string) Filter { // If FilterOut is false, it filters out those Events not associated with the specified Device Profile listed in FilterValues. // If FilterOut is true, it out those Events that are associated with the specified Device Profile listed in FilterValues. func (f Filter) FilterByProfileName(ctx interfaces.AppFunctionContext, data interface{}) (continuePipeline bool, result interface{}) { + f.ctx = ctx event, err := f.setupForFiltering("FilterByProfileName", "ProfileName", ctx.LoggingClient(), data) if err != nil { return false, err @@ -65,6 +67,7 @@ func (f Filter) FilterByProfileName(ctx interfaces.AppFunctionContext, data inte // If FilterOut is false, it filters out those Events not associated with the specified Device Names listed in FilterValues. // If FilterOut is true, it out those Events that are associated with the specified Device Names listed in FilterValues. func (f Filter) FilterByDeviceName(ctx interfaces.AppFunctionContext, data interface{}) (continuePipeline bool, result interface{}) { + f.ctx = ctx event, err := f.setupForFiltering("FilterByDeviceName", "DeviceName", ctx.LoggingClient(), data) if err != nil { return false, err @@ -82,6 +85,7 @@ func (f Filter) FilterByDeviceName(ctx interfaces.AppFunctionContext, data inter // If FilterOut is false, it filters out those Events not associated with the specified Source listed in FilterValues. // If FilterOut is true, it out those Events that are associated with the specified Source listed in FilterValues. func (f Filter) FilterBySourceName(ctx interfaces.AppFunctionContext, data interface{}) (continuePipeline bool, result interface{}) { + f.ctx = ctx event, err := f.setupForFiltering("FilterBySourceName", "SourceName", ctx.LoggingClient(), data) if err != nil { return false, err @@ -100,12 +104,13 @@ func (f Filter) FilterBySourceName(ctx interfaces.AppFunctionContext, data inter // If FilterOut is true, it out those Event Readings that are associated with the specified Resource Names listed in FilterValues. // This function will return an error and stop the pipeline if a non-edgex event is received or if no data is received. func (f Filter) FilterByResourceName(ctx interfaces.AppFunctionContext, data interface{}) (continuePipeline bool, result interface{}) { + f.ctx = ctx existingEvent, err := f.setupForFiltering("FilterByResourceName", "ResourceName", ctx.LoggingClient(), data) if err != nil { return false, err } - // No filter values, so pass all event and all readings thru, rather than filtering them all out. + // No filter values, so pass all event and all readings through, rather than filtering them all out. if len(f.FilterValues) == 0 { return true, *existingEvent } @@ -127,10 +132,10 @@ func (f Filter) FilterByResourceName(ctx interfaces.AppFunctionContext, data int } if !readingFilteredOut { - ctx.LoggingClient().Debugf("Reading accepted: %s", reading.ResourceName) + ctx.LoggingClient().Debugf("Reading accepted in pipeline '%s' for resource %s", f.ctx.PipelineId(), reading.ResourceName) auxEvent.Readings = append(auxEvent.Readings, reading) } else { - ctx.LoggingClient().Debugf("Reading not accepted: %s", reading.ResourceName) + ctx.LoggingClient().Debugf("Reading not accepted in pipeline '%s' for resource %s", f.ctx.PipelineId(), reading.ResourceName) } } } else { @@ -144,20 +149,20 @@ func (f Filter) FilterByResourceName(ctx interfaces.AppFunctionContext, data int } if readingFilteredFor { - ctx.LoggingClient().Debugf("Reading accepted: %s", reading.ResourceName) + ctx.LoggingClient().Debugf("Reading accepted in pipeline '%s' for resource %s", f.ctx.PipelineId(), reading.ResourceName) auxEvent.Readings = append(auxEvent.Readings, reading) } else { - ctx.LoggingClient().Debugf("Reading not accepted: %s", reading.ResourceName) + ctx.LoggingClient().Debugf("Reading not accepted in pipeline '%s' for resource %s", f.ctx.PipelineId(), reading.ResourceName) } } } if len(auxEvent.Readings) > 0 { - ctx.LoggingClient().Debugf("Event accepted: %d remaining reading(s)", len(auxEvent.Readings)) + ctx.LoggingClient().Debugf("Event accepted: %d remaining reading(s) in pipeline '%s'", len(auxEvent.Readings), f.ctx.PipelineId()) return true, auxEvent } - ctx.LoggingClient().Debug("Event not accepted: 0 remaining readings") + ctx.LoggingClient().Debugf("Event not accepted: 0 remaining readings in pipeline '%s'", f.ctx.PipelineId()) return false, nil } @@ -166,22 +171,22 @@ func (f Filter) setupForFiltering(funcName string, filterProperty string, lc log if f.FilterOut { mode = "Out" } - lc.Debugf("Filtering %s by %s. FilterValues are: '[%v]'", mode, filterProperty, f.FilterValues) + lc.Debugf("Filtering %s by %s in. FilterValues are: '[%v]'", mode, filterProperty, f.FilterValues) if data == nil { - return nil, fmt.Errorf("%s: no Event Received", funcName) + return nil, fmt.Errorf("%s: no Event Received in pipeline '%s'", funcName, f.ctx.PipelineId()) } event, ok := data.(dtos.Event) if !ok { - return nil, fmt.Errorf("%s: type received is not an Event", funcName) + return nil, fmt.Errorf("%s: type received is not an Event in pipeline '%s'", funcName, f.ctx.PipelineId()) } return &event, nil } func (f Filter) doEventFilter(filterProperty string, value string, lc logger.LoggingClient) bool { - // No names to filter for, so pass events thru rather than filtering them all out. + // No names to filter for, so pass events through rather than filtering them all out. if len(f.FilterValues) == 0 { return true } @@ -189,10 +194,10 @@ func (f Filter) doEventFilter(filterProperty string, value string, lc logger.Log for _, name := range f.FilterValues { if value == name { if f.FilterOut { - lc.Debugf("Event not accepted for %s=%s", filterProperty, value) + lc.Debugf("Event not accepted for %s=%s in pipeline '%s'", filterProperty, value, f.ctx.PipelineId()) return false } else { - lc.Debugf("Event accepted for %s=%s", filterProperty, value) + lc.Debugf("Event accepted for %s=%s in pipeline '%s'", filterProperty, value, f.ctx.PipelineId()) return true } } @@ -200,10 +205,10 @@ func (f Filter) doEventFilter(filterProperty string, value string, lc logger.Log // Will only get here if Event's SourceName didn't match any names in FilterValues if f.FilterOut { - lc.Debugf("Event accepted for %s=%s", filterProperty, value) + lc.Debugf("Event accepted for %s=%s in pipeline '%s'", filterProperty, value, f.ctx.PipelineId()) return true } else { - lc.Debugf("Event not accepted for %s=%s", filterProperty, value) + lc.Debugf("Event not accepted for %s=%s in pipeline '%s'", filterProperty, value, f.ctx.PipelineId()) return false } } diff --git a/pkg/transforms/filter_test.go b/pkg/transforms/filter_test.go index 15ff6fd2c..b9df7da26 100644 --- a/pkg/transforms/filter_test.go +++ b/pkg/transforms/filter_test.go @@ -78,7 +78,7 @@ func TestFilter_FilterByProfileName(t *testing.T) { if test.EventIn == nil { continuePipeline, result := filter.FilterByProfileName(ctx, nil) - assert.EqualError(t, result.(error), "FilterByProfileName: no Event Received") + assert.Contains(t, result.(error).Error(), "FilterByProfileName: no Event Received") assert.False(t, continuePipeline) } else { continuePipeline, result := filter.FilterByProfileName(ctx, *test.EventIn) @@ -128,7 +128,7 @@ func TestFilter_FilterByDeviceName(t *testing.T) { if test.EventIn == nil { continuePipeline, result := filter.FilterByDeviceName(ctx, nil) - assert.EqualError(t, result.(error), "FilterByDeviceName: no Event Received") + assert.Contains(t, result.(error).Error(), "FilterByDeviceName: no Event Received") assert.False(t, continuePipeline) } else { continuePipeline, result := filter.FilterByDeviceName(ctx, *test.EventIn) @@ -178,7 +178,7 @@ func TestFilter_FilterBySourceName(t *testing.T) { if test.EventIn == nil { continuePipeline, result := filter.FilterBySourceName(ctx, nil) - assert.EqualError(t, result.(error), "FilterBySourceName: no Event Received") + assert.Contains(t, result.(error).Error(), "FilterBySourceName: no Event Received") assert.False(t, continuePipeline) } else { continuePipeline, result := filter.FilterBySourceName(ctx, *test.EventIn) @@ -258,7 +258,7 @@ func TestFilter_FilterByResourceName(t *testing.T) { if test.EventIn == nil { continuePipeline, result := filter.FilterByResourceName(ctx, nil) - assert.EqualError(t, result.(error), "FilterByResourceName: no Event Received") + assert.Contains(t, result.(error).Error(), "FilterByResourceName: no Event Received") assert.False(t, continuePipeline) } else { continuePipeline, result := filter.FilterByResourceName(ctx, *test.EventIn) diff --git a/pkg/transforms/http.go b/pkg/transforms/http.go index 8891fa48b..2336fe60a 100644 --- a/pkg/transforms/http.go +++ b/pkg/transforms/http.go @@ -18,7 +18,6 @@ package transforms import ( "bytes" - "errors" "fmt" "io" "net/http" @@ -120,19 +119,19 @@ func (sender HTTPSender) HTTPPut(ctx interfaces.AppFunctionContext, data interfa func (sender HTTPSender) httpSend(ctx interfaces.AppFunctionContext, data interface{}, method string) (bool, interface{}) { lc := ctx.LoggingClient() - lc.Debug("HTTP Exporting") + lc.Debugf("HTTP Exporting in pipeline '%s'", ctx.PipelineId()) if data == nil { // We didn't receive a result - return false, errors.New("No Data Received") + return false, fmt.Errorf("function HTTP%s in pipeline '%s': No Data Received", method, ctx.PipelineId()) } if sender.persistOnError && sender.continueOnSendError { - return false, errors.New("persistOnError & continueOnSendError can not both be set to true for HTTP Export") + return false, fmt.Errorf("in pipeline '%s' persistOnError & continueOnSendError can not both be set to true for HTTP Export", ctx.PipelineId()) } if sender.continueOnSendError && !sender.returnInputData { - return false, errors.New("continueOnSendError can only be used in conjunction returnInputData for multiple HTTP Export") + return false, fmt.Errorf("in pipeline '%s' continueOnSendError can only be used in conjunction returnInputData for multiple HTTP Export", ctx.PipelineId()) } if sender.mimeType == "" { @@ -144,7 +143,7 @@ func (sender HTTPSender) httpSend(ctx interfaces.AppFunctionContext, data interf return false, err } - usingSecrets, err := sender.determineIfUsingSecrets() + usingSecrets, err := sender.determineIfUsingSecrets(ctx) if err != nil { return false, err } @@ -173,25 +172,26 @@ func (sender HTTPSender) httpSend(ctx interfaces.AppFunctionContext, data interf return false, err } - lc.Debugf("Setting HTTP Header '%s' with secret value from SecretStore at path='%s' & name='%s", + lc.Debugf("Setting HTTP Header '%s' with secret value from SecretStore at path='%s' & name='%s in pipeline '%s'", sender.httpHeaderName, sender.secretPath, - sender.secretName) + sender.secretName, + ctx.PipelineId()) req.Header.Set(sender.httpHeaderName, theSecrets[sender.secretName]) } req.Header.Set("Content-Type", sender.mimeType) - ctx.LoggingClient().Debugf("POSTing data to %s", sender.url) + ctx.LoggingClient().Debugf("POSTing data to %s in pipeline '%s'", sender.url, ctx.PipelineId()) response, err := client.Do(req) // Pipeline continues if we get a 2xx response, non-2xx response may stop pipeline if err != nil || response.StatusCode < 200 || response.StatusCode >= 300 { if err == nil { - err = fmt.Errorf("export failed with %d HTTP status code", response.StatusCode) + err = fmt.Errorf("export failed with %d HTTP status code in pipeline '%s'", response.StatusCode, ctx.PipelineId()) } else { - err = fmt.Errorf("export failed: %w", err) + err = fmt.Errorf("export failed in pipeline '%s': %s", ctx.PipelineId(), err.Error()) } // If continuing on send error then can't be persisting on error since Store and Forward retries starting @@ -203,14 +203,14 @@ func (sender HTTPSender) httpSend(ctx interfaces.AppFunctionContext, data interf // Continuing pipeline on error // This is in support of sending to multiple export destinations by chaining export functions in the pipeline. - ctx.LoggingClient().Errorf("Continuing pipeline on error: %s", err.Error()) + ctx.LoggingClient().Errorf("Continuing pipeline on error in pipeline '%s': %s", ctx.PipelineId(), err.Error()) // Return the input data since must have some data for the next function to operate on. return true, data } - ctx.LoggingClient().Debugf("Sent %s bytes of data. Response status is %s", len(exportData), response.Status) - ctx.LoggingClient().Trace("Data exported", "Transport", "HTTP", common.CorrelationHeader, ctx.CorrelationID()) + ctx.LoggingClient().Debugf("Sent %d bytes of data in pipeline '%s'. Response status is %s", len(exportData), ctx.PipelineId(), response.Status) + ctx.LoggingClient().Tracef("Data exported for pipeline '%s' (%s=%s)", ctx.PipelineId(), common.CorrelationHeader, ctx.CorrelationID()) // This allows multiple HTTP Exports to be chained in the pipeline to send the same data to different destinations // Don't need to read the response data since not going to return it so just return now. @@ -229,26 +229,26 @@ func (sender HTTPSender) httpSend(ctx interfaces.AppFunctionContext, data interf return true, responseData } -func (sender HTTPSender) determineIfUsingSecrets() (bool, error) { +func (sender HTTPSender) determineIfUsingSecrets(ctx interfaces.AppFunctionContext) (bool, error) { // not using secrets if both are empty if len(sender.secretPath) == 0 && len(sender.secretName) == 0 { if len(sender.httpHeaderName) == 0 { return false, nil } - return false, errors.New("secretPath & secretName must be specified when HTTP Header Name is specified") + return false, fmt.Errorf("in pipeline '%s', secretPath & secretName must be specified when HTTP Header Name is specified", ctx.PipelineId()) } //check if one field but not others are provided for secrets if len(sender.secretPath) != 0 && len(sender.secretName) == 0 { - return false, errors.New("secretPath was specified but no secretName was provided") + return false, fmt.Errorf("in pipeline '%s', secretPath was specified but no secretName was provided", ctx.PipelineId()) } if len(sender.secretName) != 0 && len(sender.secretPath) == 0 { - return false, errors.New("HTTP Header secretName was provided but no secretPath was provided") + return false, fmt.Errorf("in pipeline '%s', HTTP Header secretName was provided but no secretPath was provided", ctx.PipelineId()) } if len(sender.httpHeaderName) == 0 { - return false, errors.New("HTTP Header Name required when using secrets") + return false, fmt.Errorf("in pipeline '%s', HTTP Header Name required when using secrets", ctx.PipelineId()) } // using secrets, all required fields are provided diff --git a/pkg/transforms/http_test.go b/pkg/transforms/http_test.go index b8a1173f6..d6e0a3753 100644 --- a/pkg/transforms/http_test.go +++ b/pkg/transforms/http_test.go @@ -235,7 +235,7 @@ func TestHTTPPostPutWithSecrets(t *testing.T) { assert.Equal(t, test.ExpectToContinue, continuePipeline) if !test.ExpectToContinue { - require.EqualError(t, err.(error), test.ExpectedErrorMessage) + require.Contains(t, err.(error).Error(), test.ExpectedErrorMessage) } assert.Equal(t, test.ExpectedMethod, methodUsed) ctx.RemoveValue("test") @@ -249,7 +249,7 @@ func TestHTTPPostNoParameterPassed(t *testing.T) { assert.False(t, continuePipeline, "Pipeline should stop") assert.Error(t, result.(error), "Result should be an error") - assert.Equal(t, "No Data Received", result.(error).Error()) + assert.Contains(t, result.(error).Error(), "No Data Received") } func TestHTTPPutNoParameterPassed(t *testing.T) { @@ -258,7 +258,7 @@ func TestHTTPPutNoParameterPassed(t *testing.T) { assert.False(t, continuePipeline, "Pipeline should stop") assert.Error(t, result.(error), "Result should be an error") - assert.Equal(t, "No Data Received", result.(error).Error()) + assert.Contains(t, result.(error).Error(), "No Data Received") } func TestHTTPPostInvalidParameter(t *testing.T) { diff --git a/pkg/transforms/jsonlogic.go b/pkg/transforms/jsonlogic.go index c09538748..79fe0bfff 100644 --- a/pkg/transforms/jsonlogic.go +++ b/pkg/transforms/jsonlogic.go @@ -19,7 +19,6 @@ package transforms import ( "bytes" "encoding/json" - "errors" "fmt" "strconv" "strings" @@ -46,7 +45,7 @@ func NewJSONLogic(rule string) JSONLogic { func (logic JSONLogic) Evaluate(ctx interfaces.AppFunctionContext, data interface{}) (bool, interface{}) { if data == nil { // We didn't receive a result - return false, errors.New("No Data Received") + return false, fmt.Errorf("function Evaluate in pipeline '%s': No Data Received", ctx.PipelineId()) } coercedData, err := util.CoerceType(data) @@ -58,20 +57,20 @@ func (logic JSONLogic) Evaluate(ctx interfaces.AppFunctionContext, data interfac rule := strings.NewReader(logic.Rule) var logicResult bytes.Buffer - ctx.LoggingClient().Debug("Applying JSONLogic Rule") + ctx.LoggingClient().Debugf("Applying JSONLogic Rule in pipeline '%s'", ctx.PipelineId()) err = jsonlogic.Apply(rule, reader, &logicResult) if err != nil { - return false, fmt.Errorf("unable to apply JSONLogic rule: %s", err.Error()) + return false, fmt.Errorf("unable to apply JSONLogic rule in pipeline '%s': %s", ctx.PipelineId(), err.Error()) } var result bool decoder := json.NewDecoder(&logicResult) err = decoder.Decode(&result) if err != nil { - return false, fmt.Errorf("unable to decode JSONLogic result: %s", err.Error()) + return false, fmt.Errorf("unable to decode JSONLogic result in pipeline '%s': %s", ctx.PipelineId(), err.Error()) } - ctx.LoggingClient().Debug("Condition met: " + strconv.FormatBool(result)) + ctx.LoggingClient().Debugf("Condition met in pipeline '%s': %s", ctx.PipelineId(), strconv.FormatBool(result)) return result, data } diff --git a/pkg/transforms/jsonlogic_test.go b/pkg/transforms/jsonlogic_test.go index 8d6b25d30..8c9492191 100644 --- a/pkg/transforms/jsonlogic_test.go +++ b/pkg/transforms/jsonlogic_test.go @@ -68,7 +68,7 @@ func TestJSONLogicValidJSONBadRule(t *testing.T) { assert.NotNil(t, result) assert.False(t, continuePipeline) require.IsType(t, fmt.Errorf(""), result) - assert.Equal(t, "unable to apply JSONLogic rule: The operator \"notAnOperator\" is not supported", result.(error).Error()) + assert.Contains(t, result.(error).Error(), "unable to apply JSONLogic rule") } func TestJSONLogicNoData(t *testing.T) { diff --git a/pkg/transforms/mqttsecret.go b/pkg/transforms/mqttsecret.go index 2f6d469bb..ee5d47eeb 100644 --- a/pkg/transforms/mqttsecret.go +++ b/pkg/transforms/mqttsecret.go @@ -17,7 +17,6 @@ package transforms import ( - "errors" "fmt" "strings" "sync" @@ -113,7 +112,7 @@ func (sender *MQTTSecretSender) initializeMQTTClient(ctx interfaces.AppFunctionC if len(sender.mqttConfig.KeepAlive) > 0 { keepAlive, err := time.ParseDuration(sender.mqttConfig.KeepAlive) if err != nil { - return fmt.Errorf("Unable to parse KeepAlive value of '%s': %w", sender.mqttConfig.KeepAlive, err) + return fmt.Errorf("in pipeline '%s', unable to parse KeepAlive value of '%s': %s", ctx.PipelineId(), sender.mqttConfig.KeepAlive, err.Error()) } sender.opts.SetKeepAlive(keepAlive) @@ -122,7 +121,7 @@ func (sender *MQTTSecretSender) initializeMQTTClient(ctx interfaces.AppFunctionC if len(sender.mqttConfig.ConnectTimeout) > 0 { timeout, err := time.ParseDuration(sender.mqttConfig.ConnectTimeout) if err != nil { - return fmt.Errorf("Unable to parse ConnectTimeout value of '%s': %w", sender.mqttConfig.ConnectTimeout, err) + return fmt.Errorf("in pipeline '%s', unable to parse ConnectTimeout value of '%s': %s", ctx.PipelineId(), sender.mqttConfig.ConnectTimeout, err.Error()) } sender.opts.SetConnectTimeout(timeout) @@ -130,7 +129,7 @@ func (sender *MQTTSecretSender) initializeMQTTClient(ctx interfaces.AppFunctionC client, err := mqttFactory.Create(sender.opts) if err != nil { - return err + return fmt.Errorf("in pipeline '%s', unable to create MQTT Client: %s", ctx.PipelineId(), err.Error()) } sender.client = client @@ -156,9 +155,9 @@ func (sender *MQTTSecretSender) connectToBroker(ctx interfaces.AppFunctionContex if sender.persistOnError { subMessage = "persisting Event for later retry" } - return fmt.Errorf("Could not connect to mqtt server for export, %s. Error: %s", subMessage, token.Error().Error()) + return fmt.Errorf("in pipeline '%s', could not connect to mqtt server for export, %s. Error: %s", ctx.PipelineId(), subMessage, token.Error().Error()) } - ctx.LoggingClient().Info("Connected to mqtt server for export") + ctx.LoggingClient().Infof("Connected to mqtt server for export in pipeline '%s'", ctx.PipelineId()) return nil } @@ -167,7 +166,7 @@ func (sender *MQTTSecretSender) connectToBroker(ctx interfaces.AppFunctionContex func (sender *MQTTSecretSender) MQTTSend(ctx interfaces.AppFunctionContext, data interface{}) (bool, interface{}) { if data == nil { // We didn't receive a result - return false, errors.New("No Data Received") + return false, fmt.Errorf("function MQTTSend in pipeline '%s': No Data Received", ctx.PipelineId()) } exportData, err := util.CoerceType(data) @@ -191,7 +190,7 @@ func (sender *MQTTSecretSender) MQTTSend(ctx interfaces.AppFunctionContext, data publishTopic, err := sender.topicFormatter.invoke(sender.mqttConfig.Topic, ctx, data) if err != nil { - return false, fmt.Errorf("MQTT topic formatting failed: %s", err.Error()) + return false, fmt.Errorf("in pipeline '%s', MQTT topic formatting failed: %s", ctx.PipelineId(), err.Error()) } token := sender.client.Publish(publishTopic, sender.mqttConfig.QoS, sender.mqttConfig.Retain, exportData) @@ -201,8 +200,8 @@ func (sender *MQTTSecretSender) MQTTSend(ctx interfaces.AppFunctionContext, data return false, token.Error() } - ctx.LoggingClient().Debug("Sent data to MQTT Broker") - ctx.LoggingClient().Trace("Data exported", "Transport", "MQTT", common.CorrelationHeader, ctx.CorrelationID()) + ctx.LoggingClient().Debugf("Sent data to MQTT Broker in pipeline '%s'", ctx.PipelineId()) + ctx.LoggingClient().Tracef("Data exported", "Transport", "MQTT", "pipeline", ctx.PipelineId(), common.CorrelationHeader, ctx.CorrelationID()) return true, nil } diff --git a/pkg/transforms/responsedata.go b/pkg/transforms/responsedata.go index 5109863aa..09cfbbcc5 100644 --- a/pkg/transforms/responsedata.go +++ b/pkg/transforms/responsedata.go @@ -17,6 +17,8 @@ package transforms import ( + "fmt" + "github.com/edgexfoundry/app-functions-sdk-go/v2/pkg/interfaces" "github.com/edgexfoundry/app-functions-sdk-go/v2/pkg/util" ) @@ -35,11 +37,11 @@ func NewResponseData() ResponseData { // It will return an error and stop the pipeline if the input data is not of type []byte, string or json.Marshaller func (f ResponseData) SetResponseData(ctx interfaces.AppFunctionContext, data interface{}) (bool, interface{}) { - ctx.LoggingClient().Debug("Setting response data") + ctx.LoggingClient().Debugf("Setting response data in pipeline '%s'", ctx.PipelineId()) if data == nil { // We didn't receive a result - return false, nil + return false, fmt.Errorf("function SetResponseData in pipeline '%s': No Data Received", ctx.PipelineId()) } byteData, err := util.CoerceType(data) diff --git a/pkg/transforms/responsedata_test.go b/pkg/transforms/responsedata_test.go index de232fdfa..4098ddaa8 100644 --- a/pkg/transforms/responsedata_test.go +++ b/pkg/transforms/responsedata_test.go @@ -72,7 +72,7 @@ func TestSetResponseDataEvent(t *testing.T) { func TestSetResponseDataNoData(t *testing.T) { target := NewResponseData() continuePipeline, result := target.SetResponseData(ctx, nil) - assert.Nil(t, result) + assert.Contains(t, result.(error).Error(), "No Data Received") assert.False(t, continuePipeline) } diff --git a/pkg/transforms/tags.go b/pkg/transforms/tags.go index 121c1899b..24b8509ca 100644 --- a/pkg/transforms/tags.go +++ b/pkg/transforms/tags.go @@ -17,7 +17,7 @@ package transforms import ( - "errors" + "fmt" "github.com/edgexfoundry/app-functions-sdk-go/v2/pkg/interfaces" @@ -38,15 +38,15 @@ func NewTags(tags map[string]string) Tags { // AddTags adds the pre-configured list of tags to the Event's tags collection. func (t *Tags) AddTags(ctx interfaces.AppFunctionContext, data interface{}) (bool, interface{}) { - ctx.LoggingClient().Debug("Adding tags to Event") + ctx.LoggingClient().Debugf("Adding tags to Event in pipeline '%s'", ctx.PipelineId()) if data == nil { - return false, errors.New("no Event Received") + return false, fmt.Errorf("function AddTags in pipeline '%s': No Data Received", ctx.PipelineId()) } event, ok := data.(dtos.Event) if !ok { - return false, errors.New("type received is not an Event") + return false, fmt.Errorf("function AddTags in pipeline '%s', type received is not an Event", ctx.PipelineId()) } if len(t.tags) > 0 { @@ -57,9 +57,9 @@ func (t *Tags) AddTags(ctx interfaces.AppFunctionContext, data interface{}) (boo for tag, value := range t.tags { event.Tags[tag] = value } - ctx.LoggingClient().Debugf("Tags added to Event. Event tags=%v", event.Tags) + ctx.LoggingClient().Debugf("Tags added to Event in pipeline '%s'. Event tags=%v", ctx.PipelineId(), event.Tags) } else { - ctx.LoggingClient().Debug("No tags added to Event. Add tags list is empty.") + ctx.LoggingClient().Debugf("No tags added to Event in pipeline '%s'. Add tags list is empty.", ctx.PipelineId()) } return true, event diff --git a/pkg/transforms/tags_test.go b/pkg/transforms/tags_test.go index 5c78b8051..50a0557d5 100644 --- a/pkg/transforms/tags_test.go +++ b/pkg/transforms/tags_test.go @@ -58,8 +58,8 @@ func TestTags_AddTags(t *testing.T) { {"Happy path - no existing Event tags", dtos.Event{}, tagsToAdd, tagsToAdd, false, ""}, {"Happy path - Event has existing tags", eventWithExistingTags, tagsToAdd, allTagsAdded, false, ""}, {"Happy path - No tags added", eventWithExistingTags, map[string]string{}, eventWithExistingTags.Tags, false, ""}, - {"Error - No data", nil, nil, nil, true, "no Event Received"}, - {"Error - Input not event", "Not an Event", nil, nil, true, "not an Event"}, + {"Error - No data", nil, nil, nil, true, "No Data Received"}, + {"Error - Input not event", "Not an Event", nil, nil, true, "type received is not an Event"}, } for _, testCase := range tests {