From 61e98401d1d3bd10a2c677f5c9c9db0f12626e5b Mon Sep 17 00:00:00 2001 From: stephane benoist <29753229+stebenoist@users.noreply.github.com> Date: Mon, 26 Nov 2018 17:03:13 +0100 Subject: [PATCH 01/19] Refactor events structs --- commands/deployments/dep_events.go | 12 ++--- events/events.go | 30 +++++------ events/events_test.go | 10 ++-- events/structs.go | 83 +++++++++--------------------- events/structs_test.go | 69 ------------------------- rest/structs.go | 4 +- 6 files changed, 53 insertions(+), 155 deletions(-) delete mode 100644 events/structs_test.go diff --git a/commands/deployments/dep_events.go b/commands/deployments/dep_events.go index 9d4ad602b..e1ccf56e8 100644 --- a/commands/deployments/dep_events.go +++ b/commands/deployments/dep_events.go @@ -132,7 +132,7 @@ func StreamsEvents(client *httputil.YorcClient, deploymentID string, colorize, f if colorize { ts = color.CyanString("%s", event.Timestamp) } - evType, err := events.StatusUpdateTypeString(event.Type) + evType, err := events.ParseStatusChangeType(event.Type) if err != nil { if colorize { fmt.Printf("%s: ", color.MagentaString("Warning")) @@ -142,15 +142,15 @@ func StreamsEvents(client *httputil.YorcClient, deploymentID string, colorize, f fmt.Printf("Unknown event type: %q\n", event.Type) } switch evType { - case events.InstanceStatusChangeType: + case events.StatusChangeTypeInstance: fmt.Printf("%s:\t Deployment: %s\t Node: %s\t Instance: %s\t State: %s\n", ts, event.DeploymentID, event.Node, event.Instance, event.Status) - case events.DeploymentStatusChangeType: + case events.StatusChangeTypeDeployment: fmt.Printf("%s:\t Deployment: %s\t Deployment Status: %s\n", ts, event.DeploymentID, event.Status) - case events.CustomCommandStatusChangeType: + case events.StatusChangeTypeCustomCommand: fmt.Printf("%s:\t Deployment: %s\t Task %q (custom command)\t Status: %s\n", ts, event.DeploymentID, event.TaskID, event.Status) - case events.ScalingStatusChangeType: + case events.StatusChangeTypeScaling: fmt.Printf("%s:\t Deployment: %s\t Task %q (scaling)\t Status: %s\n", ts, event.DeploymentID, event.TaskID, event.Status) - case events.WorkflowStatusChangeType: + case events.StatusChangeTypeWorkflow: fmt.Printf("%s:\t Deployment: %s\t Task %q (workflow)\t Status: %s\n", ts, event.DeploymentID, event.TaskID, event.Status) } diff --git a/events/events.go b/events/events.go index 7a795f753..1c263ccf1 100644 --- a/events/events.go +++ b/events/events.go @@ -44,7 +44,7 @@ func InstanceStatusChange(kv *api.KV, deploymentID, nodeName, instance, status s func PublishAndLogInstanceStatusChange(ctx context.Context, kv *api.KV, deploymentID, nodeName, instance, status string) (string, error) { ctx = AddLogOptionalFields(ctx, LogOptionalFields{NodeID: nodeName, InstanceID: instance}) - id, err := storeStatusUpdateEvent(kv, deploymentID, InstanceStatusChangeType, nodeName+"\n"+status+"\n"+instance) + id, err := storeStatusUpdateEvent(kv, deploymentID, StatusChangeTypeInstance, nodeName+"\n"+status+"\n"+instance) if err != nil { return "", err } @@ -65,7 +65,7 @@ func DeploymentStatusChange(kv *api.KV, deploymentID, status string) (string, er // // PublishAndLogDeploymentStatusChange returns the published event id func PublishAndLogDeploymentStatusChange(ctx context.Context, kv *api.KV, deploymentID, status string) (string, error) { - id, err := storeStatusUpdateEvent(kv, deploymentID, DeploymentStatusChangeType, status) + id, err := storeStatusUpdateEvent(kv, deploymentID, StatusChangeTypeDeployment, status) if err != nil { return "", err } @@ -89,7 +89,7 @@ func PublishAndLogCustomCommandStatusChange(ctx context.Context, kv *api.KV, dep if ctx == nil { ctx = NewContext(context.Background(), LogOptionalFields{ExecutionID: taskID}) } - id, err := storeStatusUpdateEvent(kv, deploymentID, CustomCommandStatusChangeType, taskID+"\n"+status) + id, err := storeStatusUpdateEvent(kv, deploymentID, StatusChangeTypeCustomCommand, taskID+"\n"+status) if err != nil { return "", err } @@ -113,7 +113,7 @@ func PublishAndLogScalingStatusChange(ctx context.Context, kv *api.KV, deploymen if ctx == nil { ctx = NewContext(context.Background(), LogOptionalFields{ExecutionID: taskID}) } - id, err := storeStatusUpdateEvent(kv, deploymentID, ScalingStatusChangeType, taskID+"\n"+status) + id, err := storeStatusUpdateEvent(kv, deploymentID, StatusChangeTypeScaling, taskID+"\n"+status) if err != nil { return "", err } @@ -137,7 +137,7 @@ func PublishAndLogWorkflowStatusChange(ctx context.Context, kv *api.KV, deployme if ctx == nil { ctx = NewContext(context.Background(), LogOptionalFields{ExecutionID: taskID}) } - id, err := storeStatusUpdateEvent(kv, deploymentID, WorkflowStatusChangeType, taskID+"\n"+status) + id, err := storeStatusUpdateEvent(kv, deploymentID, StatusChangeTypeWorkflow, taskID+"\n"+status) if err != nil { return "", err } @@ -148,7 +148,7 @@ func PublishAndLogWorkflowStatusChange(ctx context.Context, kv *api.KV, deployme // Create a KVPair corresponding to an event and put it to Consul under the event prefix, // in a sub-tree corresponding to its deployment // The eventType goes to the KVPair's Flags field -func storeStatusUpdateEvent(kv *api.KV, deploymentID string, eventType StatusUpdateType, data string) (string, error) { +func storeStatusUpdateEvent(kv *api.KV, deploymentID string, eventType StatusChangeType, data string) (string, error) { now := time.Now().Format(time.RFC3339Nano) eventsPrefix := path.Join(consulutil.EventsPrefix, deploymentID) err := consulutil.StoreConsulKeyAsStringWithFlags(path.Join(eventsPrefix, now), data, uint64(eventType)) @@ -159,8 +159,8 @@ func storeStatusUpdateEvent(kv *api.KV, deploymentID string, eventType StatusUpd } // StatusEvents return a list of events (StatusUpdate instances) for all, or a given deployment -func StatusEvents(kv *api.KV, deploymentID string, waitIndex uint64, timeout time.Duration) ([]StatusUpdate, uint64, error) { - events := make([]StatusUpdate, 0) +func StatusEvents(kv *api.KV, deploymentID string, waitIndex uint64, timeout time.Duration) ([]EventStatusChange, uint64, error) { + events := make([]EventStatusChange, 0) var eventsPrefix string var depIDProvided bool @@ -192,24 +192,24 @@ func StatusEvents(kv *api.KV, deploymentID string, waitIndex uint64, timeout tim } values := strings.Split(string(kvp.Value), "\n") - eventType := StatusUpdateType(kvp.Flags) + eventType := StatusChangeType(kvp.Flags) switch eventType { - case InstanceStatusChangeType: + case StatusChangeTypeInstance: if len(values) != 3 { return events, qm.LastIndex, errors.Errorf("Unexpected event value %q for event %q", string(kvp.Value), kvp.Key) } - events = append(events, StatusUpdate{Timestamp: eventTimestamp, Type: eventType.String(), Node: values[0], Status: values[1], Instance: values[2], DeploymentID: deploymentID}) - case DeploymentStatusChangeType: + events = append(events, EventStatusChange{Timestamp: eventTimestamp, Type: eventType.String(), Node: values[0], Status: values[1], Instance: values[2], DeploymentID: deploymentID}) + case StatusChangeTypeDeployment: if len(values) != 1 { return events, qm.LastIndex, errors.Errorf("Unexpected event value %q for event %q", string(kvp.Value), kvp.Key) } - events = append(events, StatusUpdate{Timestamp: eventTimestamp, Type: eventType.String(), Status: values[0], DeploymentID: deploymentID}) - case CustomCommandStatusChangeType, ScalingStatusChangeType, WorkflowStatusChangeType: + events = append(events, EventStatusChange{Timestamp: eventTimestamp, Type: eventType.String(), Status: values[0], DeploymentID: deploymentID}) + case StatusChangeTypeCustomCommand, StatusChangeTypeScaling, StatusChangeTypeWorkflow, StatusChangeTypeWorkflowStep: if len(values) != 2 { return events, qm.LastIndex, errors.Errorf("Unexpected event value %q for event %q", string(kvp.Value), kvp.Key) } - events = append(events, StatusUpdate{Timestamp: eventTimestamp, Type: eventType.String(), TaskID: values[0], Status: values[1], DeploymentID: deploymentID}) + events = append(events, EventStatusChange{Timestamp: eventTimestamp, Type: eventType.String(), TaskID: values[0], Status: values[1], DeploymentID: deploymentID}) default: return events, qm.LastIndex, errors.Errorf("Unsupported event type %d for event %q", kvp.Flags, kvp.Key) } diff --git a/events/events_test.go b/events/events_test.go index 0a052a6ad..dad139b0c 100644 --- a/events/events_test.go +++ b/events/events_test.go @@ -388,35 +388,35 @@ func testconsulGetStatusEvents(t *testing.T, kv *api.KV) { require.Nil(t, err) require.Len(t, events, 5) - require.Equal(t, InstanceStatusChangeType.String(), events[0].Type) + require.Equal(t, StatusChangeTypeInstance.String(), events[0].Type) require.Equal(t, "node1", events[0].Node) require.Equal(t, "1", events[0].Instance) require.Equal(t, "started", events[0].Status) require.Equal(t, ids[0], events[0].Timestamp) require.Equal(t, "", events[0].TaskID) - require.Equal(t, DeploymentStatusChangeType.String(), events[1].Type) + require.Equal(t, StatusChangeTypeDeployment.String(), events[1].Type) require.Equal(t, "", events[1].Node) require.Equal(t, "", events[1].Instance) require.Equal(t, "deployed", events[1].Status) require.Equal(t, ids[1], events[1].Timestamp) require.Equal(t, "", events[1].TaskID) - require.Equal(t, ScalingStatusChangeType.String(), events[2].Type) + require.Equal(t, StatusChangeTypeScaling.String(), events[2].Type) require.Equal(t, "", events[2].Node) require.Equal(t, "", events[2].Instance) require.Equal(t, "failed", events[2].Status) require.Equal(t, ids[2], events[2].Timestamp) require.Equal(t, "t2", events[2].TaskID) - require.Equal(t, CustomCommandStatusChangeType.String(), events[3].Type) + require.Equal(t, StatusChangeTypeCustomCommand.String(), events[3].Type) require.Equal(t, "", events[3].Node) require.Equal(t, "", events[3].Instance) require.Equal(t, "running", events[3].Status) require.Equal(t, ids[3], events[3].Timestamp) require.Equal(t, "t3", events[3].TaskID) - require.Equal(t, WorkflowStatusChangeType.String(), events[4].Type) + require.Equal(t, StatusChangeTypeWorkflow.String(), events[4].Type) require.Equal(t, "", events[4].Node) require.Equal(t, "", events[4].Instance) require.Equal(t, "done", events[4].Status) diff --git a/events/structs.go b/events/structs.go index 4ef8b0dda..b86d02468 100644 --- a/events/structs.go +++ b/events/structs.go @@ -14,62 +14,29 @@ package events -import ( - "fmt" - - "github.com/pkg/errors" -) - -// StatusUpdateType is the status update type -type StatusUpdateType uint64 - -const ( - // InstanceStatusChangeType is the StatusUpdate type for an instance state change event - InstanceStatusChangeType StatusUpdateType = iota - // DeploymentStatusChangeType is the StatusUpdate type for an deployment status change event - DeploymentStatusChangeType - // CustomCommandStatusChangeType is the StatusUpdate type for an custom command status change event - CustomCommandStatusChangeType - // ScalingStatusChangeType is the StatusUpdate type for an scaling status change event - ScalingStatusChangeType - // WorkflowStatusChangeType is the StatusUpdate type for an workflow status change event - WorkflowStatusChangeType -) - -// StatusUpdate represents status change event -type StatusUpdate struct { - Timestamp string `json:"timestamp"` - Type string `json:"type"` - Node string `json:"node,omitempty"` - Instance string `json:"instance,omitempty"` - TaskID string `json:"task_id,omitempty"` - DeploymentID string `json:"deployment_id"` - Status string `json:"status"` -} - -const _StatusUpdateType_name = "instancedeploymentcustom-commandscalingworkflow" - -var _StatusUpdateType_index = [...]uint8{0, 8, 18, 32, 39, 47} - -func (i StatusUpdateType) String() string { - if i >= StatusUpdateType(len(_StatusUpdateType_index)-1) { - return fmt.Sprintf("StatusUpdateType(%d)", i) - } - return _StatusUpdateType_name[_StatusUpdateType_index[i]:_StatusUpdateType_index[i+1]] -} - -var _StatusUpdateTypeNameToValue_map = map[string]StatusUpdateType{ - _StatusUpdateType_name[0:8]: 0, - _StatusUpdateType_name[8:18]: 1, - _StatusUpdateType_name[18:32]: 2, - _StatusUpdateType_name[32:39]: 3, - _StatusUpdateType_name[39:47]: 4, -} - -// StatusUpdateTypeString returns a StatusUpdateType given its string representation -func StatusUpdateTypeString(s string) (StatusUpdateType, error) { - if val, ok := _StatusUpdateTypeNameToValue_map[s]; ok { - return val, nil - } - return 0, errors.Errorf("%s does not belong to StatusUpdateType values", s) +//go:generate go-enum -f=structs.go --lower + +// LogLevel x ENUM( +// Instance, +// Deployment, +// CustomCommand, +// Scaling, +// Workflow, +// WorkflowStep +// ) +type StatusChangeType int + +// EventOptionalInfo is event's additional info +type EventOptionalInfo map[string]interface{} + +// EventStatusChange represents status change event +type EventStatusChange struct { + Timestamp string `json:"timestamp"` + Type string `json:"type"` + Node string `json:"node,omitempty"` + Instance string `json:"instance,omitempty"` + TaskID string `json:"task_id,omitempty"` + DeploymentID string `json:"deployment_id"` + Status string `json:"status"` + AdditionalInfo EventOptionalInfo } diff --git a/events/structs_test.go b/events/structs_test.go deleted file mode 100644 index 3f17e4b6c..000000000 --- a/events/structs_test.go +++ /dev/null @@ -1,69 +0,0 @@ -// Copyright 2018 Bull S.A.S. Atos Technologies - Bull, Rue Jean Jaures, B.P.68, 78340, Les Clayes-sous-Bois, France. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package events - -import "testing" - -func TestStatusUpdateType_String(t *testing.T) { - tests := []struct { - name string - i StatusUpdateType - want string - }{ - {"InstanceToString", InstanceStatusChangeType, "instance"}, - {"DeploymentToString", DeploymentStatusChangeType, "deployment"}, - {"CustomCommandToString", CustomCommandStatusChangeType, "custom-command"}, - {"ScalingToString", ScalingStatusChangeType, "scaling"}, - {"WorkflowToString", WorkflowStatusChangeType, "workflow"}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := tt.i.String(); got != tt.want { - t.Errorf("StatusUpdateType.String() = %v, want %v", got, tt.want) - } - }) - } -} - -func TestStatusUpdateTypeString(t *testing.T) { - type args struct { - s string - } - tests := []struct { - name string - args args - want StatusUpdateType - wantErr bool - }{ - {"InstanceFromString", args{"instance"}, InstanceStatusChangeType, false}, - {"DeploymentFromString", args{"deployment"}, DeploymentStatusChangeType, false}, - {"CustomCommandFromString", args{"custom-command"}, CustomCommandStatusChangeType, false}, - {"ScalingFromString", args{"scaling"}, ScalingStatusChangeType, false}, - {"WorkflowFromString", args{"workflow"}, WorkflowStatusChangeType, false}, - {"UnknownFromString", args{"err"}, InstanceStatusChangeType, true}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := StatusUpdateTypeString(tt.args.s) - if (err != nil) != tt.wantErr { - t.Errorf("StatusUpdateTypeString() error = %v, wantErr %v", err, tt.wantErr) - return - } - if got != tt.want { - t.Errorf("StatusUpdateTypeString() = %v, want %v", got, tt.want) - } - }) - } -} diff --git a/rest/structs.go b/rest/structs.go index cf0e28916..5e824b7f8 100644 --- a/rest/structs.go +++ b/rest/structs.go @@ -102,8 +102,8 @@ type DeploymentsCollection struct { // EventsCollection is a collection of instances status change events type EventsCollection struct { - Events []events.StatusUpdate `json:"events"` - LastIndex uint64 `json:"last_index"` + Events []events.EventStatusChange `json:"events"` + LastIndex uint64 `json:"last_index"` } // LogsCollection is a collection of logs events From 325a027adde931ec52cce277f006f850a1035264 Mon Sep 17 00:00:00 2001 From: stephane benoist <29753229+stebenoist@users.noreply.github.com> Date: Mon, 26 Nov 2018 17:06:10 +0100 Subject: [PATCH 02/19] Add missing generated file --- events/structs_enum.go | 80 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 events/structs_enum.go diff --git a/events/structs_enum.go b/events/structs_enum.go new file mode 100644 index 000000000..0b86f61ae --- /dev/null +++ b/events/structs_enum.go @@ -0,0 +1,80 @@ +// Copyright 2018 Bull S.A.S. Atos Technologies - Bull, Rue Jean Jaures, B.P.68, 78340, Les Clayes-sous-Bois, France. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by go-enum +// DO NOT EDIT! + +package events + +import ( + "fmt" + "strings" +) + +const ( + // StatusChangeTypeInstance is a StatusChangeType of type Instance + StatusChangeTypeInstance StatusChangeType = iota + // StatusChangeTypeDeployment is a StatusChangeType of type Deployment + StatusChangeTypeDeployment + // StatusChangeTypeCustomCommand is a StatusChangeType of type CustomCommand + StatusChangeTypeCustomCommand + // StatusChangeTypeScaling is a StatusChangeType of type Scaling + StatusChangeTypeScaling + // StatusChangeTypeWorkflow is a StatusChangeType of type Workflow + StatusChangeTypeWorkflow + // StatusChangeTypeWorkflowStep is a StatusChangeType of type WorkflowStep + StatusChangeTypeWorkflowStep +) + +const _StatusChangeTypeName = "InstanceDeploymentCustomCommandScalingWorkflowWorkflowStep" + +var _StatusChangeTypeMap = map[StatusChangeType]string{ + 0: _StatusChangeTypeName[0:8], + 1: _StatusChangeTypeName[8:18], + 2: _StatusChangeTypeName[18:31], + 3: _StatusChangeTypeName[31:38], + 4: _StatusChangeTypeName[38:46], + 5: _StatusChangeTypeName[46:58], +} + +// String implements the Stringer interface. +func (x StatusChangeType) String() string { + if str, ok := _StatusChangeTypeMap[x]; ok { + return str + } + return fmt.Sprintf("StatusChangeType(%d)", x) +} + +var _StatusChangeTypeValue = map[string]StatusChangeType{ + _StatusChangeTypeName[0:8]: 0, + strings.ToLower(_StatusChangeTypeName[0:8]): 0, + _StatusChangeTypeName[8:18]: 1, + strings.ToLower(_StatusChangeTypeName[8:18]): 1, + _StatusChangeTypeName[18:31]: 2, + strings.ToLower(_StatusChangeTypeName[18:31]): 2, + _StatusChangeTypeName[31:38]: 3, + strings.ToLower(_StatusChangeTypeName[31:38]): 3, + _StatusChangeTypeName[38:46]: 4, + strings.ToLower(_StatusChangeTypeName[38:46]): 4, + _StatusChangeTypeName[46:58]: 5, + strings.ToLower(_StatusChangeTypeName[46:58]): 5, +} + +// ParseStatusChangeType attempts to convert a string to a StatusChangeType +func ParseStatusChangeType(name string) (StatusChangeType, error) { + if x, ok := _StatusChangeTypeValue[name]; ok { + return x, nil + } + return StatusChangeType(0), fmt.Errorf("%s is not a valid StatusChangeType", name) +} From 2022a70ad573bd1fd83a9c732365ddacd614b403 Mon Sep 17 00:00:00 2001 From: benoist-s Date: Tue, 27 Nov 2018 17:34:17 +0100 Subject: [PATCH 03/19] Wip on refactoring events --- commands/deployments/dep_events.go | 30 +- events/consul_test.go | 60 +-- events/events.go | 72 +-- events/events_test.go | 791 ++++++++++++++--------------- events/log_entries.go | 6 +- events/structs.go | 76 ++- rest/structs.go | 5 +- 7 files changed, 514 insertions(+), 526 deletions(-) diff --git a/commands/deployments/dep_events.go b/commands/deployments/dep_events.go index e1ccf56e8..58564eb3b 100644 --- a/commands/deployments/dep_events.go +++ b/commands/deployments/dep_events.go @@ -27,7 +27,6 @@ import ( "github.com/pkg/errors" "github.com/spf13/cobra" "github.com/ystia/yorc/commands/httputil" - "github.com/ystia/yorc/events" "github.com/ystia/yorc/rest" ) @@ -128,36 +127,13 @@ func StreamsEvents(client *httputil.YorcClient, deploymentID string, colorize, f } lastIdx = evts.LastIndex for _, event := range evts.Events { - ts := event.Timestamp if colorize { - ts = color.CyanString("%s", event.Timestamp) - } - evType, err := events.ParseStatusChangeType(event.Type) - if err != nil { - if colorize { - fmt.Printf("%s: ", color.MagentaString("Warning")) - } else { - fmt.Print("Warning: ") - } - fmt.Printf("Unknown event type: %q\n", event.Type) - } - switch evType { - case events.StatusChangeTypeInstance: - fmt.Printf("%s:\t Deployment: %s\t Node: %s\t Instance: %s\t State: %s\n", ts, event.DeploymentID, event.Node, event.Instance, event.Status) - case events.StatusChangeTypeDeployment: - fmt.Printf("%s:\t Deployment: %s\t Deployment Status: %s\n", ts, event.DeploymentID, event.Status) - case events.StatusChangeTypeCustomCommand: - fmt.Printf("%s:\t Deployment: %s\t Task %q (custom command)\t Status: %s\n", ts, event.DeploymentID, event.TaskID, event.Status) - case events.StatusChangeTypeScaling: - fmt.Printf("%s:\t Deployment: %s\t Task %q (scaling)\t Status: %s\n", ts, event.DeploymentID, event.TaskID, event.Status) - case events.StatusChangeTypeWorkflow: - fmt.Printf("%s:\t Deployment: %s\t Task %q (workflow)\t Status: %s\n", ts, event.DeploymentID, event.TaskID, event.Status) + fmt.Printf("%s\n", color.MagentaString("%s", format(event))) + } else { + fmt.Printf("%s\n", format(event)) } - } - response.Body.Close() - if stop { return } diff --git a/events/consul_test.go b/events/consul_test.go index fa5f72caa..3d8efa2a5 100644 --- a/events/consul_test.go +++ b/events/consul_test.go @@ -27,36 +27,36 @@ func TestRunConsulEventsPackageTests(t *testing.T) { defer srv.Stop() t.Run("groupEvents", func(t *testing.T) { - t.Run("TestConsulPubSubStatusChange", func(t *testing.T) { - testConsulPubSubStatusChange(t, kv) - }) - t.Run("TestConsulPubSubNewEvents", func(t *testing.T) { - testConsulPubSubNewEvents(t, kv) - }) - t.Run("TestConsulPubSubNewEventsTimeout", func(t *testing.T) { - testConsulPubSubNewEventsTimeout(t, kv) - }) - t.Run("TestConsulPubSubNewEventsWithIndex", func(t *testing.T) { - testConsulPubSubNewEventsWithIndex(t, kv) - }) - t.Run("TestConsulPubSubNewNodeEvents", func(t *testing.T) { - testConsulPubSubNewNodeEvents(t, kv) - }) - t.Run("TestDeploymentStatusChange", func(t *testing.T) { - testconsulDeploymentStatusChange(t, kv) - }) - t.Run("TestCustomCommandStatusChange", func(t *testing.T) { - testconsulCustomCommandStatusChange(t, kv) - }) - t.Run("TestScalingStatusChange", func(t *testing.T) { - testconsulScalingStatusChange(t, kv) - }) - t.Run("TestWorkflowStatusChange", func(t *testing.T) { - testconsulWorkflowStatusChange(t, kv) - }) - t.Run("TestGetStatusEvents", func(t *testing.T) { - testconsulGetStatusEvents(t, kv) - }) + //t.Run("TestConsulPubSubStatusChange", func(t *testing.T) { + // testConsulPubSubStatusChange(t, kv) + //}) + //t.Run("TestConsulPubSubNewEvents", func(t *testing.T) { + // testConsulPubSubNewEvents(t, kv) + //}) + //t.Run("TestConsulPubSubNewEventsTimeout", func(t *testing.T) { + // testConsulPubSubNewEventsTimeout(t, kv) + //}) + //t.Run("TestConsulPubSubNewEventsWithIndex", func(t *testing.T) { + // testConsulPubSubNewEventsWithIndex(t, kv) + //}) + //t.Run("TestConsulPubSubNewNodeEvents", func(t *testing.T) { + // testConsulPubSubNewNodeEvents(t, kv) + //}) + //t.Run("TestDeploymentStatusChange", func(t *testing.T) { + // testconsulDeploymentStatusChange(t, kv) + //}) + //t.Run("TestCustomCommandStatusChange", func(t *testing.T) { + // testconsulCustomCommandStatusChange(t, kv) + //}) + //t.Run("TestScalingStatusChange", func(t *testing.T) { + // testconsulScalingStatusChange(t, kv) + //}) + //t.Run("TestWorkflowStatusChange", func(t *testing.T) { + // testconsulWorkflowStatusChange(t, kv) + //}) + //t.Run("TestGetStatusEvents", func(t *testing.T) { + // testconsulGetStatusEvents(t, kv) + //}) t.Run("TestGetLogs", func(t *testing.T) { testconsulGetLogs(t, kv) }) diff --git a/events/events.go b/events/events.go index 1c263ccf1..92062066c 100644 --- a/events/events.go +++ b/events/events.go @@ -18,7 +18,6 @@ import ( "context" "path" "strconv" - "strings" "time" "encoding/json" @@ -44,7 +43,8 @@ func InstanceStatusChange(kv *api.KV, deploymentID, nodeName, instance, status s func PublishAndLogInstanceStatusChange(ctx context.Context, kv *api.KV, deploymentID, nodeName, instance, status string) (string, error) { ctx = AddLogOptionalFields(ctx, LogOptionalFields{NodeID: nodeName, InstanceID: instance}) - id, err := storeStatusUpdateEvent(kv, deploymentID, StatusChangeTypeInstance, nodeName+"\n"+status+"\n"+instance) + e := newEventStatusChange(ctx, StatusChangeTypeInstance, nil, deploymentID, status) + id, err := e.register() if err != nil { return "", err } @@ -65,7 +65,8 @@ func DeploymentStatusChange(kv *api.KV, deploymentID, status string) (string, er // // PublishAndLogDeploymentStatusChange returns the published event id func PublishAndLogDeploymentStatusChange(ctx context.Context, kv *api.KV, deploymentID, status string) (string, error) { - id, err := storeStatusUpdateEvent(kv, deploymentID, StatusChangeTypeDeployment, status) + e := newEventStatusChange(ctx, StatusChangeTypeDeployment, nil, deploymentID, status) + id, err := e.register() if err != nil { return "", err } @@ -89,7 +90,8 @@ func PublishAndLogCustomCommandStatusChange(ctx context.Context, kv *api.KV, dep if ctx == nil { ctx = NewContext(context.Background(), LogOptionalFields{ExecutionID: taskID}) } - id, err := storeStatusUpdateEvent(kv, deploymentID, StatusChangeTypeCustomCommand, taskID+"\n"+status) + e := newEventStatusChange(ctx, StatusChangeTypeCustomCommand, nil, deploymentID, status) + id, err := e.register() if err != nil { return "", err } @@ -113,7 +115,8 @@ func PublishAndLogScalingStatusChange(ctx context.Context, kv *api.KV, deploymen if ctx == nil { ctx = NewContext(context.Background(), LogOptionalFields{ExecutionID: taskID}) } - id, err := storeStatusUpdateEvent(kv, deploymentID, StatusChangeTypeScaling, taskID+"\n"+status) + e := newEventStatusChange(ctx, StatusChangeTypeScaling, nil, deploymentID, status) + id, err := e.register() if err != nil { return "", err } @@ -137,7 +140,8 @@ func PublishAndLogWorkflowStatusChange(ctx context.Context, kv *api.KV, deployme if ctx == nil { ctx = NewContext(context.Background(), LogOptionalFields{ExecutionID: taskID}) } - id, err := storeStatusUpdateEvent(kv, deploymentID, StatusChangeTypeWorkflow, taskID+"\n"+status) + e := newEventStatusChange(ctx, StatusChangeTypeWorkflow, nil, deploymentID, status) + id, err := e.register() if err != nil { return "", err } @@ -145,75 +149,31 @@ func PublishAndLogWorkflowStatusChange(ctx context.Context, kv *api.KV, deployme return id, nil } -// Create a KVPair corresponding to an event and put it to Consul under the event prefix, -// in a sub-tree corresponding to its deployment -// The eventType goes to the KVPair's Flags field -func storeStatusUpdateEvent(kv *api.KV, deploymentID string, eventType StatusChangeType, data string) (string, error) { - now := time.Now().Format(time.RFC3339Nano) - eventsPrefix := path.Join(consulutil.EventsPrefix, deploymentID) - err := consulutil.StoreConsulKeyAsStringWithFlags(path.Join(eventsPrefix, now), data, uint64(eventType)) - if err != nil { - return "", err - } - return now, nil -} - // StatusEvents return a list of events (StatusUpdate instances) for all, or a given deployment -func StatusEvents(kv *api.KV, deploymentID string, waitIndex uint64, timeout time.Duration) ([]EventStatusChange, uint64, error) { - events := make([]EventStatusChange, 0) - +func StatusEvents(kv *api.KV, deploymentID string, waitIndex uint64, timeout time.Duration) ([]json.RawMessage, uint64, error) { + events := make([]json.RawMessage, 0) var eventsPrefix string - var depIDProvided bool if deploymentID != "" { // the returned list of events must correspond to the provided deploymentID eventsPrefix = path.Join(consulutil.EventsPrefix, deploymentID) - depIDProvided = true } else { // the returned list of events must correspond to all the deployments eventsPrefix = path.Join(consulutil.EventsPrefix) - depIDProvided = false } kvps, qm, err := kv.List(eventsPrefix, &api.QueryOptions{WaitIndex: waitIndex, WaitTime: timeout}) if err != nil || qm == nil { return events, 0, err } + log.Debugf("Found %d events before accessing index[%q]", len(kvps), strconv.FormatUint(qm.LastIndex, 10)) for _, kvp := range kvps { if kvp.ModifyIndex <= waitIndex { continue } - var eventTimestamp string - if depIDProvided { - eventTimestamp = strings.TrimPrefix(kvp.Key, eventsPrefix+"/") - } else { - depIDAndTimestamp := strings.Split(strings.TrimPrefix(kvp.Key, eventsPrefix+"/"), "/") - deploymentID = depIDAndTimestamp[0] - eventTimestamp = depIDAndTimestamp[1] - } - - values := strings.Split(string(kvp.Value), "\n") - eventType := StatusChangeType(kvp.Flags) - - switch eventType { - case StatusChangeTypeInstance: - if len(values) != 3 { - return events, qm.LastIndex, errors.Errorf("Unexpected event value %q for event %q", string(kvp.Value), kvp.Key) - } - events = append(events, EventStatusChange{Timestamp: eventTimestamp, Type: eventType.String(), Node: values[0], Status: values[1], Instance: values[2], DeploymentID: deploymentID}) - case StatusChangeTypeDeployment: - if len(values) != 1 { - return events, qm.LastIndex, errors.Errorf("Unexpected event value %q for event %q", string(kvp.Value), kvp.Key) - } - events = append(events, EventStatusChange{Timestamp: eventTimestamp, Type: eventType.String(), Status: values[0], DeploymentID: deploymentID}) - case StatusChangeTypeCustomCommand, StatusChangeTypeScaling, StatusChangeTypeWorkflow, StatusChangeTypeWorkflowStep: - if len(values) != 2 { - return events, qm.LastIndex, errors.Errorf("Unexpected event value %q for event %q", string(kvp.Value), kvp.Key) - } - events = append(events, EventStatusChange{Timestamp: eventTimestamp, Type: eventType.String(), TaskID: values[0], Status: values[1], DeploymentID: deploymentID}) - default: - return events, qm.LastIndex, errors.Errorf("Unsupported event type %d for event %q", kvp.Flags, kvp.Key) - } + //eventType := StatusChangeType(kvp.Flags) + events = append(events, kvp.Value) } + log.Debugf("Found %d events after index", len(events)) return events, qm.LastIndex, nil } diff --git a/events/events_test.go b/events/events_test.go index dad139b0c..0a6c5de58 100644 --- a/events/events_test.go +++ b/events/events_test.go @@ -17,413 +17,410 @@ package events import ( "encoding/json" "fmt" - "path" - "strings" "testing" "time" "github.com/hashicorp/consul/api" "github.com/pkg/errors" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/ystia/yorc/helper/consulutil" "github.com/ystia/yorc/testutil" ) -func testConsulPubSubStatusChange(t *testing.T, kv *api.KV) { - t.Parallel() - deploymentID := testutil.BuildDeploymentID(t) - - var testData = []struct { - node string - instance string - status string - }{ - {"node1", "0", "initial"}, - {"node2", "0", "initial"}, - {"node1", "0", "created"}, - {"node1", "0", "started"}, - {"node2", "0", "created"}, - {"node3", "0", "initial"}, - {"node2", "0", "configured"}, - {"node3", "0", "created"}, - {"node2", "0", "started"}, - {"node3", "0", "error"}, - } - - ids := make([]string, 0) - for _, tc := range testData { - id, err := InstanceStatusChange(kv, deploymentID, tc.node, tc.instance, tc.status) - assert.Nil(t, err) - ids = append(ids, id) - } - prefix := path.Join(consulutil.EventsPrefix, deploymentID) - kvps, _, err := kv.List(prefix, nil) - assert.Nil(t, err) - assert.Len(t, kvps, len(testData)) - - for index, kvp := range kvps { - assert.Equal(t, ids[index], strings.TrimPrefix(kvp.Key, prefix+"/")) - tc := testData[index] - assert.Equal(t, tc.node+"\n"+tc.status+"\n"+tc.instance, string(kvp.Value)) - } -} - -func testConsulPubSubNewEvents(t *testing.T, kv *api.KV) { - // Do not run this test in // as it cause some concurrency issue - // t.Parallel() - deploymentID := testutil.BuildDeploymentID(t) - - nodeName := "node1" - instance := "0" - nodeStatus := "error" - - ready := make(chan struct{}) - - go func() { - i, err := GetStatusEventsIndex(kv, deploymentID) - require.Nil(t, err) - ready <- struct{}{} - events, _, err := StatusEvents(kv, deploymentID, i, 5*time.Minute) - assert.Nil(t, err) - require.Len(t, events, 1) - assert.Equal(t, events[0].Node, nodeName) - assert.Equal(t, events[0].Status, nodeStatus) - assert.Equal(t, events[0].Instance, instance) - }() - <-ready - _, err := InstanceStatusChange(kv, deploymentID, nodeName, instance, nodeStatus) - assert.Nil(t, err) -} - -func testConsulPubSubNewEventsTimeout(t *testing.T, kv *api.KV) { - t.Parallel() - deploymentID := testutil.BuildDeploymentID(t) - - timeout := 25 * time.Millisecond - - t1 := time.Now() - events, _, err := StatusEvents(kv, deploymentID, 1, timeout) - t2 := time.Now() - assert.Nil(t, err) - require.Len(t, events, 0) - assert.WithinDuration(t, t1, t2, timeout+50*time.Millisecond) -} - -func testConsulPubSubNewEventsWithIndex(t *testing.T, kv *api.KV) { - t.Parallel() - deploymentID := testutil.BuildDeploymentID(t) - - var testData = []struct { - node string - instance string - status string - }{ - {"node1", "0", "initial"}, - {"node1", "1", "initial"}, - {"node1", "0", "creating"}, - {"node1", "1", "creating"}, - } - - for _, tc := range testData { - _, err := InstanceStatusChange(kv, deploymentID, tc.node, tc.instance, tc.status) - assert.Nil(t, err) - } - - events, lastIdx, err := StatusEvents(kv, deploymentID, 1, 5*time.Minute) - assert.Nil(t, err) - require.Len(t, events, 4) - for index, event := range events { - assert.Equal(t, testData[index].node, event.Node) - assert.Equal(t, testData[index].instance, event.Instance) - assert.Equal(t, testData[index].status, event.Status) - } - - testData = []struct { - node string - instance string - status string - }{ - {"node1", "0", "created"}, - {"node1", "0", "configuring"}, - {"node1", "0", "configured"}, - } - - for _, tc := range testData { - _, err = InstanceStatusChange(kv, deploymentID, tc.node, tc.instance, tc.status) - assert.Nil(t, err) - } - - events, lastIdx, err = StatusEvents(kv, deploymentID, lastIdx, 5*time.Minute) - assert.Nil(t, err) - require.Len(t, events, 3) - require.NotZero(t, lastIdx) - - for index, event := range events { - assert.Equal(t, testData[index].node, event.Node) - assert.Equal(t, testData[index].instance, event.Instance) - assert.Equal(t, testData[index].status, event.Status) - } -} - -func testConsulPubSubNewNodeEvents(t *testing.T, kv *api.KV) { - t.Parallel() - deploymentID := testutil.BuildDeploymentID(t) - - nodeName := "node1" - instance := "0" - nodeStatus := "error" - - _, err := InstanceStatusChange(kv, deploymentID, nodeName, instance, nodeStatus) - assert.Nil(t, err) - -} - -func testconsulDeploymentStatusChange(t *testing.T, kv *api.KV) { - t.Parallel() - deploymentID := testutil.BuildDeploymentID(t) - type args struct { - kv *api.KV - status string - } - tests := []struct { - name string - args args - wantErr bool - }{ - {"TestStatusInitial", args{kv, "initial"}, false}, - {"TestStatusDepInProgress", args{kv, "deployment_in_progress"}, false}, - {"TestStatusDepDeployed", args{kv, "deployed"}, false}, - } - ids := make([]string, 0) - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := DeploymentStatusChange(tt.args.kv, deploymentID, tt.args.status) - if (err != nil) != tt.wantErr { - t.Errorf("DeploymentStatusChange() error = %v, wantErr %v", err, tt.wantErr) - return - } - if got != "" { - ids = append(ids, got) - } - }) - } - - prefix := path.Join(consulutil.EventsPrefix, deploymentID) - kvps, _, err := kv.List(prefix, nil) - assert.Nil(t, err) - assert.Len(t, kvps, len(tests)) - - for index, kvp := range kvps { - assert.Equal(t, ids[index], strings.TrimPrefix(kvp.Key, prefix+"/")) - tc := tests[index] - assert.Equal(t, tc.args.status, string(kvp.Value)) - } - -} - -func testconsulCustomCommandStatusChange(t *testing.T, kv *api.KV) { - t.Parallel() - deploymentID := testutil.BuildDeploymentID(t) - type args struct { - kv *api.KV - taskID string - status string - } - tests := []struct { - name string - args args - wantErr bool - }{ - {"TestStatusInitial", args{kv, "1", "initial"}, false}, - {"TestStatusDepInProgress", args{kv, "1", "running"}, false}, - {"TestStatusDepDeployed", args{kv, "2", "initial"}, false}, - {"TestStatusDepDeployed", args{kv, "2", "running"}, false}, - {"TestStatusDepDeployed", args{kv, "1", "done"}, false}, - {"TestStatusDepDeployed", args{kv, "2", "done"}, false}, - } - ids := make([]string, 0) - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := CustomCommandStatusChange(tt.args.kv, deploymentID, tt.args.taskID, tt.args.status) - if (err != nil) != tt.wantErr { - t.Errorf("CustomCommandStatusChange() error = %v, wantErr %v", err, tt.wantErr) - return - } - if got != "" { - ids = append(ids, got) - } - }) - } - - prefix := path.Join(consulutil.EventsPrefix, deploymentID) - kvps, _, err := kv.List(prefix, nil) - assert.Nil(t, err) - assert.Len(t, kvps, len(tests)) - - for index, kvp := range kvps { - assert.Equal(t, ids[index], strings.TrimPrefix(kvp.Key, prefix+"/")) - tc := tests[index] - assert.Equal(t, tc.args.taskID+"\n"+tc.args.status, string(kvp.Value)) - } - -} - -func testconsulScalingStatusChange(t *testing.T, kv *api.KV) { - t.Parallel() - deploymentID := testutil.BuildDeploymentID(t) - type args struct { - kv *api.KV - taskID string - status string - } - tests := []struct { - name string - args args - wantErr bool - }{ - {"TestStatusInitial", args{kv, "1", "initial"}, false}, - {"TestStatusDepInProgress", args{kv, "1", "running"}, false}, - {"TestStatusDepDeployed", args{kv, "2", "initial"}, false}, - {"TestStatusDepDeployed", args{kv, "2", "running"}, false}, - {"TestStatusDepDeployed", args{kv, "1", "done"}, false}, - {"TestStatusDepDeployed", args{kv, "2", "done"}, false}, - } - ids := make([]string, 0) - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := ScalingStatusChange(tt.args.kv, deploymentID, tt.args.taskID, tt.args.status) - if (err != nil) != tt.wantErr { - t.Errorf("ScalingStatusChange() error = %v, wantErr %v", err, tt.wantErr) - return - } - if got != "" { - ids = append(ids, got) - } - }) - } - - prefix := path.Join(consulutil.EventsPrefix, deploymentID) - kvps, _, err := kv.List(prefix, nil) - assert.Nil(t, err) - assert.Len(t, kvps, len(tests)) - - for index, kvp := range kvps { - assert.Equal(t, ids[index], strings.TrimPrefix(kvp.Key, prefix+"/")) - tc := tests[index] - assert.Equal(t, tc.args.taskID+"\n"+tc.args.status, string(kvp.Value)) - } - -} - -func testconsulWorkflowStatusChange(t *testing.T, kv *api.KV) { - t.Parallel() - deploymentID := testutil.BuildDeploymentID(t) - type args struct { - kv *api.KV - taskID string - status string - } - tests := []struct { - name string - args args - wantErr bool - }{ - {"TestStatusInitial", args{kv, "1", "initial"}, false}, - {"TestStatusDepInProgress", args{kv, "1", "running"}, false}, - {"TestStatusDepDeployed", args{kv, "2", "initial"}, false}, - {"TestStatusDepDeployed", args{kv, "2", "running"}, false}, - {"TestStatusDepDeployed", args{kv, "1", "done"}, false}, - {"TestStatusDepDeployed", args{kv, "2", "done"}, false}, - } - ids := make([]string, 0) - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := WorkflowStatusChange(tt.args.kv, deploymentID, tt.args.taskID, tt.args.status) - if (err != nil) != tt.wantErr { - t.Errorf("WorkflowStatusChange() error = %v, wantErr %v", err, tt.wantErr) - return - } - if got != "" { - ids = append(ids, got) - } - }) - } - - prefix := path.Join(consulutil.EventsPrefix, deploymentID) - kvps, _, err := kv.List(prefix, nil) - assert.Nil(t, err) - assert.Len(t, kvps, len(tests)) - - for index, kvp := range kvps { - assert.Equal(t, ids[index], strings.TrimPrefix(kvp.Key, prefix+"/")) - tc := tests[index] - assert.Equal(t, tc.args.taskID+"\n"+tc.args.status, string(kvp.Value)) - } - -} - -func testconsulGetStatusEvents(t *testing.T, kv *api.KV) { - t.Parallel() - deploymentID := testutil.BuildDeploymentID(t) - ids := make([]string, 5) - id, err := InstanceStatusChange(kv, deploymentID, "node1", "1", "started") - require.Nil(t, err) - ids[0] = id - id, err = DeploymentStatusChange(kv, deploymentID, "deployed") - require.Nil(t, err) - ids[1] = id - id, err = ScalingStatusChange(kv, deploymentID, "t2", "failed") - require.Nil(t, err) - ids[2] = id - id, err = CustomCommandStatusChange(kv, deploymentID, "t3", "running") - require.Nil(t, err) - ids[3] = id - id, err = WorkflowStatusChange(kv, deploymentID, "t4", "done") - require.Nil(t, err) - ids[4] = id - - events, _, err := StatusEvents(kv, deploymentID, 0, 5*time.Minute) - require.Nil(t, err) - require.Len(t, events, 5) - - require.Equal(t, StatusChangeTypeInstance.String(), events[0].Type) - require.Equal(t, "node1", events[0].Node) - require.Equal(t, "1", events[0].Instance) - require.Equal(t, "started", events[0].Status) - require.Equal(t, ids[0], events[0].Timestamp) - require.Equal(t, "", events[0].TaskID) - - require.Equal(t, StatusChangeTypeDeployment.String(), events[1].Type) - require.Equal(t, "", events[1].Node) - require.Equal(t, "", events[1].Instance) - require.Equal(t, "deployed", events[1].Status) - require.Equal(t, ids[1], events[1].Timestamp) - require.Equal(t, "", events[1].TaskID) - - require.Equal(t, StatusChangeTypeScaling.String(), events[2].Type) - require.Equal(t, "", events[2].Node) - require.Equal(t, "", events[2].Instance) - require.Equal(t, "failed", events[2].Status) - require.Equal(t, ids[2], events[2].Timestamp) - require.Equal(t, "t2", events[2].TaskID) - - require.Equal(t, StatusChangeTypeCustomCommand.String(), events[3].Type) - require.Equal(t, "", events[3].Node) - require.Equal(t, "", events[3].Instance) - require.Equal(t, "running", events[3].Status) - require.Equal(t, ids[3], events[3].Timestamp) - require.Equal(t, "t3", events[3].TaskID) - - require.Equal(t, StatusChangeTypeWorkflow.String(), events[4].Type) - require.Equal(t, "", events[4].Node) - require.Equal(t, "", events[4].Instance) - require.Equal(t, "done", events[4].Status) - require.Equal(t, ids[4], events[4].Timestamp) - require.Equal(t, "t4", events[4].TaskID) - -} +// +//func testConsulPubSubStatusChange(t *testing.T, kv *api.KV) { +// t.Parallel() +// deploymentID := testutil.BuildDeploymentID(t) +// +// var testData = []struct { +// node string +// instance string +// status string +// }{ +// {"node1", "0", "initial"}, +// {"node2", "0", "initial"}, +// {"node1", "0", "created"}, +// {"node1", "0", "started"}, +// {"node2", "0", "created"}, +// {"node3", "0", "initial"}, +// {"node2", "0", "configured"}, +// {"node3", "0", "created"}, +// {"node2", "0", "started"}, +// {"node3", "0", "error"}, +// } +// +// ids := make([]string, 0) +// for _, tc := range testData { +// id, err := InstanceStatusChange(kv, deploymentID, tc.node, tc.instance, tc.status) +// assert.Nil(t, err) +// ids = append(ids, id) +// } +// prefix := path.Join(consulutil.EventsPrefix, deploymentID) +// kvps, _, err := kv.List(prefix, nil) +// assert.Nil(t, err) +// assert.Len(t, kvps, len(testData)) +// +// for index, kvp := range kvps { +// assert.Equal(t, ids[index], strings.TrimPrefix(kvp.Key, prefix+"/")) +// tc := testData[index] +// assert.Equal(t, tc.node+"\n"+tc.status+"\n"+tc.instance, string(kvp.Value)) +// } +//} +// +//func testConsulPubSubNewEvents(t *testing.T, kv *api.KV) { +// // Do not run this test in // as it cause some concurrency issue +// // t.Parallel() +// deploymentID := testutil.BuildDeploymentID(t) +// +// nodeName := "node1" +// instance := "0" +// nodeStatus := "error" +// +// ready := make(chan struct{}) +// +// go func() { +// i, err := GetStatusEventsIndex(kv, deploymentID) +// require.Nil(t, err) +// ready <- struct{}{} +// events, _, err := StatusEvents(kv, deploymentID, i, 5*time.Minute) +// assert.Nil(t, err) +// require.Len(t, events, 1) +// assert.Equal(t, events[0].Node, nodeName) +// assert.Equal(t, events[0].Status, nodeStatus) +// assert.Equal(t, events[0].Instance, instance) +// }() +// <-ready +// _, err := InstanceStatusChange(kv, deploymentID, nodeName, instance, nodeStatus) +// assert.Nil(t, err) +//} +// +//func testConsulPubSubNewEventsTimeout(t *testing.T, kv *api.KV) { +// t.Parallel() +// deploymentID := testutil.BuildDeploymentID(t) +// +// timeout := 25 * time.Millisecond +// +// t1 := time.Now() +// events, _, err := StatusEvents(kv, deploymentID, 1, timeout) +// t2 := time.Now() +// assert.Nil(t, err) +// require.Len(t, events, 0) +// assert.WithinDuration(t, t1, t2, timeout+50*time.Millisecond) +//} +// +//func testConsulPubSubNewEventsWithIndex(t *testing.T, kv *api.KV) { +// t.Parallel() +// deploymentID := testutil.BuildDeploymentID(t) +// +// var testData = []struct { +// node string +// instance string +// status string +// }{ +// {"node1", "0", "initial"}, +// {"node1", "1", "initial"}, +// {"node1", "0", "creating"}, +// {"node1", "1", "creating"}, +// } +// +// for _, tc := range testData { +// _, err := InstanceStatusChange(kv, deploymentID, tc.node, tc.instance, tc.status) +// assert.Nil(t, err) +// } +// +// events, lastIdx, err := StatusEvents(kv, deploymentID, 1, 5*time.Minute) +// assert.Nil(t, err) +// require.Len(t, events, 4) +// for index, event := range events { +// assert.Equal(t, testData[index].node, event.Node) +// assert.Equal(t, testData[index].instance, event.Instance) +// assert.Equal(t, testData[index].status, event.Status) +// } +// +// testData = []struct { +// node string +// instance string +// status string +// }{ +// {"node1", "0", "created"}, +// {"node1", "0", "configuring"}, +// {"node1", "0", "configured"}, +// } +// +// for _, tc := range testData { +// _, err = InstanceStatusChange(kv, deploymentID, tc.node, tc.instance, tc.status) +// assert.Nil(t, err) +// } +// +// events, lastIdx, err = StatusEvents(kv, deploymentID, lastIdx, 5*time.Minute) +// assert.Nil(t, err) +// require.Len(t, events, 3) +// require.NotZero(t, lastIdx) +// +// for index, event := range events { +// assert.Equal(t, testData[index].node, event.Node) +// assert.Equal(t, testData[index].instance, event.Instance) +// assert.Equal(t, testData[index].status, event.Status) +// } +//} +// +//func testConsulPubSubNewNodeEvents(t *testing.T, kv *api.KV) { +// t.Parallel() +// deploymentID := testutil.BuildDeploymentID(t) +// +// nodeName := "node1" +// instance := "0" +// nodeStatus := "error" +// +// _, err := InstanceStatusChange(kv, deploymentID, nodeName, instance, nodeStatus) +// assert.Nil(t, err) +// +//} +// +//func testconsulDeploymentStatusChange(t *testing.T, kv *api.KV) { +// t.Parallel() +// deploymentID := testutil.BuildDeploymentID(t) +// type args struct { +// kv *api.KV +// status string +// } +// tests := []struct { +// name string +// args args +// wantErr bool +// }{ +// {"TestStatusInitial", args{kv, "initial"}, false}, +// {"TestStatusDepInProgress", args{kv, "deployment_in_progress"}, false}, +// {"TestStatusDepDeployed", args{kv, "deployed"}, false}, +// } +// ids := make([]string, 0) +// for _, tt := range tests { +// t.Run(tt.name, func(t *testing.T) { +// got, err := DeploymentStatusChange(tt.args.kv, deploymentID, tt.args.status) +// if (err != nil) != tt.wantErr { +// t.Errorf("DeploymentStatusChange() error = %v, wantErr %v", err, tt.wantErr) +// return +// } +// if got != "" { +// ids = append(ids, got) +// } +// }) +// } +// +// prefix := path.Join(consulutil.EventsPrefix, deploymentID) +// kvps, _, err := kv.List(prefix, nil) +// assert.Nil(t, err) +// assert.Len(t, kvps, len(tests)) +// +// for index, kvp := range kvps { +// assert.Equal(t, ids[index], strings.TrimPrefix(kvp.Key, prefix+"/")) +// tc := tests[index] +// assert.Equal(t, tc.args.status, string(kvp.Value)) +// } +// +//} +// +//func testconsulCustomCommandStatusChange(t *testing.T, kv *api.KV) { +// t.Parallel() +// deploymentID := testutil.BuildDeploymentID(t) +// type args struct { +// kv *api.KV +// taskID string +// status string +// } +// tests := []struct { +// name string +// args args +// wantErr bool +// }{ +// {"TestStatusInitial", args{kv, "1", "initial"}, false}, +// {"TestStatusDepInProgress", args{kv, "1", "running"}, false}, +// {"TestStatusDepDeployed", args{kv, "2", "initial"}, false}, +// {"TestStatusDepDeployed", args{kv, "2", "running"}, false}, +// {"TestStatusDepDeployed", args{kv, "1", "done"}, false}, +// {"TestStatusDepDeployed", args{kv, "2", "done"}, false}, +// } +// ids := make([]string, 0) +// for _, tt := range tests { +// t.Run(tt.name, func(t *testing.T) { +// got, err := CustomCommandStatusChange(tt.args.kv, deploymentID, tt.args.taskID, tt.args.status) +// if (err != nil) != tt.wantErr { +// t.Errorf("CustomCommandStatusChange() error = %v, wantErr %v", err, tt.wantErr) +// return +// } +// if got != "" { +// ids = append(ids, got) +// } +// }) +// } +// +// prefix := path.Join(consulutil.EventsPrefix, deploymentID) +// kvps, _, err := kv.List(prefix, nil) +// assert.Nil(t, err) +// assert.Len(t, kvps, len(tests)) +// +// for index, kvp := range kvps { +// assert.Equal(t, ids[index], strings.TrimPrefix(kvp.Key, prefix+"/")) +// tc := tests[index] +// assert.Equal(t, tc.args.taskID+"\n"+tc.args.status, string(kvp.Value)) +// } +// +//} +// +//func testconsulScalingStatusChange(t *testing.T, kv *api.KV) { +// t.Parallel() +// deploymentID := testutil.BuildDeploymentID(t) +// type args struct { +// kv *api.KV +// taskID string +// status string +// } +// tests := []struct { +// name string +// args args +// wantErr bool +// }{ +// {"TestStatusInitial", args{kv, "1", "initial"}, false}, +// {"TestStatusDepInProgress", args{kv, "1", "running"}, false}, +// {"TestStatusDepDeployed", args{kv, "2", "initial"}, false}, +// {"TestStatusDepDeployed", args{kv, "2", "running"}, false}, +// {"TestStatusDepDeployed", args{kv, "1", "done"}, false}, +// {"TestStatusDepDeployed", args{kv, "2", "done"}, false}, +// } +// ids := make([]string, 0) +// for _, tt := range tests { +// t.Run(tt.name, func(t *testing.T) { +// got, err := ScalingStatusChange(tt.args.kv, deploymentID, tt.args.taskID, tt.args.status) +// if (err != nil) != tt.wantErr { +// t.Errorf("ScalingStatusChange() error = %v, wantErr %v", err, tt.wantErr) +// return +// } +// if got != "" { +// ids = append(ids, got) +// } +// }) +// } +// +// prefix := path.Join(consulutil.EventsPrefix, deploymentID) +// kvps, _, err := kv.List(prefix, nil) +// assert.Nil(t, err) +// assert.Len(t, kvps, len(tests)) +// +// for index, kvp := range kvps { +// assert.Equal(t, ids[index], strings.TrimPrefix(kvp.Key, prefix+"/")) +// tc := tests[index] +// assert.Equal(t, tc.args.taskID+"\n"+tc.args.status, string(kvp.Value)) +// } +// +//} +// +//func testconsulWorkflowStatusChange(t *testing.T, kv *api.KV) { +// t.Parallel() +// deploymentID := testutil.BuildDeploymentID(t) +// type args struct { +// kv *api.KV +// taskID string +// status string +// } +// tests := []struct { +// name string +// args args +// wantErr bool +// }{ +// {"TestStatusInitial", args{kv, "1", "initial"}, false}, +// {"TestStatusDepInProgress", args{kv, "1", "running"}, false}, +// {"TestStatusDepDeployed", args{kv, "2", "initial"}, false}, +// {"TestStatusDepDeployed", args{kv, "2", "running"}, false}, +// {"TestStatusDepDeployed", args{kv, "1", "done"}, false}, +// {"TestStatusDepDeployed", args{kv, "2", "done"}, false}, +// } +// ids := make([]string, 0) +// for _, tt := range tests { +// t.Run(tt.name, func(t *testing.T) { +// got, err := WorkflowStatusChange(tt.args.kv, deploymentID, tt.args.taskID, tt.args.status) +// if (err != nil) != tt.wantErr { +// t.Errorf("WorkflowStatusChange() error = %v, wantErr %v", err, tt.wantErr) +// return +// } +// if got != "" { +// ids = append(ids, got) +// } +// }) +// } +// +// prefix := path.Join(consulutil.EventsPrefix, deploymentID) +// kvps, _, err := kv.List(prefix, nil) +// assert.Nil(t, err) +// assert.Len(t, kvps, len(tests)) +// +// for index, kvp := range kvps { +// assert.Equal(t, ids[index], strings.TrimPrefix(kvp.Key, prefix+"/")) +// tc := tests[index] +// assert.Equal(t, tc.args.taskID+"\n"+tc.args.status, string(kvp.Value)) +// } +// +//} +// +//func testconsulGetStatusEvents(t *testing.T, kv *api.KV) { +// t.Parallel() +// deploymentID := testutil.BuildDeploymentID(t) +// ids := make([]string, 5) +// id, err := InstanceStatusChange(kv, deploymentID, "node1", "1", "started") +// require.Nil(t, err) +// ids[0] = id +// id, err = DeploymentStatusChange(kv, deploymentID, "deployed") +// require.Nil(t, err) +// ids[1] = id +// id, err = ScalingStatusChange(kv, deploymentID, "t2", "failed") +// require.Nil(t, err) +// ids[2] = id +// id, err = CustomCommandStatusChange(kv, deploymentID, "t3", "running") +// require.Nil(t, err) +// ids[3] = id +// id, err = WorkflowStatusChange(kv, deploymentID, "t4", "done") +// require.Nil(t, err) +// ids[4] = id +// +// events, _, err := StatusEvents(kv, deploymentID, 0, 5*time.Minute) +// require.Nil(t, err) +// require.Len(t, events, 5) +// +// require.Equal(t, StatusChangeTypeInstance.String(), events[0].Type) +// require.Equal(t, "node1", events[0].Node) +// require.Equal(t, "1", events[0].Instance) +// require.Equal(t, "started", events[0].Status) +// require.Equal(t, ids[0], events[0].Timestamp) +// require.Equal(t, "", events[0].TaskID) +// +// require.Equal(t, StatusChangeTypeDeployment.String(), events[1].Type) +// require.Equal(t, "", events[1].Node) +// require.Equal(t, "", events[1].Instance) +// require.Equal(t, "deployed", events[1].Status) +// require.Equal(t, ids[1], events[1].Timestamp) +// require.Equal(t, "", events[1].TaskID) +// +// require.Equal(t, StatusChangeTypeScaling.String(), events[2].Type) +// require.Equal(t, "", events[2].Node) +// require.Equal(t, "", events[2].Instance) +// require.Equal(t, "failed", events[2].Status) +// require.Equal(t, ids[2], events[2].Timestamp) +// require.Equal(t, "t2", events[2].TaskID) +// +// require.Equal(t, StatusChangeTypeCustomCommand.String(), events[3].Type) +// require.Equal(t, "", events[3].Node) +// require.Equal(t, "", events[3].Instance) +// require.Equal(t, "running", events[3].Status) +// require.Equal(t, ids[3], events[3].Timestamp) +// require.Equal(t, "t3", events[3].TaskID) +// +// require.Equal(t, StatusChangeTypeWorkflow.String(), events[4].Type) +// require.Equal(t, "", events[4].Node) +// require.Equal(t, "", events[4].Instance) +// require.Equal(t, "done", events[4].Status) +// require.Equal(t, ids[4], events[4].Timestamp) +// require.Equal(t, "t4", events[4].TaskID) +// +//} func testconsulGetLogs(t *testing.T, kv *api.KV) { t.Parallel() diff --git a/events/log_entries.go b/events/log_entries.go index 16b675426..b25db3ee7 100644 --- a/events/log_entries.go +++ b/events/log_entries.go @@ -42,10 +42,10 @@ type LogEntryDraft struct { additionalInfo LogOptionalFields } -// LogOptionalFields are log's additional info +// LogOptionalFields are log's additional additionalInfo type LogOptionalFields map[FieldType]interface{} -// FieldType is allowed/expected additional info types +// FieldType is allowed/expected additional additionalInfo types type FieldType int const ( @@ -220,7 +220,7 @@ func (e LogEntry) toFlatMap() map[string]interface{} { // order flatMap["timestamp"] = e.timestamp.Format(time.RFC3339Nano) - // NewLogEntry additional info + // NewLogEntry additional additionalInfo for k, v := range e.additionalInfo { flatMap[k.String()] = v } diff --git a/events/structs.go b/events/structs.go index b86d02468..b30296ffc 100644 --- a/events/structs.go +++ b/events/structs.go @@ -14,6 +14,15 @@ package events +import ( + "context" + "encoding/json" + "github.com/ystia/yorc/helper/consulutil" + "github.com/ystia/yorc/log" + "path" + "time" +) + //go:generate go-enum -f=structs.go --lower // LogLevel x ENUM( @@ -26,17 +35,64 @@ package events // ) type StatusChangeType int -// EventOptionalInfo is event's additional info -type EventOptionalInfo map[string]interface{} +// EventAdditionalInfo allows to provide custom/specific additional information for event +type EventAdditionalInfo map[string]interface{} // EventStatusChange represents status change event type EventStatusChange struct { - Timestamp string `json:"timestamp"` - Type string `json:"type"` - Node string `json:"node,omitempty"` - Instance string `json:"instance,omitempty"` - TaskID string `json:"task_id,omitempty"` - DeploymentID string `json:"deployment_id"` - Status string `json:"status"` - AdditionalInfo EventOptionalInfo + timestamp string + eventType StatusChangeType + deploymentID string + status string + additionalInfo EventAdditionalInfo +} + +// Create a KVPair corresponding to an event and put it to Consul under the event prefix, +// in a sub-tree corresponding to its deployment +// The eventType goes to the KVPair's Flags field +// The content is JSON format +func (e EventStatusChange) register() (string, error) { + e.timestamp = time.Now().Format(time.RFC3339Nano) + eventsPrefix := path.Join(consulutil.EventsPrefix, e.deploymentID) + + // For presentation purpose, each field is in flat json object + flat := e.flat() + b, err := json.Marshal(flat) + if err != nil { + log.Printf("Failed to marshal event [%+v]: due to error:%+v", e, err) + } + + //FIXME Do we still need to store event type as a flag ? + err = consulutil.StoreConsulKeyWithFlags(path.Join(eventsPrefix, e.timestamp), b, uint64(e.eventType)) + if err != nil { + return "", err + } + return e.timestamp, nil +} + +func (e EventStatusChange) flat() map[string]interface{} { + flat := make(map[string]interface{}) + + flat["deploymentId"] = e.deploymentID + flat["status"] = e.status + flat["timestamp"] = e.timestamp + for k, v := range e.additionalInfo { + flat[k] = v + } + return flat +} + +// newEventStatusChange allows to create a new EventStatusChange pointer by retrieving context log information +func newEventStatusChange(ctx context.Context, eventType StatusChangeType, info EventAdditionalInfo, deploymentID, status string) *EventStatusChange { + event := &EventStatusChange{additionalInfo: info, eventType: eventType, status: status, deploymentID: deploymentID} + if event.additionalInfo == nil { + event.additionalInfo = make(EventAdditionalInfo) + } + logInfo, ok := FromContext(ctx) + if ok { + for k, v := range logInfo { + event.additionalInfo[k.String()] = v + } + } + return event } diff --git a/rest/structs.go b/rest/structs.go index 5e824b7f8..136fbf994 100644 --- a/rest/structs.go +++ b/rest/structs.go @@ -20,7 +20,6 @@ import ( "bytes" "encoding/json" - "github.com/ystia/yorc/events" "github.com/ystia/yorc/prov/hostspool" "github.com/ystia/yorc/registry" "github.com/ystia/yorc/tosca" @@ -102,8 +101,8 @@ type DeploymentsCollection struct { // EventsCollection is a collection of instances status change events type EventsCollection struct { - Events []events.EventStatusChange `json:"events"` - LastIndex uint64 `json:"last_index"` + Events []json.RawMessage `json:"events"` + LastIndex uint64 `json:"last_index"` } // LogsCollection is a collection of logs events From fc49b23e809fb96c4455119d2f504ff0d1b157cf Mon Sep 17 00:00:00 2001 From: benoist-s Date: Thu, 29 Nov 2018 17:20:53 +0100 Subject: [PATCH 04/19] Add func allowing publishing event status for workflow step and Alien task changes Update tests --- events/consul_test.go | 60 +-- events/events.go | 90 ++++- events/events_test.go | 818 +++++++++++++++++++++-------------------- events/structs.go | 136 +++++-- events/structs_enum.go | 7 +- 5 files changed, 653 insertions(+), 458 deletions(-) diff --git a/events/consul_test.go b/events/consul_test.go index 3d8efa2a5..fa5f72caa 100644 --- a/events/consul_test.go +++ b/events/consul_test.go @@ -27,36 +27,36 @@ func TestRunConsulEventsPackageTests(t *testing.T) { defer srv.Stop() t.Run("groupEvents", func(t *testing.T) { - //t.Run("TestConsulPubSubStatusChange", func(t *testing.T) { - // testConsulPubSubStatusChange(t, kv) - //}) - //t.Run("TestConsulPubSubNewEvents", func(t *testing.T) { - // testConsulPubSubNewEvents(t, kv) - //}) - //t.Run("TestConsulPubSubNewEventsTimeout", func(t *testing.T) { - // testConsulPubSubNewEventsTimeout(t, kv) - //}) - //t.Run("TestConsulPubSubNewEventsWithIndex", func(t *testing.T) { - // testConsulPubSubNewEventsWithIndex(t, kv) - //}) - //t.Run("TestConsulPubSubNewNodeEvents", func(t *testing.T) { - // testConsulPubSubNewNodeEvents(t, kv) - //}) - //t.Run("TestDeploymentStatusChange", func(t *testing.T) { - // testconsulDeploymentStatusChange(t, kv) - //}) - //t.Run("TestCustomCommandStatusChange", func(t *testing.T) { - // testconsulCustomCommandStatusChange(t, kv) - //}) - //t.Run("TestScalingStatusChange", func(t *testing.T) { - // testconsulScalingStatusChange(t, kv) - //}) - //t.Run("TestWorkflowStatusChange", func(t *testing.T) { - // testconsulWorkflowStatusChange(t, kv) - //}) - //t.Run("TestGetStatusEvents", func(t *testing.T) { - // testconsulGetStatusEvents(t, kv) - //}) + t.Run("TestConsulPubSubStatusChange", func(t *testing.T) { + testConsulPubSubStatusChange(t, kv) + }) + t.Run("TestConsulPubSubNewEvents", func(t *testing.T) { + testConsulPubSubNewEvents(t, kv) + }) + t.Run("TestConsulPubSubNewEventsTimeout", func(t *testing.T) { + testConsulPubSubNewEventsTimeout(t, kv) + }) + t.Run("TestConsulPubSubNewEventsWithIndex", func(t *testing.T) { + testConsulPubSubNewEventsWithIndex(t, kv) + }) + t.Run("TestConsulPubSubNewNodeEvents", func(t *testing.T) { + testConsulPubSubNewNodeEvents(t, kv) + }) + t.Run("TestDeploymentStatusChange", func(t *testing.T) { + testconsulDeploymentStatusChange(t, kv) + }) + t.Run("TestCustomCommandStatusChange", func(t *testing.T) { + testconsulCustomCommandStatusChange(t, kv) + }) + t.Run("TestScalingStatusChange", func(t *testing.T) { + testconsulScalingStatusChange(t, kv) + }) + t.Run("TestWorkflowStatusChange", func(t *testing.T) { + testconsulWorkflowStatusChange(t, kv) + }) + t.Run("TestGetStatusEvents", func(t *testing.T) { + testconsulGetStatusEvents(t, kv) + }) t.Run("TestGetLogs", func(t *testing.T) { testconsulGetLogs(t, kv) }) diff --git a/events/events.go b/events/events.go index 92062066c..55fad7c1b 100644 --- a/events/events.go +++ b/events/events.go @@ -43,7 +43,13 @@ func InstanceStatusChange(kv *api.KV, deploymentID, nodeName, instance, status s func PublishAndLogInstanceStatusChange(ctx context.Context, kv *api.KV, deploymentID, nodeName, instance, status string) (string, error) { ctx = AddLogOptionalFields(ctx, LogOptionalFields{NodeID: nodeName, InstanceID: instance}) - e := newEventStatusChange(ctx, StatusChangeTypeInstance, nil, deploymentID, status) + info := make(Info) + info[infoNodeID] = nodeName + info[infoInstanceID] = instance + e, err := newStatusChange(StatusChangeTypeInstance, info, deploymentID, status) + if err != nil { + return "", err + } id, err := e.register() if err != nil { return "", err @@ -65,7 +71,10 @@ func DeploymentStatusChange(kv *api.KV, deploymentID, status string) (string, er // // PublishAndLogDeploymentStatusChange returns the published event id func PublishAndLogDeploymentStatusChange(ctx context.Context, kv *api.KV, deploymentID, status string) (string, error) { - e := newEventStatusChange(ctx, StatusChangeTypeDeployment, nil, deploymentID, status) + e, err := newStatusChange(StatusChangeTypeDeployment, nil, deploymentID, status) + if err != nil { + return "", err + } id, err := e.register() if err != nil { return "", err @@ -90,7 +99,12 @@ func PublishAndLogCustomCommandStatusChange(ctx context.Context, kv *api.KV, dep if ctx == nil { ctx = NewContext(context.Background(), LogOptionalFields{ExecutionID: taskID}) } - e := newEventStatusChange(ctx, StatusChangeTypeCustomCommand, nil, deploymentID, status) + info := make(Info) + info[infoExecutionID] = taskID + e, err := newStatusChange(StatusChangeTypeCustomCommand, info, deploymentID, status) + if err != nil { + return "", err + } id, err := e.register() if err != nil { return "", err @@ -115,7 +129,12 @@ func PublishAndLogScalingStatusChange(ctx context.Context, kv *api.KV, deploymen if ctx == nil { ctx = NewContext(context.Background(), LogOptionalFields{ExecutionID: taskID}) } - e := newEventStatusChange(ctx, StatusChangeTypeScaling, nil, deploymentID, status) + info := make(Info) + info[infoExecutionID] = taskID + e, err := newStatusChange(StatusChangeTypeScaling, info, deploymentID, status) + if err != nil { + return "", err + } id, err := e.register() if err != nil { return "", err @@ -133,6 +152,62 @@ func WorkflowStatusChange(kv *api.KV, deploymentID, taskID, status string) (stri return PublishAndLogWorkflowStatusChange(nil, kv, deploymentID, taskID, status) } +// PublishAndLogWorkflowStepStatusChange publishes a status change for a workflow step execution and log this change into the log API +// +// PublishAndLogWorkflowStepStatusChange returns the published event id +func PublishAndLogWorkflowStepStatusChange(ctx context.Context, kv *api.KV, deploymentID, taskID, workflowName, nodeName, stepName, operationName, targetNodeID, targetInstanceID, status string) (string, error) { + if ctx == nil { + ctx = NewContext(context.Background(), LogOptionalFields{ExecutionID: taskID}) + } + info := make(Info) + info[infoExecutionID] = taskID + info[infoWorkFlowID] = workflowName + info[infoNodeID] = nodeName + info[infoStepID] = stepName + info[infoOperationName] = operationName + info[infoTargetNodeID] = targetNodeID + info[infoTargetInstanceID] = targetInstanceID + e, err := newStatusChange(StatusChangeTypeWorkflowStep, info, deploymentID, status) + if err != nil { + return "", err + } + id, err := e.register() + if err != nil { + return "", err + } + WithContextOptionalFields(ctx).NewLogEntry(LogLevelINFO, deploymentID).Registerf("Status for workflow task %q changed to %q", taskID, status) + return id, nil +} + +// PublishAndLogAlienTaskStatusChange publishes a status change for a task execution and log this change into the log API +// +// PublishAndLogAlienTaskStatusChange returns the published event id +func PublishAndLogAlienTaskStatusChange(ctx context.Context, kv *api.KV, deploymentID, taskID, taskExecutionID, workflowName, nodeName, stepName, operationName, targetNodeID, targetInstanceID, status string) (string, error) { + if ctx == nil { + ctx = NewContext(context.Background(), LogOptionalFields{ExecutionID: taskID}) + } + info := make(Info) + info[infoExecutionID] = taskID + // Warning: Alien task corresponds to what we call taskExecution + info[infoAlienTaskID] = taskExecutionID + info[infoWorkFlowID] = workflowName + info[infoNodeID] = nodeName + info[infoWorkflowStepID] = stepName + info[infoOperationName] = operationName + info[infoTargetNodeID] = targetNodeID + info[infoTargetInstanceID] = targetInstanceID + e, err := newStatusChange(StatusChangeTypeWorkflow, info, deploymentID, status) + if err != nil { + return "", err + } + id, err := e.register() + if err != nil { + return "", err + } + WithContextOptionalFields(ctx).NewLogEntry(LogLevelINFO, deploymentID).Registerf("Status for workflow task %q changed to %q", taskID, status) + return id, nil +} + // PublishAndLogWorkflowStatusChange publishes a status change for a workflow task and log this change into the log API // // PublishAndLogWorkflowStatusChange returns the published event id @@ -140,7 +215,12 @@ func PublishAndLogWorkflowStatusChange(ctx context.Context, kv *api.KV, deployme if ctx == nil { ctx = NewContext(context.Background(), LogOptionalFields{ExecutionID: taskID}) } - e := newEventStatusChange(ctx, StatusChangeTypeWorkflow, nil, deploymentID, status) + info := make(Info) + info[infoExecutionID] = taskID + e, err := newStatusChange(StatusChangeTypeWorkflow, info, deploymentID, status) + if err != nil { + return "", err + } id, err := e.register() if err != nil { return "", err diff --git a/events/events_test.go b/events/events_test.go index 0a6c5de58..64ae0545b 100644 --- a/events/events_test.go +++ b/events/events_test.go @@ -24,403 +24,433 @@ import ( "github.com/pkg/errors" "github.com/stretchr/testify/require" + "github.com/stretchr/testify/assert" + "github.com/ystia/yorc/helper/consulutil" "github.com/ystia/yorc/testutil" + "path" + "strings" ) -// -//func testConsulPubSubStatusChange(t *testing.T, kv *api.KV) { -// t.Parallel() -// deploymentID := testutil.BuildDeploymentID(t) -// -// var testData = []struct { -// node string -// instance string -// status string -// }{ -// {"node1", "0", "initial"}, -// {"node2", "0", "initial"}, -// {"node1", "0", "created"}, -// {"node1", "0", "started"}, -// {"node2", "0", "created"}, -// {"node3", "0", "initial"}, -// {"node2", "0", "configured"}, -// {"node3", "0", "created"}, -// {"node2", "0", "started"}, -// {"node3", "0", "error"}, -// } -// -// ids := make([]string, 0) -// for _, tc := range testData { -// id, err := InstanceStatusChange(kv, deploymentID, tc.node, tc.instance, tc.status) -// assert.Nil(t, err) -// ids = append(ids, id) -// } -// prefix := path.Join(consulutil.EventsPrefix, deploymentID) -// kvps, _, err := kv.List(prefix, nil) -// assert.Nil(t, err) -// assert.Len(t, kvps, len(testData)) -// -// for index, kvp := range kvps { -// assert.Equal(t, ids[index], strings.TrimPrefix(kvp.Key, prefix+"/")) -// tc := testData[index] -// assert.Equal(t, tc.node+"\n"+tc.status+"\n"+tc.instance, string(kvp.Value)) -// } -//} -// -//func testConsulPubSubNewEvents(t *testing.T, kv *api.KV) { -// // Do not run this test in // as it cause some concurrency issue -// // t.Parallel() -// deploymentID := testutil.BuildDeploymentID(t) -// -// nodeName := "node1" -// instance := "0" -// nodeStatus := "error" -// -// ready := make(chan struct{}) -// -// go func() { -// i, err := GetStatusEventsIndex(kv, deploymentID) -// require.Nil(t, err) -// ready <- struct{}{} -// events, _, err := StatusEvents(kv, deploymentID, i, 5*time.Minute) -// assert.Nil(t, err) -// require.Len(t, events, 1) -// assert.Equal(t, events[0].Node, nodeName) -// assert.Equal(t, events[0].Status, nodeStatus) -// assert.Equal(t, events[0].Instance, instance) -// }() -// <-ready -// _, err := InstanceStatusChange(kv, deploymentID, nodeName, instance, nodeStatus) -// assert.Nil(t, err) -//} -// -//func testConsulPubSubNewEventsTimeout(t *testing.T, kv *api.KV) { -// t.Parallel() -// deploymentID := testutil.BuildDeploymentID(t) -// -// timeout := 25 * time.Millisecond -// -// t1 := time.Now() -// events, _, err := StatusEvents(kv, deploymentID, 1, timeout) -// t2 := time.Now() -// assert.Nil(t, err) -// require.Len(t, events, 0) -// assert.WithinDuration(t, t1, t2, timeout+50*time.Millisecond) -//} -// -//func testConsulPubSubNewEventsWithIndex(t *testing.T, kv *api.KV) { -// t.Parallel() -// deploymentID := testutil.BuildDeploymentID(t) -// -// var testData = []struct { -// node string -// instance string -// status string -// }{ -// {"node1", "0", "initial"}, -// {"node1", "1", "initial"}, -// {"node1", "0", "creating"}, -// {"node1", "1", "creating"}, -// } -// -// for _, tc := range testData { -// _, err := InstanceStatusChange(kv, deploymentID, tc.node, tc.instance, tc.status) -// assert.Nil(t, err) -// } -// -// events, lastIdx, err := StatusEvents(kv, deploymentID, 1, 5*time.Minute) -// assert.Nil(t, err) -// require.Len(t, events, 4) -// for index, event := range events { -// assert.Equal(t, testData[index].node, event.Node) -// assert.Equal(t, testData[index].instance, event.Instance) -// assert.Equal(t, testData[index].status, event.Status) -// } -// -// testData = []struct { -// node string -// instance string -// status string -// }{ -// {"node1", "0", "created"}, -// {"node1", "0", "configuring"}, -// {"node1", "0", "configured"}, -// } -// -// for _, tc := range testData { -// _, err = InstanceStatusChange(kv, deploymentID, tc.node, tc.instance, tc.status) -// assert.Nil(t, err) -// } -// -// events, lastIdx, err = StatusEvents(kv, deploymentID, lastIdx, 5*time.Minute) -// assert.Nil(t, err) -// require.Len(t, events, 3) -// require.NotZero(t, lastIdx) -// -// for index, event := range events { -// assert.Equal(t, testData[index].node, event.Node) -// assert.Equal(t, testData[index].instance, event.Instance) -// assert.Equal(t, testData[index].status, event.Status) -// } -//} -// -//func testConsulPubSubNewNodeEvents(t *testing.T, kv *api.KV) { -// t.Parallel() -// deploymentID := testutil.BuildDeploymentID(t) -// -// nodeName := "node1" -// instance := "0" -// nodeStatus := "error" -// -// _, err := InstanceStatusChange(kv, deploymentID, nodeName, instance, nodeStatus) -// assert.Nil(t, err) -// -//} -// -//func testconsulDeploymentStatusChange(t *testing.T, kv *api.KV) { -// t.Parallel() -// deploymentID := testutil.BuildDeploymentID(t) -// type args struct { -// kv *api.KV -// status string -// } -// tests := []struct { -// name string -// args args -// wantErr bool -// }{ -// {"TestStatusInitial", args{kv, "initial"}, false}, -// {"TestStatusDepInProgress", args{kv, "deployment_in_progress"}, false}, -// {"TestStatusDepDeployed", args{kv, "deployed"}, false}, -// } -// ids := make([]string, 0) -// for _, tt := range tests { -// t.Run(tt.name, func(t *testing.T) { -// got, err := DeploymentStatusChange(tt.args.kv, deploymentID, tt.args.status) -// if (err != nil) != tt.wantErr { -// t.Errorf("DeploymentStatusChange() error = %v, wantErr %v", err, tt.wantErr) -// return -// } -// if got != "" { -// ids = append(ids, got) -// } -// }) -// } -// -// prefix := path.Join(consulutil.EventsPrefix, deploymentID) -// kvps, _, err := kv.List(prefix, nil) -// assert.Nil(t, err) -// assert.Len(t, kvps, len(tests)) -// -// for index, kvp := range kvps { -// assert.Equal(t, ids[index], strings.TrimPrefix(kvp.Key, prefix+"/")) -// tc := tests[index] -// assert.Equal(t, tc.args.status, string(kvp.Value)) -// } -// -//} -// -//func testconsulCustomCommandStatusChange(t *testing.T, kv *api.KV) { -// t.Parallel() -// deploymentID := testutil.BuildDeploymentID(t) -// type args struct { -// kv *api.KV -// taskID string -// status string -// } -// tests := []struct { -// name string -// args args -// wantErr bool -// }{ -// {"TestStatusInitial", args{kv, "1", "initial"}, false}, -// {"TestStatusDepInProgress", args{kv, "1", "running"}, false}, -// {"TestStatusDepDeployed", args{kv, "2", "initial"}, false}, -// {"TestStatusDepDeployed", args{kv, "2", "running"}, false}, -// {"TestStatusDepDeployed", args{kv, "1", "done"}, false}, -// {"TestStatusDepDeployed", args{kv, "2", "done"}, false}, -// } -// ids := make([]string, 0) -// for _, tt := range tests { -// t.Run(tt.name, func(t *testing.T) { -// got, err := CustomCommandStatusChange(tt.args.kv, deploymentID, tt.args.taskID, tt.args.status) -// if (err != nil) != tt.wantErr { -// t.Errorf("CustomCommandStatusChange() error = %v, wantErr %v", err, tt.wantErr) -// return -// } -// if got != "" { -// ids = append(ids, got) -// } -// }) -// } -// -// prefix := path.Join(consulutil.EventsPrefix, deploymentID) -// kvps, _, err := kv.List(prefix, nil) -// assert.Nil(t, err) -// assert.Len(t, kvps, len(tests)) -// -// for index, kvp := range kvps { -// assert.Equal(t, ids[index], strings.TrimPrefix(kvp.Key, prefix+"/")) -// tc := tests[index] -// assert.Equal(t, tc.args.taskID+"\n"+tc.args.status, string(kvp.Value)) -// } -// -//} -// -//func testconsulScalingStatusChange(t *testing.T, kv *api.KV) { -// t.Parallel() -// deploymentID := testutil.BuildDeploymentID(t) -// type args struct { -// kv *api.KV -// taskID string -// status string -// } -// tests := []struct { -// name string -// args args -// wantErr bool -// }{ -// {"TestStatusInitial", args{kv, "1", "initial"}, false}, -// {"TestStatusDepInProgress", args{kv, "1", "running"}, false}, -// {"TestStatusDepDeployed", args{kv, "2", "initial"}, false}, -// {"TestStatusDepDeployed", args{kv, "2", "running"}, false}, -// {"TestStatusDepDeployed", args{kv, "1", "done"}, false}, -// {"TestStatusDepDeployed", args{kv, "2", "done"}, false}, -// } -// ids := make([]string, 0) -// for _, tt := range tests { -// t.Run(tt.name, func(t *testing.T) { -// got, err := ScalingStatusChange(tt.args.kv, deploymentID, tt.args.taskID, tt.args.status) -// if (err != nil) != tt.wantErr { -// t.Errorf("ScalingStatusChange() error = %v, wantErr %v", err, tt.wantErr) -// return -// } -// if got != "" { -// ids = append(ids, got) -// } -// }) -// } -// -// prefix := path.Join(consulutil.EventsPrefix, deploymentID) -// kvps, _, err := kv.List(prefix, nil) -// assert.Nil(t, err) -// assert.Len(t, kvps, len(tests)) -// -// for index, kvp := range kvps { -// assert.Equal(t, ids[index], strings.TrimPrefix(kvp.Key, prefix+"/")) -// tc := tests[index] -// assert.Equal(t, tc.args.taskID+"\n"+tc.args.status, string(kvp.Value)) -// } -// -//} -// -//func testconsulWorkflowStatusChange(t *testing.T, kv *api.KV) { -// t.Parallel() -// deploymentID := testutil.BuildDeploymentID(t) -// type args struct { -// kv *api.KV -// taskID string -// status string -// } -// tests := []struct { -// name string -// args args -// wantErr bool -// }{ -// {"TestStatusInitial", args{kv, "1", "initial"}, false}, -// {"TestStatusDepInProgress", args{kv, "1", "running"}, false}, -// {"TestStatusDepDeployed", args{kv, "2", "initial"}, false}, -// {"TestStatusDepDeployed", args{kv, "2", "running"}, false}, -// {"TestStatusDepDeployed", args{kv, "1", "done"}, false}, -// {"TestStatusDepDeployed", args{kv, "2", "done"}, false}, -// } -// ids := make([]string, 0) -// for _, tt := range tests { -// t.Run(tt.name, func(t *testing.T) { -// got, err := WorkflowStatusChange(tt.args.kv, deploymentID, tt.args.taskID, tt.args.status) -// if (err != nil) != tt.wantErr { -// t.Errorf("WorkflowStatusChange() error = %v, wantErr %v", err, tt.wantErr) -// return -// } -// if got != "" { -// ids = append(ids, got) -// } -// }) -// } -// -// prefix := path.Join(consulutil.EventsPrefix, deploymentID) -// kvps, _, err := kv.List(prefix, nil) -// assert.Nil(t, err) -// assert.Len(t, kvps, len(tests)) -// -// for index, kvp := range kvps { -// assert.Equal(t, ids[index], strings.TrimPrefix(kvp.Key, prefix+"/")) -// tc := tests[index] -// assert.Equal(t, tc.args.taskID+"\n"+tc.args.status, string(kvp.Value)) -// } -// -//} -// -//func testconsulGetStatusEvents(t *testing.T, kv *api.KV) { -// t.Parallel() -// deploymentID := testutil.BuildDeploymentID(t) -// ids := make([]string, 5) -// id, err := InstanceStatusChange(kv, deploymentID, "node1", "1", "started") -// require.Nil(t, err) -// ids[0] = id -// id, err = DeploymentStatusChange(kv, deploymentID, "deployed") -// require.Nil(t, err) -// ids[1] = id -// id, err = ScalingStatusChange(kv, deploymentID, "t2", "failed") -// require.Nil(t, err) -// ids[2] = id -// id, err = CustomCommandStatusChange(kv, deploymentID, "t3", "running") -// require.Nil(t, err) -// ids[3] = id -// id, err = WorkflowStatusChange(kv, deploymentID, "t4", "done") -// require.Nil(t, err) -// ids[4] = id -// -// events, _, err := StatusEvents(kv, deploymentID, 0, 5*time.Minute) -// require.Nil(t, err) -// require.Len(t, events, 5) -// -// require.Equal(t, StatusChangeTypeInstance.String(), events[0].Type) -// require.Equal(t, "node1", events[0].Node) -// require.Equal(t, "1", events[0].Instance) -// require.Equal(t, "started", events[0].Status) -// require.Equal(t, ids[0], events[0].Timestamp) -// require.Equal(t, "", events[0].TaskID) -// -// require.Equal(t, StatusChangeTypeDeployment.String(), events[1].Type) -// require.Equal(t, "", events[1].Node) -// require.Equal(t, "", events[1].Instance) -// require.Equal(t, "deployed", events[1].Status) -// require.Equal(t, ids[1], events[1].Timestamp) -// require.Equal(t, "", events[1].TaskID) -// -// require.Equal(t, StatusChangeTypeScaling.String(), events[2].Type) -// require.Equal(t, "", events[2].Node) -// require.Equal(t, "", events[2].Instance) -// require.Equal(t, "failed", events[2].Status) -// require.Equal(t, ids[2], events[2].Timestamp) -// require.Equal(t, "t2", events[2].TaskID) -// -// require.Equal(t, StatusChangeTypeCustomCommand.String(), events[3].Type) -// require.Equal(t, "", events[3].Node) -// require.Equal(t, "", events[3].Instance) -// require.Equal(t, "running", events[3].Status) -// require.Equal(t, ids[3], events[3].Timestamp) -// require.Equal(t, "t3", events[3].TaskID) -// -// require.Equal(t, StatusChangeTypeWorkflow.String(), events[4].Type) -// require.Equal(t, "", events[4].Node) -// require.Equal(t, "", events[4].Instance) -// require.Equal(t, "done", events[4].Status) -// require.Equal(t, ids[4], events[4].Timestamp) -// require.Equal(t, "t4", events[4].TaskID) -// -//} +func testConsulPubSubStatusChange(t *testing.T, kv *api.KV) { + t.Parallel() + deploymentID := testutil.BuildDeploymentID(t) + + var testData = []struct { + node string + instance string + status string + }{ + {"node1", "0", "initial"}, + {"node2", "0", "initial"}, + {"node1", "0", "created"}, + {"node1", "0", "started"}, + {"node2", "0", "created"}, + {"node3", "0", "initial"}, + {"node2", "0", "configured"}, + {"node3", "0", "created"}, + {"node2", "0", "started"}, + {"node3", "0", "error"}, + } + + ids := make([]string, 0) + for _, tc := range testData { + id, err := InstanceStatusChange(kv, deploymentID, tc.node, tc.instance, tc.status) + assert.Nil(t, err) + ids = append(ids, id) + } + prefix := path.Join(consulutil.EventsPrefix, deploymentID) + kvps, _, err := kv.List(prefix, nil) + assert.Nil(t, err) + assert.Len(t, kvps, len(testData)) + + for index, kvp := range kvps { + assert.Equal(t, ids[index], strings.TrimPrefix(kvp.Key, prefix+"/")) + tc := testData[index] + res := toStatusChangeMap(t, string(kvp.Value)) + assert.Equal(t, tc.node, res[infoNodeID.String()], "unexpected node value for statusChange") + assert.Equal(t, tc.status, res["status"], "unexpected status value for statusChange") + assert.Equal(t, deploymentID, res["deploymentId"], "unexpected deploymentID value for statusChange") + assert.Equal(t, tc.instance, res[infoInstanceID.String()], "unexpected instance value for statusChange") + } +} + +func toStatusChangeMap(t *testing.T, statusChange string) map[string]string { + var data map[string]string + err := json.Unmarshal([]byte(statusChange), &data) + require.Nil(t, err) + return data +} + +func testConsulPubSubNewEvents(t *testing.T, kv *api.KV) { + // Do not run this test in // as it cause some concurrency issue + // t.Parallel() + deploymentID := testutil.BuildDeploymentID(t) + + nodeName := "node1" + instance := "0" + nodeStatus := "error" + + ready := make(chan struct{}) + + go func() { + i, err := GetStatusEventsIndex(kv, deploymentID) + require.Nil(t, err) + ready <- struct{}{} + events, _, err := StatusEvents(kv, deploymentID, i, 5*time.Minute) + assert.Nil(t, err) + require.Len(t, events, 1) + + event := toStatusChangeMap(t, string(events[0])) + assert.Equal(t, event[infoNodeID.String()], nodeName) + assert.Equal(t, event["status"], nodeStatus) + assert.Equal(t, event[infoInstanceID.String()], instance) + }() + <-ready + _, err := InstanceStatusChange(kv, deploymentID, nodeName, instance, nodeStatus) + assert.Nil(t, err) +} + +func testConsulPubSubNewEventsTimeout(t *testing.T, kv *api.KV) { + t.Parallel() + deploymentID := testutil.BuildDeploymentID(t) + + timeout := 25 * time.Millisecond + + t1 := time.Now() + events, _, err := StatusEvents(kv, deploymentID, 1, timeout) + t2 := time.Now() + assert.Nil(t, err) + require.Len(t, events, 0) + assert.WithinDuration(t, t1, t2, timeout+50*time.Millisecond) +} + +func testConsulPubSubNewEventsWithIndex(t *testing.T, kv *api.KV) { + t.Parallel() + deploymentID := testutil.BuildDeploymentID(t) + + var testData = []struct { + node string + instance string + status string + }{ + {"node1", "0", "initial"}, + {"node1", "1", "initial"}, + {"node1", "0", "creating"}, + {"node1", "1", "creating"}, + } + + for _, tc := range testData { + _, err := InstanceStatusChange(kv, deploymentID, tc.node, tc.instance, tc.status) + assert.Nil(t, err) + } + + rawEvents, lastIdx, err := StatusEvents(kv, deploymentID, 1, 5*time.Minute) + assert.Nil(t, err) + require.Len(t, rawEvents, 4) + for index, event := range rawEvents { + evt := toStatusChangeMap(t, string(event)) + assert.Equal(t, testData[index].node, evt[infoNodeID.String()]) + assert.Equal(t, testData[index].instance, evt[infoInstanceID.String()]) + assert.Equal(t, testData[index].status, evt["status"]) + } + + testData = []struct { + node string + instance string + status string + }{ + {"node1", "0", "created"}, + {"node1", "0", "configuring"}, + {"node1", "0", "configured"}, + } + + for _, tc := range testData { + _, err = InstanceStatusChange(kv, deploymentID, tc.node, tc.instance, tc.status) + assert.Nil(t, err) + } + + rawEvents, lastIdx, err = StatusEvents(kv, deploymentID, lastIdx, 5*time.Minute) + assert.Nil(t, err) + require.Len(t, rawEvents, 3) + require.NotZero(t, lastIdx) + + for index, rawEvent := range rawEvents { + event := toStatusChangeMap(t, string(rawEvent)) + assert.Equal(t, testData[index].node, event[infoNodeID.String()]) + assert.Equal(t, testData[index].instance, event[infoInstanceID.String()]) + assert.Equal(t, testData[index].status, event["status"]) + } +} + +func testConsulPubSubNewNodeEvents(t *testing.T, kv *api.KV) { + t.Parallel() + deploymentID := testutil.BuildDeploymentID(t) + + nodeName := "node1" + instance := "0" + nodeStatus := "error" + + _, err := InstanceStatusChange(kv, deploymentID, nodeName, instance, nodeStatus) + assert.Nil(t, err) + +} + +func testconsulDeploymentStatusChange(t *testing.T, kv *api.KV) { + t.Parallel() + deploymentID := testutil.BuildDeploymentID(t) + type args struct { + kv *api.KV + status string + } + tests := []struct { + name string + args args + wantErr bool + }{ + {"TestStatusInitial", args{kv, "initial"}, false}, + {"TestStatusDepInProgress", args{kv, "deployment_in_progress"}, false}, + {"TestStatusDepDeployed", args{kv, "deployed"}, false}, + } + ids := make([]string, 0) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := DeploymentStatusChange(tt.args.kv, deploymentID, tt.args.status) + if (err != nil) != tt.wantErr { + t.Errorf("DeploymentStatusChange() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != "" { + ids = append(ids, got) + } + }) + } + + prefix := path.Join(consulutil.EventsPrefix, deploymentID) + kvps, _, err := kv.List(prefix, nil) + assert.Nil(t, err) + assert.Len(t, kvps, len(tests)) + + for index, kvp := range kvps { + assert.Equal(t, ids[index], strings.TrimPrefix(kvp.Key, prefix+"/")) + tc := tests[index] + event := toStatusChangeMap(t, string(kvp.Value)) + assert.Equal(t, tc.args.status, event["status"]) + } + +} + +func testconsulCustomCommandStatusChange(t *testing.T, kv *api.KV) { + t.Parallel() + deploymentID := testutil.BuildDeploymentID(t) + type args struct { + kv *api.KV + taskID string + status string + } + tests := []struct { + name string + args args + wantErr bool + }{ + {"TestStatusInitial", args{kv, "1", "initial"}, false}, + {"TestStatusDepInProgress", args{kv, "1", "running"}, false}, + {"TestStatusDepDeployed", args{kv, "2", "initial"}, false}, + {"TestStatusDepDeployed", args{kv, "2", "running"}, false}, + {"TestStatusDepDeployed", args{kv, "1", "done"}, false}, + {"TestStatusDepDeployed", args{kv, "2", "done"}, false}, + } + ids := make([]string, 0) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := CustomCommandStatusChange(tt.args.kv, deploymentID, tt.args.taskID, tt.args.status) + if (err != nil) != tt.wantErr { + t.Errorf("CustomCommandStatusChange() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != "" { + ids = append(ids, got) + } + }) + } + + prefix := path.Join(consulutil.EventsPrefix, deploymentID) + kvps, _, err := kv.List(prefix, nil) + assert.Nil(t, err) + assert.Len(t, kvps, len(tests)) + + for index, kvp := range kvps { + assert.Equal(t, ids[index], strings.TrimPrefix(kvp.Key, prefix+"/")) + tc := tests[index] + event := toStatusChangeMap(t, string(kvp.Value)) + assert.Equal(t, tc.args.status, event["status"]) + assert.Equal(t, tc.args.taskID, event[infoExecutionID.String()]) + } + +} + +func testconsulScalingStatusChange(t *testing.T, kv *api.KV) { + t.Parallel() + deploymentID := testutil.BuildDeploymentID(t) + type args struct { + kv *api.KV + taskID string + status string + } + tests := []struct { + name string + args args + wantErr bool + }{ + {"TestStatusInitial", args{kv, "1", "initial"}, false}, + {"TestStatusDepInProgress", args{kv, "1", "running"}, false}, + {"TestStatusDepDeployed", args{kv, "2", "initial"}, false}, + {"TestStatusDepDeployed", args{kv, "2", "running"}, false}, + {"TestStatusDepDeployed", args{kv, "1", "done"}, false}, + {"TestStatusDepDeployed", args{kv, "2", "done"}, false}, + } + ids := make([]string, 0) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := ScalingStatusChange(tt.args.kv, deploymentID, tt.args.taskID, tt.args.status) + if (err != nil) != tt.wantErr { + t.Errorf("ScalingStatusChange() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != "" { + ids = append(ids, got) + } + }) + } + + prefix := path.Join(consulutil.EventsPrefix, deploymentID) + kvps, _, err := kv.List(prefix, nil) + assert.Nil(t, err) + assert.Len(t, kvps, len(tests)) + + for index, kvp := range kvps { + assert.Equal(t, ids[index], strings.TrimPrefix(kvp.Key, prefix+"/")) + tc := tests[index] + event := toStatusChangeMap(t, string(kvp.Value)) + assert.Equal(t, tc.args.status, event["status"]) + assert.Equal(t, tc.args.taskID, event[infoExecutionID.String()]) + } + +} + +func testconsulWorkflowStatusChange(t *testing.T, kv *api.KV) { + t.Parallel() + deploymentID := testutil.BuildDeploymentID(t) + type args struct { + kv *api.KV + taskID string + status string + } + tests := []struct { + name string + args args + wantErr bool + }{ + {"TestStatusInitial", args{kv, "1", "initial"}, false}, + {"TestStatusDepInProgress", args{kv, "1", "running"}, false}, + {"TestStatusDepDeployed", args{kv, "2", "initial"}, false}, + {"TestStatusDepDeployed", args{kv, "2", "running"}, false}, + {"TestStatusDepDeployed", args{kv, "1", "done"}, false}, + {"TestStatusDepDeployed", args{kv, "2", "done"}, false}, + } + ids := make([]string, 0) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := WorkflowStatusChange(tt.args.kv, deploymentID, tt.args.taskID, tt.args.status) + if (err != nil) != tt.wantErr { + t.Errorf("WorkflowStatusChange() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != "" { + ids = append(ids, got) + } + }) + } + + prefix := path.Join(consulutil.EventsPrefix, deploymentID) + kvps, _, err := kv.List(prefix, nil) + assert.Nil(t, err) + assert.Len(t, kvps, len(tests)) + + for index, kvp := range kvps { + assert.Equal(t, ids[index], strings.TrimPrefix(kvp.Key, prefix+"/")) + tc := tests[index] + event := toStatusChangeMap(t, string(kvp.Value)) + assert.Equal(t, tc.args.status, event["status"]) + assert.Equal(t, tc.args.taskID, event[infoExecutionID.String()]) + } + +} + +func testconsulGetStatusEvents(t *testing.T, kv *api.KV) { + t.Parallel() + deploymentID := testutil.BuildDeploymentID(t) + ids := make([]string, 5) + id, err := InstanceStatusChange(kv, deploymentID, "node1", "1", "started") + require.Nil(t, err) + ids[0] = id + id, err = DeploymentStatusChange(kv, deploymentID, "deployed") + require.Nil(t, err) + ids[1] = id + id, err = ScalingStatusChange(kv, deploymentID, "t2", "failed") + require.Nil(t, err) + ids[2] = id + id, err = CustomCommandStatusChange(kv, deploymentID, "t3", "running") + require.Nil(t, err) + ids[3] = id + id, err = WorkflowStatusChange(kv, deploymentID, "t4", "done") + require.Nil(t, err) + ids[4] = id + + rawEvents, _, err := StatusEvents(kv, deploymentID, 0, 5*time.Minute) + require.Nil(t, err) + require.Len(t, rawEvents, 5) + + events := make([]map[string]string, len(rawEvents)) + for i, rawEvent := range rawEvents { + events[i] = toStatusChangeMap(t, string(rawEvent)) + } + + require.Equal(t, StatusChangeTypeInstance.String(), events[0]["eventType"]) + require.Equal(t, "node1", events[0][infoNodeID.String()]) + require.Equal(t, "1", events[0][infoInstanceID.String()]) + require.Equal(t, "started", events[0]["status"]) + require.Equal(t, ids[0], events[0]["timestamp"]) + require.Equal(t, "", events[0][infoExecutionID.String()]) + + require.Equal(t, StatusChangeTypeDeployment.String(), events[1]["eventType"]) + require.Equal(t, "", events[1][infoNodeID.String()]) + require.Equal(t, "", events[1][infoInstanceID.String()]) + require.Equal(t, "deployed", events[1]["status"]) + require.Equal(t, ids[1], events[1]["timestamp"]) + require.Equal(t, "", events[1][infoExecutionID.String()]) + + require.Equal(t, StatusChangeTypeScaling.String(), events[2]["eventType"]) + require.Equal(t, "", events[2][infoNodeID.String()]) + require.Equal(t, "", events[2][infoInstanceID.String()]) + require.Equal(t, "failed", events[2]["status"]) + require.Equal(t, ids[2], events[2]["timestamp"]) + require.Equal(t, "t2", events[2][infoExecutionID.String()]) + + require.Equal(t, StatusChangeTypeCustomCommand.String(), events[3]["eventType"]) + require.Equal(t, "", events[3][infoNodeID.String()]) + require.Equal(t, "", events[3][infoInstanceID.String()]) + require.Equal(t, "running", events[3]["status"]) + require.Equal(t, ids[3], events[3]["timestamp"]) + require.Equal(t, "t3", events[3][infoExecutionID.String()]) + + require.Equal(t, StatusChangeTypeWorkflow.String(), events[4]["eventType"]) + require.Equal(t, "", events[4][infoNodeID.String()]) + require.Equal(t, "", events[4][infoInstanceID.String()]) + require.Equal(t, "done", events[4]["status"]) + require.Equal(t, ids[4], events[4]["timestamp"]) + require.Equal(t, "t4", events[4][infoExecutionID.String()]) + +} func testconsulGetLogs(t *testing.T, kv *api.KV) { t.Parallel() diff --git a/events/structs.go b/events/structs.go index b30296ffc..349eb2c71 100644 --- a/events/structs.go +++ b/events/structs.go @@ -15,43 +15,96 @@ package events import ( - "context" "encoding/json" + "github.com/pkg/errors" "github.com/ystia/yorc/helper/consulutil" "github.com/ystia/yorc/log" "path" + "strings" "time" ) //go:generate go-enum -f=structs.go --lower -// LogLevel x ENUM( +// StatusChangeType x ENUM( // Instance, // Deployment, // CustomCommand, // Scaling, // Workflow, // WorkflowStep +// AlienTask // ) type StatusChangeType int -// EventAdditionalInfo allows to provide custom/specific additional information for event -type EventAdditionalInfo map[string]interface{} +// Info allows to provide custom/specific additional information for event +type Info map[infoType]interface{} -// EventStatusChange represents status change event -type EventStatusChange struct { - timestamp string - eventType StatusChangeType - deploymentID string - status string - additionalInfo EventAdditionalInfo +// infoType represents Event status change information type +type infoType int + +const ( + infoWorkFlowID infoType = iota + + infoExecutionID + + infoNodeID + + infoInstanceID + + infoStepID + + infoOperationName + + infoTargetNodeID + + infoTargetInstanceID + + infoAlienTaskID + + infoWorkflowStepID +) + +func (i infoType) String() string { + switch i { + case infoWorkFlowID: + return "workFlowID" + case infoExecutionID: + return "executionID" + case infoNodeID: + return "nodeId" + case infoInstanceID: + return "instanceId" + case infoStepID: + return "stepId" + case infoOperationName: + return "operationName" + case infoTargetNodeID: + return "targetNodeId" + case infoTargetInstanceID: + return "targetInstanceId" + case infoAlienTaskID: + return "taskId" + case infoWorkflowStepID: + return "workflowStepId" + } + return "" +} + +// StatusChange represents status change event +type StatusChange struct { + timestamp string + eventType StatusChangeType + deploymentID string + status string + info Info } // Create a KVPair corresponding to an event and put it to Consul under the event prefix, // in a sub-tree corresponding to its deployment // The eventType goes to the KVPair's Flags field // The content is JSON format -func (e EventStatusChange) register() (string, error) { +func (e *StatusChange) register() (string, error) { e.timestamp = time.Now().Format(time.RFC3339Nano) eventsPrefix := path.Join(consulutil.EventsPrefix, e.deploymentID) @@ -61,38 +114,65 @@ func (e EventStatusChange) register() (string, error) { if err != nil { log.Printf("Failed to marshal event [%+v]: due to error:%+v", e, err) } - - //FIXME Do we still need to store event type as a flag ? - err = consulutil.StoreConsulKeyWithFlags(path.Join(eventsPrefix, e.timestamp), b, uint64(e.eventType)) + err = consulutil.StoreConsulKey(path.Join(eventsPrefix, e.timestamp), b) if err != nil { return "", err } return e.timestamp, nil } -func (e EventStatusChange) flat() map[string]interface{} { +func (e *StatusChange) flat() map[string]interface{} { flat := make(map[string]interface{}) flat["deploymentId"] = e.deploymentID flat["status"] = e.status flat["timestamp"] = e.timestamp - for k, v := range e.additionalInfo { - flat[k] = v + flat["eventType"] = e.eventType.String() + for k, v := range e.info { + flat[k.String()] = v } return flat } -// newEventStatusChange allows to create a new EventStatusChange pointer by retrieving context log information -func newEventStatusChange(ctx context.Context, eventType StatusChangeType, info EventAdditionalInfo, deploymentID, status string) *EventStatusChange { - event := &EventStatusChange{additionalInfo: info, eventType: eventType, status: status, deploymentID: deploymentID} - if event.additionalInfo == nil { - event.additionalInfo = make(EventAdditionalInfo) +// newStatusChange allows to create a new StatusChange if mandatory information is ok +func newStatusChange(eventType StatusChangeType, info Info, deploymentID, status string) (*StatusChange, error) { + e := &StatusChange{info: info, eventType: eventType, status: status, deploymentID: deploymentID} + if err := e.check(); err != nil { + return nil, err } - logInfo, ok := FromContext(ctx) - if ok { - for k, v := range logInfo { - event.additionalInfo[k.String()] = v + return e, nil +} + +func (e *StatusChange) check() error { + if e.deploymentID == "" || e.status == "" { + return errors.New("DeploymentID and status are mandatory parameters for EventStatusVChange") + } + + mandatoryMap := map[StatusChangeType][]infoType{ + StatusChangeTypeInstance: {infoNodeID, infoInstanceID}, + StatusChangeTypeCustomCommand: {infoExecutionID}, + StatusChangeTypeScaling: {infoExecutionID}, + StatusChangeTypeWorkflow: {infoExecutionID}, + StatusChangeTypeWorkflowStep: {infoExecutionID, infoWorkFlowID, infoNodeID, infoStepID, infoOperationName}, + StatusChangeTypeAlienTask: {infoExecutionID, infoWorkFlowID, infoNodeID, infoWorkflowStepID, infoOperationName, infoAlienTaskID}, + } + // Check mandatory info in function of status change type + if mandatoryInfos, is := mandatoryMap[e.eventType]; is { + for _, mandatoryInfo := range mandatoryInfos { + missing := make([]string, 0) + if e.info == nil { + for _, m := range mandatoryInfos { + missing = append(missing, m.String()) + } + } + if _, is := e.info[mandatoryInfo]; !is { + missing = append(missing, mandatoryInfo.String()) + } + + if len(missing) > 0 { + return errors.Errorf("Missing mandatory parameters:%q for event:%q", strings.Join(missing, ","), e.eventType.String()) + } } } - return event + return nil } diff --git a/events/structs_enum.go b/events/structs_enum.go index 0b86f61ae..38583f47b 100644 --- a/events/structs_enum.go +++ b/events/structs_enum.go @@ -35,9 +35,11 @@ const ( StatusChangeTypeWorkflow // StatusChangeTypeWorkflowStep is a StatusChangeType of type WorkflowStep StatusChangeTypeWorkflowStep + // StatusChangeTypeAlienTask is a StatusChangeType of type AlienTask + StatusChangeTypeAlienTask ) -const _StatusChangeTypeName = "InstanceDeploymentCustomCommandScalingWorkflowWorkflowStep" +const _StatusChangeTypeName = "InstanceDeploymentCustomCommandScalingWorkflowWorkflowStepAlienTask" var _StatusChangeTypeMap = map[StatusChangeType]string{ 0: _StatusChangeTypeName[0:8], @@ -46,6 +48,7 @@ var _StatusChangeTypeMap = map[StatusChangeType]string{ 3: _StatusChangeTypeName[31:38], 4: _StatusChangeTypeName[38:46], 5: _StatusChangeTypeName[46:58], + 6: _StatusChangeTypeName[58:67], } // String implements the Stringer interface. @@ -69,6 +72,8 @@ var _StatusChangeTypeValue = map[string]StatusChangeType{ strings.ToLower(_StatusChangeTypeName[38:46]): 4, _StatusChangeTypeName[46:58]: 5, strings.ToLower(_StatusChangeTypeName[46:58]): 5, + _StatusChangeTypeName[58:67]: 6, + strings.ToLower(_StatusChangeTypeName[58:67]): 6, } // ParseStatusChangeType attempts to convert a string to a StatusChangeType From fc514c99620ea0589318cd8c59079db18475a014 Mon Sep 17 00:00:00 2001 From: stephane benoist <29753229+stebenoist@users.noreply.github.com> Date: Fri, 30 Nov 2018 16:08:12 +0100 Subject: [PATCH 05/19] Update events API Publish WorkflowStep and AlienTask events --- events/consul_test.go | 6 ++ events/events.go | 47 ++++++----- events/events_test.go | 172 +++++++++++++++++++++++++++++++++-------- events/structs.go | 86 +++++++++++++-------- tasks/workflow/step.go | 40 +++++++--- 5 files changed, 255 insertions(+), 96 deletions(-) diff --git a/events/consul_test.go b/events/consul_test.go index fa5f72caa..eeace9fed 100644 --- a/events/consul_test.go +++ b/events/consul_test.go @@ -54,6 +54,12 @@ func TestRunConsulEventsPackageTests(t *testing.T) { t.Run("TestWorkflowStatusChange", func(t *testing.T) { testconsulWorkflowStatusChange(t, kv) }) + t.Run("TestWorkflowStepStatusChange", func(t *testing.T) { + testconsulWorkflowStepStatusChange(t, kv) + }) + t.Run("testAlienTaskStatusChange", func(t *testing.T) { + testconsulAlienTaskStatusChange(t, kv) + }) t.Run("TestGetStatusEvents", func(t *testing.T) { testconsulGetStatusEvents(t, kv) }) diff --git a/events/events.go b/events/events.go index 55fad7c1b..0ad9ddf2b 100644 --- a/events/events.go +++ b/events/events.go @@ -100,7 +100,7 @@ func PublishAndLogCustomCommandStatusChange(ctx context.Context, kv *api.KV, dep ctx = NewContext(context.Background(), LogOptionalFields{ExecutionID: taskID}) } info := make(Info) - info[infoExecutionID] = taskID + info[infoAlienExecutionID] = taskID e, err := newStatusChange(StatusChangeTypeCustomCommand, info, deploymentID, status) if err != nil { return "", err @@ -130,7 +130,7 @@ func PublishAndLogScalingStatusChange(ctx context.Context, kv *api.KV, deploymen ctx = NewContext(context.Background(), LogOptionalFields{ExecutionID: taskID}) } info := make(Info) - info[infoExecutionID] = taskID + info[infoAlienExecutionID] = taskID e, err := newStatusChange(StatusChangeTypeScaling, info, deploymentID, status) if err != nil { return "", err @@ -155,18 +155,22 @@ func WorkflowStatusChange(kv *api.KV, deploymentID, taskID, status string) (stri // PublishAndLogWorkflowStepStatusChange publishes a status change for a workflow step execution and log this change into the log API // // PublishAndLogWorkflowStepStatusChange returns the published event id -func PublishAndLogWorkflowStepStatusChange(ctx context.Context, kv *api.KV, deploymentID, taskID, workflowName, nodeName, stepName, operationName, targetNodeID, targetInstanceID, status string) (string, error) { +func PublishAndLogWorkflowStepStatusChange(ctx context.Context, kv *api.KV, deploymentID, taskID string, wfStepInfo *WorkflowStepInfo, status string) (string, error) { if ctx == nil { ctx = NewContext(context.Background(), LogOptionalFields{ExecutionID: taskID}) } + if wfStepInfo == nil { + return "", errors.Errorf("WorkflowStep information param must be provided") + } info := make(Info) - info[infoExecutionID] = taskID - info[infoWorkFlowID] = workflowName - info[infoNodeID] = nodeName - info[infoStepID] = stepName - info[infoOperationName] = operationName - info[infoTargetNodeID] = targetNodeID - info[infoTargetInstanceID] = targetInstanceID + info[infoAlienExecutionID] = taskID + info[infoInstanceID] = wfStepInfo.InstanceName + info[infoWorkFlowID] = wfStepInfo.WorkflowName + info[infoNodeID] = wfStepInfo.NodeName + info[infoWorkflowStepID] = wfStepInfo.StepName + info[infoOperationName] = wfStepInfo.OperationName + info[infoTargetNodeID] = wfStepInfo.TargetNodeID + info[infoTargetInstanceID] = wfStepInfo.TargetInstanceID e, err := newStatusChange(StatusChangeTypeWorkflowStep, info, deploymentID, status) if err != nil { return "", err @@ -182,21 +186,22 @@ func PublishAndLogWorkflowStepStatusChange(ctx context.Context, kv *api.KV, depl // PublishAndLogAlienTaskStatusChange publishes a status change for a task execution and log this change into the log API // // PublishAndLogAlienTaskStatusChange returns the published event id -func PublishAndLogAlienTaskStatusChange(ctx context.Context, kv *api.KV, deploymentID, taskID, taskExecutionID, workflowName, nodeName, stepName, operationName, targetNodeID, targetInstanceID, status string) (string, error) { +func PublishAndLogAlienTaskStatusChange(ctx context.Context, kv *api.KV, deploymentID, taskID, taskExecutionID string, wfStepInfo *WorkflowStepInfo, status string) (string, error) { if ctx == nil { ctx = NewContext(context.Background(), LogOptionalFields{ExecutionID: taskID}) } info := make(Info) - info[infoExecutionID] = taskID + info[infoAlienExecutionID] = taskID // Warning: Alien task corresponds to what we call taskExecution - info[infoAlienTaskID] = taskExecutionID - info[infoWorkFlowID] = workflowName - info[infoNodeID] = nodeName - info[infoWorkflowStepID] = stepName - info[infoOperationName] = operationName - info[infoTargetNodeID] = targetNodeID - info[infoTargetInstanceID] = targetInstanceID - e, err := newStatusChange(StatusChangeTypeWorkflow, info, deploymentID, status) + info[infoAlienTaskExecutionID] = taskExecutionID + info[infoWorkFlowID] = wfStepInfo.WorkflowName + info[infoNodeID] = wfStepInfo.NodeName + info[infoWorkflowStepID] = wfStepInfo.StepName + info[infoInstanceID] = wfStepInfo.InstanceName + info[infoOperationName] = wfStepInfo.OperationName + info[infoTargetNodeID] = wfStepInfo.TargetNodeID + info[infoTargetInstanceID] = wfStepInfo.TargetInstanceID + e, err := newStatusChange(StatusChangeTypeAlienTask, info, deploymentID, status) if err != nil { return "", err } @@ -216,7 +221,7 @@ func PublishAndLogWorkflowStatusChange(ctx context.Context, kv *api.KV, deployme ctx = NewContext(context.Background(), LogOptionalFields{ExecutionID: taskID}) } info := make(Info) - info[infoExecutionID] = taskID + info[infoAlienExecutionID] = taskID e, err := newStatusChange(StatusChangeTypeWorkflow, info, deploymentID, status) if err != nil { return "", err diff --git a/events/events_test.go b/events/events_test.go index 64ae0545b..096adfacf 100644 --- a/events/events_test.go +++ b/events/events_test.go @@ -24,6 +24,7 @@ import ( "github.com/pkg/errors" "github.com/stretchr/testify/require" + "context" "github.com/stretchr/testify/assert" "github.com/ystia/yorc/helper/consulutil" "github.com/ystia/yorc/testutil" @@ -68,8 +69,8 @@ func testConsulPubSubStatusChange(t *testing.T, kv *api.KV) { tc := testData[index] res := toStatusChangeMap(t, string(kvp.Value)) assert.Equal(t, tc.node, res[infoNodeID.String()], "unexpected node value for statusChange") - assert.Equal(t, tc.status, res["status"], "unexpected status value for statusChange") - assert.Equal(t, deploymentID, res["deploymentId"], "unexpected deploymentID value for statusChange") + assert.Equal(t, tc.status, res[infoStatus.String()], "unexpected status value for statusChange") + assert.Equal(t, deploymentID, res[infoDeploymentID.String()], "unexpected deploymentID value for statusChange") assert.Equal(t, tc.instance, res[infoInstanceID.String()], "unexpected instance value for statusChange") } } @@ -102,7 +103,7 @@ func testConsulPubSubNewEvents(t *testing.T, kv *api.KV) { event := toStatusChangeMap(t, string(events[0])) assert.Equal(t, event[infoNodeID.String()], nodeName) - assert.Equal(t, event["status"], nodeStatus) + assert.Equal(t, event[infoStatus.String()], nodeStatus) assert.Equal(t, event[infoInstanceID.String()], instance) }() <-ready @@ -151,7 +152,7 @@ func testConsulPubSubNewEventsWithIndex(t *testing.T, kv *api.KV) { evt := toStatusChangeMap(t, string(event)) assert.Equal(t, testData[index].node, evt[infoNodeID.String()]) assert.Equal(t, testData[index].instance, evt[infoInstanceID.String()]) - assert.Equal(t, testData[index].status, evt["status"]) + assert.Equal(t, testData[index].status, evt[infoStatus.String()]) } testData = []struct { @@ -178,7 +179,7 @@ func testConsulPubSubNewEventsWithIndex(t *testing.T, kv *api.KV) { event := toStatusChangeMap(t, string(rawEvent)) assert.Equal(t, testData[index].node, event[infoNodeID.String()]) assert.Equal(t, testData[index].instance, event[infoInstanceID.String()]) - assert.Equal(t, testData[index].status, event["status"]) + assert.Equal(t, testData[index].status, event[infoStatus.String()]) } } @@ -234,7 +235,7 @@ func testconsulDeploymentStatusChange(t *testing.T, kv *api.KV) { assert.Equal(t, ids[index], strings.TrimPrefix(kvp.Key, prefix+"/")) tc := tests[index] event := toStatusChangeMap(t, string(kvp.Value)) - assert.Equal(t, tc.args.status, event["status"]) + assert.Equal(t, tc.args.status, event[infoStatus.String()]) } } @@ -282,8 +283,8 @@ func testconsulCustomCommandStatusChange(t *testing.T, kv *api.KV) { assert.Equal(t, ids[index], strings.TrimPrefix(kvp.Key, prefix+"/")) tc := tests[index] event := toStatusChangeMap(t, string(kvp.Value)) - assert.Equal(t, tc.args.status, event["status"]) - assert.Equal(t, tc.args.taskID, event[infoExecutionID.String()]) + assert.Equal(t, tc.args.status, event[infoStatus.String()]) + assert.Equal(t, tc.args.taskID, event[infoAlienExecutionID.String()]) } } @@ -331,8 +332,8 @@ func testconsulScalingStatusChange(t *testing.T, kv *api.KV) { assert.Equal(t, ids[index], strings.TrimPrefix(kvp.Key, prefix+"/")) tc := tests[index] event := toStatusChangeMap(t, string(kvp.Value)) - assert.Equal(t, tc.args.status, event["status"]) - assert.Equal(t, tc.args.taskID, event[infoExecutionID.String()]) + assert.Equal(t, tc.args.status, event[infoStatus.String()]) + assert.Equal(t, tc.args.taskID, event[infoAlienExecutionID.String()]) } } @@ -380,10 +381,117 @@ func testconsulWorkflowStatusChange(t *testing.T, kv *api.KV) { assert.Equal(t, ids[index], strings.TrimPrefix(kvp.Key, prefix+"/")) tc := tests[index] event := toStatusChangeMap(t, string(kvp.Value)) - assert.Equal(t, tc.args.status, event["status"]) - assert.Equal(t, tc.args.taskID, event[infoExecutionID.String()]) + assert.Equal(t, tc.args.status, event[infoStatus.String()]) + assert.Equal(t, tc.args.taskID, event[infoAlienExecutionID.String()]) } +} + +func testconsulWorkflowStepStatusChange(t *testing.T, kv *api.KV) { + t.Parallel() + deploymentID := testutil.BuildDeploymentID(t) + type args struct { + kv *api.KV + taskID string + status string + wfInfo *WorkflowStepInfo + } + tests := []struct { + name string + args args + wantErr bool + }{ + {"TestStatusInitial", args{kv, "1", "initial", &WorkflowStepInfo{WorkflowName: "install", InstanceName: "0", StepName: "step1", NodeName: "node1"}}, false}, + {"TestStatusDepInProgress", args{kv, "1", "running", &WorkflowStepInfo{WorkflowName: "install", InstanceName: "0", StepName: "step1", NodeName: "node1"}}, false}, + {"TestStatusDepDeployed", args{kv, "2", "initial", &WorkflowStepInfo{WorkflowName: "install", InstanceName: "0", StepName: "step1", NodeName: "node1"}}, false}, + {"TestStatusDepDeployed", args{kv, "2", "running", &WorkflowStepInfo{WorkflowName: "install", InstanceName: "0", StepName: "step1", NodeName: "node1"}}, false}, + {"TestStatusDepDeployed", args{kv, "1", "done", &WorkflowStepInfo{WorkflowName: "install", InstanceName: "0", StepName: "step1", NodeName: "node1"}}, false}, + {"TestStatusDepDeployed", args{kv, "2", "done", &WorkflowStepInfo{WorkflowName: "install", InstanceName: "0", StepName: "step1", NodeName: "node1"}}, false}, + } + ids := make([]string, 0) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := PublishAndLogWorkflowStepStatusChange(context.Background(), tt.args.kv, deploymentID, tt.args.taskID, tt.args.wfInfo, tt.args.status) + if (err != nil) != tt.wantErr { + t.Errorf("WorkflowStatusChange() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != "" { + ids = append(ids, got) + } + }) + } + + prefix := path.Join(consulutil.EventsPrefix, deploymentID) + kvps, _, err := kv.List(prefix, nil) + assert.Nil(t, err) + assert.Len(t, kvps, len(tests)) + + for index, kvp := range kvps { + assert.Equal(t, ids[index], strings.TrimPrefix(kvp.Key, prefix+"/")) + tc := tests[index] + event := toStatusChangeMap(t, string(kvp.Value)) + assert.Equal(t, tc.args.status, event[infoStatus.String()]) + assert.Equal(t, tc.args.taskID, event[infoAlienExecutionID.String()]) + assert.Equal(t, "install", event[infoWorkFlowID.String()]) + assert.Equal(t, "0", event[infoInstanceID.String()]) + assert.Equal(t, "step1", event[infoWorkflowStepID.String()]) + assert.Equal(t, "node1", event[infoNodeID.String()]) + } +} + +func testconsulAlienTaskStatusChange(t *testing.T, kv *api.KV) { + t.Parallel() + deploymentID := testutil.BuildDeploymentID(t) + type args struct { + kv *api.KV + taskID string + taskExecutionID string + status string + wfInfo *WorkflowStepInfo + } + tests := []struct { + name string + args args + wantErr bool + }{ + {"TestStatusInitial", args{kv, "1", "33", "initial", &WorkflowStepInfo{WorkflowName: "install", InstanceName: "0", StepName: "step1", NodeName: "node1"}}, false}, + {"TestStatusDepInProgress", args{kv, "1", "33", "running", &WorkflowStepInfo{WorkflowName: "install", InstanceName: "0", StepName: "step1", NodeName: "node1"}}, false}, + {"TestStatusDepDeployed", args{kv, "2", "33", "initial", &WorkflowStepInfo{WorkflowName: "install", InstanceName: "0", StepName: "step1", NodeName: "node1"}}, false}, + {"TestStatusDepDeployed", args{kv, "2", "33", "running", &WorkflowStepInfo{WorkflowName: "install", InstanceName: "0", StepName: "step1", NodeName: "node1"}}, false}, + {"TestStatusDepDeployed", args{kv, "1", "33", "done", &WorkflowStepInfo{WorkflowName: "install", InstanceName: "0", StepName: "step1", NodeName: "node1"}}, false}, + {"TestStatusDepDeployed", args{kv, "2", "33", "done", &WorkflowStepInfo{WorkflowName: "install", InstanceName: "0", StepName: "step1", NodeName: "node1"}}, false}, + } + ids := make([]string, 0) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := PublishAndLogAlienTaskStatusChange(context.Background(), tt.args.kv, deploymentID, tt.args.taskID, tt.args.taskExecutionID, tt.args.wfInfo, tt.args.status) + if (err != nil) != tt.wantErr { + t.Errorf("WorkflowStatusChange() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != "" { + ids = append(ids, got) + } + }) + } + + prefix := path.Join(consulutil.EventsPrefix, deploymentID) + kvps, _, err := kv.List(prefix, nil) + assert.Nil(t, err) + assert.Len(t, kvps, len(tests)) + for index, kvp := range kvps { + assert.Equal(t, ids[index], strings.TrimPrefix(kvp.Key, prefix+"/")) + tc := tests[index] + event := toStatusChangeMap(t, string(kvp.Value)) + assert.Equal(t, tc.args.status, event[infoStatus.String()]) + assert.Equal(t, tc.args.taskID, event[infoAlienExecutionID.String()]) + assert.Equal(t, tc.args.taskExecutionID, event[infoAlienTaskExecutionID.String()]) + assert.Equal(t, "install", event[infoWorkFlowID.String()]) + assert.Equal(t, "0", event[infoInstanceID.String()]) + assert.Equal(t, "step1", event[infoWorkflowStepID.String()]) + assert.Equal(t, "node1", event[infoNodeID.String()]) + } } func testconsulGetStatusEvents(t *testing.T, kv *api.KV) { @@ -415,40 +523,40 @@ func testconsulGetStatusEvents(t *testing.T, kv *api.KV) { events[i] = toStatusChangeMap(t, string(rawEvent)) } - require.Equal(t, StatusChangeTypeInstance.String(), events[0]["eventType"]) + require.Equal(t, StatusChangeTypeInstance.String(), events[0][infoEventType.String()]) require.Equal(t, "node1", events[0][infoNodeID.String()]) require.Equal(t, "1", events[0][infoInstanceID.String()]) - require.Equal(t, "started", events[0]["status"]) - require.Equal(t, ids[0], events[0]["timestamp"]) - require.Equal(t, "", events[0][infoExecutionID.String()]) + require.Equal(t, "started", events[0][infoStatus.String()]) + require.Equal(t, ids[0], events[0][infoTimestamp.String()]) + require.Equal(t, "", events[0][infoAlienExecutionID.String()]) - require.Equal(t, StatusChangeTypeDeployment.String(), events[1]["eventType"]) + require.Equal(t, StatusChangeTypeDeployment.String(), events[1][infoEventType.String()]) require.Equal(t, "", events[1][infoNodeID.String()]) require.Equal(t, "", events[1][infoInstanceID.String()]) - require.Equal(t, "deployed", events[1]["status"]) - require.Equal(t, ids[1], events[1]["timestamp"]) - require.Equal(t, "", events[1][infoExecutionID.String()]) + require.Equal(t, "deployed", events[1][infoStatus.String()]) + require.Equal(t, ids[1], events[1][infoTimestamp.String()]) + require.Equal(t, "", events[1][infoAlienExecutionID.String()]) - require.Equal(t, StatusChangeTypeScaling.String(), events[2]["eventType"]) + require.Equal(t, StatusChangeTypeScaling.String(), events[2][infoEventType.String()]) require.Equal(t, "", events[2][infoNodeID.String()]) require.Equal(t, "", events[2][infoInstanceID.String()]) - require.Equal(t, "failed", events[2]["status"]) - require.Equal(t, ids[2], events[2]["timestamp"]) - require.Equal(t, "t2", events[2][infoExecutionID.String()]) + require.Equal(t, "failed", events[2][infoStatus.String()]) + require.Equal(t, ids[2], events[2][infoTimestamp.String()]) + require.Equal(t, "t2", events[2][infoAlienExecutionID.String()]) - require.Equal(t, StatusChangeTypeCustomCommand.String(), events[3]["eventType"]) + require.Equal(t, StatusChangeTypeCustomCommand.String(), events[3][infoEventType.String()]) require.Equal(t, "", events[3][infoNodeID.String()]) require.Equal(t, "", events[3][infoInstanceID.String()]) - require.Equal(t, "running", events[3]["status"]) - require.Equal(t, ids[3], events[3]["timestamp"]) - require.Equal(t, "t3", events[3][infoExecutionID.String()]) + require.Equal(t, "running", events[3][infoStatus.String()]) + require.Equal(t, ids[3], events[3][infoTimestamp.String()]) + require.Equal(t, "t3", events[3][infoAlienExecutionID.String()]) - require.Equal(t, StatusChangeTypeWorkflow.String(), events[4]["eventType"]) + require.Equal(t, StatusChangeTypeWorkflow.String(), events[4][infoEventType.String()]) require.Equal(t, "", events[4][infoNodeID.String()]) require.Equal(t, "", events[4][infoInstanceID.String()]) - require.Equal(t, "done", events[4]["status"]) - require.Equal(t, ids[4], events[4]["timestamp"]) - require.Equal(t, "t4", events[4][infoExecutionID.String()]) + require.Equal(t, "done", events[4][infoStatus.String()]) + require.Equal(t, ids[4], events[4][infoTimestamp.String()]) + require.Equal(t, "t4", events[4][infoAlienExecutionID.String()]) } diff --git a/events/structs.go b/events/structs.go index 349eb2c71..a91d91a68 100644 --- a/events/structs.go +++ b/events/structs.go @@ -44,55 +44,67 @@ type Info map[infoType]interface{} type infoType int const ( - infoWorkFlowID infoType = iota + infoDeploymentID infoType = iota - infoExecutionID + infoStatus + + infoTimestamp + + infoEventType + + infoWorkFlowID + + infoAlienExecutionID infoNodeID infoInstanceID - infoStepID - infoOperationName infoTargetNodeID infoTargetInstanceID - infoAlienTaskID + infoAlienTaskExecutionID infoWorkflowStepID ) func (i infoType) String() string { switch i { + case infoDeploymentID: + return "deploymentId" + case infoStatus: + return "status" + case infoTimestamp: + return "timestamp" + case infoEventType: + return "type" case infoWorkFlowID: return "workFlowID" - case infoExecutionID: - return "executionID" + case infoAlienExecutionID: + return "alienExecutionID" case infoNodeID: return "nodeId" case infoInstanceID: return "instanceId" - case infoStepID: - return "stepId" case infoOperationName: return "operationName" case infoTargetNodeID: return "targetNodeId" case infoTargetInstanceID: return "targetInstanceId" - case infoAlienTaskID: - return "taskId" + case infoAlienTaskExecutionID: + return "alienTaskId" case infoWorkflowStepID: - return "workflowStepId" + return "stepId" } return "" } -// StatusChange represents status change event -type StatusChange struct { +// statusChange represents status change event +type statusChange struct { timestamp string eventType StatusChangeType deploymentID string @@ -100,11 +112,22 @@ type StatusChange struct { info Info } +// WorkflowStepInfo represents specific workflow step event information +type WorkflowStepInfo struct { + WorkflowName string + InstanceName string + NodeName string + StepName string + OperationName string + TargetNodeID string + TargetInstanceID string +} + // Create a KVPair corresponding to an event and put it to Consul under the event prefix, // in a sub-tree corresponding to its deployment // The eventType goes to the KVPair's Flags field // The content is JSON format -func (e *StatusChange) register() (string, error) { +func (e *statusChange) register() (string, error) { e.timestamp = time.Now().Format(time.RFC3339Nano) eventsPrefix := path.Join(consulutil.EventsPrefix, e.deploymentID) @@ -121,40 +144,42 @@ func (e *StatusChange) register() (string, error) { return e.timestamp, nil } -func (e *StatusChange) flat() map[string]interface{} { +func (e *statusChange) flat() map[string]interface{} { flat := make(map[string]interface{}) - flat["deploymentId"] = e.deploymentID - flat["status"] = e.status - flat["timestamp"] = e.timestamp - flat["eventType"] = e.eventType.String() + flat[infoDeploymentID.String()] = e.deploymentID + flat[infoStatus.String()] = e.status + flat[infoTimestamp.String()] = e.timestamp + flat[infoEventType.String()] = e.eventType.String() for k, v := range e.info { flat[k.String()] = v } return flat } -// newStatusChange allows to create a new StatusChange if mandatory information is ok -func newStatusChange(eventType StatusChangeType, info Info, deploymentID, status string) (*StatusChange, error) { - e := &StatusChange{info: info, eventType: eventType, status: status, deploymentID: deploymentID} +// newStatusChange allows to create a new statusChange if mandatory information is ok +func newStatusChange(eventType StatusChangeType, info Info, deploymentID, status string) (*statusChange, error) { + e := &statusChange{info: info, eventType: eventType, status: status, deploymentID: deploymentID} if err := e.check(); err != nil { - return nil, err + // Just print Warning log if any mandatory field is not set + // But let's us the possibility to return an error + log.Print("[WARNING] " + err.Error()) } return e, nil } -func (e *StatusChange) check() error { +func (e *statusChange) check() error { if e.deploymentID == "" || e.status == "" { return errors.New("DeploymentID and status are mandatory parameters for EventStatusVChange") } mandatoryMap := map[StatusChangeType][]infoType{ StatusChangeTypeInstance: {infoNodeID, infoInstanceID}, - StatusChangeTypeCustomCommand: {infoExecutionID}, - StatusChangeTypeScaling: {infoExecutionID}, - StatusChangeTypeWorkflow: {infoExecutionID}, - StatusChangeTypeWorkflowStep: {infoExecutionID, infoWorkFlowID, infoNodeID, infoStepID, infoOperationName}, - StatusChangeTypeAlienTask: {infoExecutionID, infoWorkFlowID, infoNodeID, infoWorkflowStepID, infoOperationName, infoAlienTaskID}, + StatusChangeTypeCustomCommand: {infoAlienExecutionID}, + StatusChangeTypeScaling: {infoAlienExecutionID}, + StatusChangeTypeWorkflow: {infoAlienExecutionID}, + StatusChangeTypeWorkflowStep: {infoAlienExecutionID, infoWorkFlowID, infoNodeID, infoWorkflowStepID, infoInstanceID}, + StatusChangeTypeAlienTask: {infoAlienExecutionID, infoWorkFlowID, infoNodeID, infoWorkflowStepID, infoInstanceID, infoAlienTaskExecutionID}, } // Check mandatory info in function of status change type if mandatoryInfos, is := mandatoryMap[e.eventType]; is { @@ -168,7 +193,6 @@ func (e *StatusChange) check() error { if _, is := e.info[mandatoryInfo]; !is { missing = append(missing, mandatoryInfo.String()) } - if len(missing) > 0 { return errors.Errorf("Missing mandatory parameters:%q for event:%q", strings.Join(missing, ","), e.eventType.String()) } diff --git a/tasks/workflow/step.go b/tasks/workflow/step.go index 97919b342..6f033eeda 100644 --- a/tasks/workflow/step.go +++ b/tasks/workflow/step.go @@ -171,6 +171,11 @@ func (s *step) run(ctx context.Context, cfg config.Configuration, kv *api.KV, de s.setStatus(tasks.TaskStepStatusDONE) return nil } + + // Let's publish initial event status for workflow step (just before run it) + eventInfo := &events.WorkflowStepInfo{WorkflowName: workflowName, NodeName: s.Target, StepName: s.Name} + events.PublishAndLogWorkflowStepStatusChange(ctx, kv, deploymentID, s.t.taskID, eventInfo, tasks.TaskStepStatusINITIAL.String()) + events.PublishAndLogAlienTaskStatusChange(ctx, kv, deploymentID, s.t.taskID, s.t.id, eventInfo, tasks.TaskStepStatusINITIAL.String()) s.setStatus(tasks.TaskStepStatusRUNNING) logOptFields, _ := events.FromContext(ctx) @@ -255,6 +260,7 @@ func (s *step) runActivity(wfCtx context.Context, kv *api.KV, cfg config.Configu return err } + eventInfo := &events.WorkflowStepInfo{WorkflowName: workflowName, NodeName: s.Target, StepName: s.Name} switch activity.Type() { case builder.ActivityTypeDelegate: nodeType, err := deployments.GetNodeType(kv, deploymentID, s.Target) @@ -268,8 +274,9 @@ func (s *step) runActivity(wfCtx context.Context, kv *api.KV, cfg config.Configu delegateOp := activity.Value() wfCtx = events.AddLogOptionalFields(wfCtx, events.LogOptionalFields{events.InterfaceName: "delegate", events.OperationName: delegateOp}) for _, instanceName := range instances { - // TODO: replace this with workflow steps events - events.WithContextOptionalFields(events.AddLogOptionalFields(wfCtx, events.LogOptionalFields{events.InstanceID: instanceName})).NewLogEntry(events.LogLevelDEBUG, deploymentID).RegisterAsString("executing delegate operation") + eventInfo.InstanceName = instanceName + events.PublishAndLogWorkflowStepStatusChange(wfCtx, kv, deploymentID, s.t.taskID, eventInfo, tasks.TaskStepStatusRUNNING.String()) + events.PublishAndLogAlienTaskStatusChange(wfCtx, kv, deploymentID, s.t.taskID, s.t.id, eventInfo, tasks.TaskStepStatusRUNNING.String()) } err = func() error { defer metrics.MeasureSince(metricsutil.CleanupMetricKey([]string{"executor", "delegate", deploymentID, nodeType, delegateOp}), time.Now()) @@ -279,15 +286,17 @@ func (s *step) runActivity(wfCtx context.Context, kv *api.KV, cfg config.Configu if err != nil { metrics.IncrCounter(metricsutil.CleanupMetricKey([]string{"executor", "delegate", deploymentID, nodeType, delegateOp, "failures"}), 1) for _, instanceName := range instances { - // TODO: replace this with workflow steps events - events.WithContextOptionalFields(events.AddLogOptionalFields(wfCtx, events.LogOptionalFields{events.InstanceID: instanceName})).NewLogEntry(events.LogLevelDEBUG, deploymentID).RegisterAsString("delegate operation failed") + eventInfo.InstanceName = instanceName + events.PublishAndLogWorkflowStepStatusChange(wfCtx, kv, deploymentID, s.t.taskID, eventInfo, tasks.TaskStepStatusERROR.String()) + events.PublishAndLogAlienTaskStatusChange(wfCtx, kv, deploymentID, s.t.taskID, s.t.id, eventInfo, tasks.TaskStepStatusERROR.String()) } return err } metrics.IncrCounter(metricsutil.CleanupMetricKey([]string{"executor", "delegate", deploymentID, nodeType, delegateOp, "successes"}), 1) for _, instanceName := range instances { - // TODO: replace this with workflow steps events - events.WithContextOptionalFields(events.AddLogOptionalFields(wfCtx, events.LogOptionalFields{events.InstanceID: instanceName})).NewLogEntry(events.LogLevelDEBUG, deploymentID).RegisterAsString("delegate operation succeeded") + eventInfo.InstanceName = instanceName + events.PublishAndLogWorkflowStepStatusChange(wfCtx, kv, deploymentID, s.t.taskID, eventInfo, tasks.TaskStepStatusDONE.String()) + events.PublishAndLogAlienTaskStatusChange(wfCtx, kv, deploymentID, s.t.taskID, s.t.id, eventInfo, tasks.TaskStepStatusDONE.String()) } case builder.ActivityTypeSetState: setNodeStatus(wfCtx, kv, s.t.taskID, deploymentID, s.Target, activity.Value()) @@ -312,8 +321,13 @@ func (s *step) runActivity(wfCtx context.Context, kv *api.KV, cfg config.Configu } wfCtx = operations.SetOperationLogFields(wfCtx, op) for _, instanceName := range instances { - // TODO: replace this with workflow steps events - events.WithContextOptionalFields(events.AddLogOptionalFields(wfCtx, events.LogOptionalFields{events.InstanceID: instanceName})).NewLogEntry(events.LogLevelDEBUG, deploymentID).RegisterAsString("executing operation") + eventInfo.OperationName = op.Name + eventInfo.InstanceName = instanceName + if op.RelOp.IsRelationshipOperation { + eventInfo.TargetNodeID = op.RelOp.TargetNodeName + } + events.PublishAndLogWorkflowStepStatusChange(wfCtx, kv, deploymentID, s.t.taskID, eventInfo, tasks.TaskStepStatusRUNNING.String()) + events.PublishAndLogAlienTaskStatusChange(wfCtx, kv, deploymentID, s.t.taskID, s.t.id, eventInfo, tasks.TaskStepStatusRUNNING.String()) } // In function of the operation, the execution is sync or async if s.Async { @@ -347,16 +361,18 @@ func (s *step) runActivity(wfCtx context.Context, kv *api.KV, cfg config.Configu if err != nil { metrics.IncrCounter(metricsutil.CleanupMetricKey([]string{"executor", "operation", deploymentID, nodeType, op.Name, "failures"}), 1) for _, instanceName := range instances { - // TODO: replace this with workflow steps events - events.WithContextOptionalFields(events.AddLogOptionalFields(wfCtx, events.LogOptionalFields{events.InstanceID: instanceName})).NewLogEntry(events.LogLevelDEBUG, deploymentID).RegisterAsString("operation failed") + eventInfo.InstanceName = instanceName + events.PublishAndLogWorkflowStepStatusChange(wfCtx, kv, deploymentID, s.t.taskID, eventInfo, tasks.TaskStepStatusERROR.String()) + events.PublishAndLogAlienTaskStatusChange(wfCtx, kv, deploymentID, s.t.taskID, s.t.id, eventInfo, tasks.TaskStepStatusERROR.String()) } return err } metrics.IncrCounter(metricsutil.CleanupMetricKey([]string{"executor", "operation", deploymentID, nodeType, op.Name, "successes"}), 1) if !s.Async { for _, instanceName := range instances { - // TODO: replace this with workflow steps events - events.WithContextOptionalFields(events.AddLogOptionalFields(wfCtx, events.LogOptionalFields{events.InstanceID: instanceName})).NewLogEntry(events.LogLevelDEBUG, deploymentID).RegisterAsString("operation succeeded") + eventInfo.InstanceName = instanceName + events.PublishAndLogWorkflowStepStatusChange(wfCtx, kv, deploymentID, s.t.taskID, eventInfo, tasks.TaskStepStatusDONE.String()) + events.PublishAndLogAlienTaskStatusChange(wfCtx, kv, deploymentID, s.t.taskID, s.t.id, eventInfo, tasks.TaskStepStatusDONE.String()) } } case builder.ActivityTypeInline: From 951b2118a6a80e07a753f147843ffd62de97973c Mon Sep 17 00:00:00 2001 From: stephane benoist <29753229+stebenoist@users.noreply.github.com> Date: Mon, 3 Dec 2018 17:07:58 +0100 Subject: [PATCH 06/19] Publish new workflow events Fix issue on task step status case Fix issue on cli events format --- commands/deployments/dep_events.go | 4 ++-- events/events.go | 5 +++-- tasks/workflow/worker.go | 5 +++++ 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/commands/deployments/dep_events.go b/commands/deployments/dep_events.go index 58564eb3b..2ae2d1b71 100644 --- a/commands/deployments/dep_events.go +++ b/commands/deployments/dep_events.go @@ -128,9 +128,9 @@ func StreamsEvents(client *httputil.YorcClient, deploymentID string, colorize, f lastIdx = evts.LastIndex for _, event := range evts.Events { if colorize { - fmt.Printf("%s\n", color.MagentaString("%s", format(event))) + fmt.Printf("%s\n", color.MagentaString("%s", string(event))) } else { - fmt.Printf("%s\n", format(event)) + fmt.Printf("%s\n", string(event)) } } response.Body.Close() diff --git a/events/events.go b/events/events.go index 0ad9ddf2b..fb1f51de7 100644 --- a/events/events.go +++ b/events/events.go @@ -26,6 +26,7 @@ import ( "github.com/pkg/errors" "github.com/ystia/yorc/helper/consulutil" "github.com/ystia/yorc/log" + "strings" ) // InstanceStatusChange publishes a status change for a given instance of a given node @@ -171,7 +172,7 @@ func PublishAndLogWorkflowStepStatusChange(ctx context.Context, kv *api.KV, depl info[infoOperationName] = wfStepInfo.OperationName info[infoTargetNodeID] = wfStepInfo.TargetNodeID info[infoTargetInstanceID] = wfStepInfo.TargetInstanceID - e, err := newStatusChange(StatusChangeTypeWorkflowStep, info, deploymentID, status) + e, err := newStatusChange(StatusChangeTypeWorkflowStep, info, deploymentID, strings.ToLower(status)) if err != nil { return "", err } @@ -201,7 +202,7 @@ func PublishAndLogAlienTaskStatusChange(ctx context.Context, kv *api.KV, deploym info[infoOperationName] = wfStepInfo.OperationName info[infoTargetNodeID] = wfStepInfo.TargetNodeID info[infoTargetInstanceID] = wfStepInfo.TargetInstanceID - e, err := newStatusChange(StatusChangeTypeAlienTask, info, deploymentID, status) + e, err := newStatusChange(StatusChangeTypeAlienTask, info, deploymentID, strings.ToLower(status)) if err != nil { return "", err } diff --git a/tasks/workflow/worker.go b/tasks/workflow/worker.go index 197b37dc4..f825b30ee 100644 --- a/tasks/workflow/worker.go +++ b/tasks/workflow/worker.go @@ -149,6 +149,7 @@ func (w *worker) monitorTaskCancellation(ctx context.Context, cancelFunc context if strings.ToLower(string(kvp.Value)) == "true" { log.Debugln("Task cancellation requested.") checkAndSetTaskStatus(ctx, t.kv, t.taskID, t.step, tasks.TaskStatusCANCELED) + events.PublishAndLogWorkflowStatusChange(ctx, w.consulClient.KV(), t.targetID, t.taskID, tasks.TaskStatusCANCELED.String()) cancelFunc() return } @@ -576,6 +577,7 @@ func (w *worker) runDeploy(ctx context.Context, t *taskExecution) { log.Printf("%+v", err) w.checkAndSetDeploymentStatus(ctx, t.targetID, deployments.DEPLOYMENT_FAILED) checkAndSetTaskStatus(ctx, t.kv, t.taskID, t.step, tasks.TaskStatusFAILED) + events.PublishAndLogWorkflowStatusChange(ctx, w.consulClient.KV(), t.targetID, t.taskID, tasks.TaskStatusFAILED.String()) return } if wfDone { @@ -585,6 +587,7 @@ func (w *worker) runDeploy(ctx context.Context, t *taskExecution) { log.Printf("%+v", err) return } + events.PublishAndLogWorkflowStatusChange(ctx, w.consulClient.KV(), t.targetID, t.taskID, tasks.TaskStatusDONE.String()) } } @@ -602,6 +605,7 @@ func (w *worker) runUndeploy(ctx context.Context, t *taskExecution) { log.Printf("%+v", err) w.checkAndSetDeploymentStatus(ctx, t.targetID, deployments.UNDEPLOYMENT_FAILED) checkAndSetTaskStatus(ctx, t.kv, t.taskID, t.step, tasks.TaskStatusFAILED) + events.PublishAndLogWorkflowStatusChange(ctx, w.consulClient.KV(), t.targetID, t.taskID, tasks.TaskStatusFAILED.String()) return } if wfDone { @@ -615,6 +619,7 @@ func (w *worker) runUndeploy(ctx context.Context, t *taskExecution) { w.runPurge(ctx, t) } } + events.PublishAndLogWorkflowStatusChange(ctx, w.consulClient.KV(), t.targetID, t.taskID, tasks.TaskStatusDONE.String()) } else if t.taskType == tasks.TaskTypePurge { w.runPurge(ctx, t) } From ae6fb44efb35bb19d28e29659237ba0d1be6a41c Mon Sep 17 00:00:00 2001 From: benoist-s Date: Tue, 4 Dec 2018 16:27:19 +0100 Subject: [PATCH 07/19] Fix some issues on events --- events/events.go | 21 ++++++------ events/events_test.go | 65 ++++++++++++++++++++---------------- events/structs.go | 12 +++---- tasks/collector.go | 2 +- tasks/collector/collector.go | 6 ++-- tasks/tasks.go | 6 ++-- tasks/workflow/step.go | 20 +++++++---- tasks/workflow/worker.go | 14 ++++---- 8 files changed, 83 insertions(+), 63 deletions(-) diff --git a/events/events.go b/events/events.go index fb1f51de7..04bff2dfc 100644 --- a/events/events.go +++ b/events/events.go @@ -47,7 +47,7 @@ func PublishAndLogInstanceStatusChange(ctx context.Context, kv *api.KV, deployme info := make(Info) info[infoNodeID] = nodeName info[infoInstanceID] = instance - e, err := newStatusChange(StatusChangeTypeInstance, info, deploymentID, status) + e, err := newStatusChange(StatusChangeTypeInstance, info, deploymentID, strings.ToLower(status)) if err != nil { return "", err } @@ -72,7 +72,7 @@ func DeploymentStatusChange(kv *api.KV, deploymentID, status string) (string, er // // PublishAndLogDeploymentStatusChange returns the published event id func PublishAndLogDeploymentStatusChange(ctx context.Context, kv *api.KV, deploymentID, status string) (string, error) { - e, err := newStatusChange(StatusChangeTypeDeployment, nil, deploymentID, status) + e, err := newStatusChange(StatusChangeTypeDeployment, nil, deploymentID, strings.ToLower(status)) if err != nil { return "", err } @@ -102,7 +102,7 @@ func PublishAndLogCustomCommandStatusChange(ctx context.Context, kv *api.KV, dep } info := make(Info) info[infoAlienExecutionID] = taskID - e, err := newStatusChange(StatusChangeTypeCustomCommand, info, deploymentID, status) + e, err := newStatusChange(StatusChangeTypeCustomCommand, info, deploymentID, strings.ToLower(status)) if err != nil { return "", err } @@ -132,7 +132,7 @@ func PublishAndLogScalingStatusChange(ctx context.Context, kv *api.KV, deploymen } info := make(Info) info[infoAlienExecutionID] = taskID - e, err := newStatusChange(StatusChangeTypeScaling, info, deploymentID, status) + e, err := newStatusChange(StatusChangeTypeScaling, info, deploymentID, strings.ToLower(status)) if err != nil { return "", err } @@ -149,8 +149,8 @@ func PublishAndLogScalingStatusChange(ctx context.Context, kv *api.KV, deploymen // WorkflowStatusChange returns the published event id // // Deprecated: use PublishAndLogWorkflowStatusChange instead -func WorkflowStatusChange(kv *api.KV, deploymentID, taskID, status string) (string, error) { - return PublishAndLogWorkflowStatusChange(nil, kv, deploymentID, taskID, status) +func WorkflowStatusChange(kv *api.KV, deploymentID, taskID, workflowName, status string) (string, error) { + return PublishAndLogWorkflowStatusChange(nil, kv, deploymentID, taskID, workflowName, status) } // PublishAndLogWorkflowStepStatusChange publishes a status change for a workflow step execution and log this change into the log API @@ -166,7 +166,7 @@ func PublishAndLogWorkflowStepStatusChange(ctx context.Context, kv *api.KV, depl info := make(Info) info[infoAlienExecutionID] = taskID info[infoInstanceID] = wfStepInfo.InstanceName - info[infoWorkFlowID] = wfStepInfo.WorkflowName + info[infoWorkflowID] = wfStepInfo.WorkflowName info[infoNodeID] = wfStepInfo.NodeName info[infoWorkflowStepID] = wfStepInfo.StepName info[infoOperationName] = wfStepInfo.OperationName @@ -195,7 +195,7 @@ func PublishAndLogAlienTaskStatusChange(ctx context.Context, kv *api.KV, deploym info[infoAlienExecutionID] = taskID // Warning: Alien task corresponds to what we call taskExecution info[infoAlienTaskExecutionID] = taskExecutionID - info[infoWorkFlowID] = wfStepInfo.WorkflowName + info[infoWorkflowID] = wfStepInfo.WorkflowName info[infoNodeID] = wfStepInfo.NodeName info[infoWorkflowStepID] = wfStepInfo.StepName info[infoInstanceID] = wfStepInfo.InstanceName @@ -217,13 +217,14 @@ func PublishAndLogAlienTaskStatusChange(ctx context.Context, kv *api.KV, deploym // PublishAndLogWorkflowStatusChange publishes a status change for a workflow task and log this change into the log API // // PublishAndLogWorkflowStatusChange returns the published event id -func PublishAndLogWorkflowStatusChange(ctx context.Context, kv *api.KV, deploymentID, taskID, status string) (string, error) { +func PublishAndLogWorkflowStatusChange(ctx context.Context, kv *api.KV, deploymentID, taskID, workflowID, status string) (string, error) { if ctx == nil { ctx = NewContext(context.Background(), LogOptionalFields{ExecutionID: taskID}) } info := make(Info) info[infoAlienExecutionID] = taskID - e, err := newStatusChange(StatusChangeTypeWorkflow, info, deploymentID, status) + info[infoWorkflowID] = workflowID + e, err := newStatusChange(StatusChangeTypeWorkflow, info, deploymentID, strings.ToLower(status)) if err != nil { return "", err } diff --git a/events/events_test.go b/events/events_test.go index 096adfacf..be6462c85 100644 --- a/events/events_test.go +++ b/events/events_test.go @@ -35,7 +35,7 @@ import ( func testConsulPubSubStatusChange(t *testing.T, kv *api.KV) { t.Parallel() deploymentID := testutil.BuildDeploymentID(t) - + ctx := context.Background() var testData = []struct { node string instance string @@ -55,7 +55,7 @@ func testConsulPubSubStatusChange(t *testing.T, kv *api.KV) { ids := make([]string, 0) for _, tc := range testData { - id, err := InstanceStatusChange(kv, deploymentID, tc.node, tc.instance, tc.status) + id, err := PublishAndLogInstanceStatusChange(ctx, kv, deploymentID, tc.node, tc.instance, tc.status) assert.Nil(t, err) ids = append(ids, id) } @@ -86,7 +86,7 @@ func testConsulPubSubNewEvents(t *testing.T, kv *api.KV) { // Do not run this test in // as it cause some concurrency issue // t.Parallel() deploymentID := testutil.BuildDeploymentID(t) - + ctx := context.Background() nodeName := "node1" instance := "0" nodeStatus := "error" @@ -107,7 +107,7 @@ func testConsulPubSubNewEvents(t *testing.T, kv *api.KV) { assert.Equal(t, event[infoInstanceID.String()], instance) }() <-ready - _, err := InstanceStatusChange(kv, deploymentID, nodeName, instance, nodeStatus) + _, err := PublishAndLogInstanceStatusChange(ctx, kv, deploymentID, nodeName, instance, nodeStatus) assert.Nil(t, err) } @@ -127,6 +127,7 @@ func testConsulPubSubNewEventsTimeout(t *testing.T, kv *api.KV) { func testConsulPubSubNewEventsWithIndex(t *testing.T, kv *api.KV) { t.Parallel() + ctx := context.Background() deploymentID := testutil.BuildDeploymentID(t) var testData = []struct { @@ -141,7 +142,7 @@ func testConsulPubSubNewEventsWithIndex(t *testing.T, kv *api.KV) { } for _, tc := range testData { - _, err := InstanceStatusChange(kv, deploymentID, tc.node, tc.instance, tc.status) + _, err := PublishAndLogInstanceStatusChange(ctx, kv, deploymentID, tc.node, tc.instance, tc.status) assert.Nil(t, err) } @@ -166,7 +167,7 @@ func testConsulPubSubNewEventsWithIndex(t *testing.T, kv *api.KV) { } for _, tc := range testData { - _, err = InstanceStatusChange(kv, deploymentID, tc.node, tc.instance, tc.status) + _, err = PublishAndLogInstanceStatusChange(ctx, kv, deploymentID, tc.node, tc.instance, tc.status) assert.Nil(t, err) } @@ -185,19 +186,21 @@ func testConsulPubSubNewEventsWithIndex(t *testing.T, kv *api.KV) { func testConsulPubSubNewNodeEvents(t *testing.T, kv *api.KV) { t.Parallel() + ctx := context.Background() deploymentID := testutil.BuildDeploymentID(t) nodeName := "node1" instance := "0" nodeStatus := "error" - _, err := InstanceStatusChange(kv, deploymentID, nodeName, instance, nodeStatus) + _, err := PublishAndLogInstanceStatusChange(ctx, kv, deploymentID, nodeName, instance, nodeStatus) assert.Nil(t, err) } func testconsulDeploymentStatusChange(t *testing.T, kv *api.KV) { t.Parallel() + ctx := context.Background() deploymentID := testutil.BuildDeploymentID(t) type args struct { kv *api.KV @@ -215,7 +218,7 @@ func testconsulDeploymentStatusChange(t *testing.T, kv *api.KV) { ids := make([]string, 0) for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := DeploymentStatusChange(tt.args.kv, deploymentID, tt.args.status) + got, err := PublishAndLogDeploymentStatusChange(ctx, tt.args.kv, deploymentID, tt.args.status) if (err != nil) != tt.wantErr { t.Errorf("DeploymentStatusChange() error = %v, wantErr %v", err, tt.wantErr) return @@ -242,6 +245,7 @@ func testconsulDeploymentStatusChange(t *testing.T, kv *api.KV) { func testconsulCustomCommandStatusChange(t *testing.T, kv *api.KV) { t.Parallel() + ctx := context.Background() deploymentID := testutil.BuildDeploymentID(t) type args struct { kv *api.KV @@ -263,7 +267,7 @@ func testconsulCustomCommandStatusChange(t *testing.T, kv *api.KV) { ids := make([]string, 0) for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := CustomCommandStatusChange(tt.args.kv, deploymentID, tt.args.taskID, tt.args.status) + got, err := PublishAndLogCustomCommandStatusChange(ctx, tt.args.kv, deploymentID, tt.args.taskID, tt.args.status) if (err != nil) != tt.wantErr { t.Errorf("CustomCommandStatusChange() error = %v, wantErr %v", err, tt.wantErr) return @@ -292,6 +296,7 @@ func testconsulCustomCommandStatusChange(t *testing.T, kv *api.KV) { func testconsulScalingStatusChange(t *testing.T, kv *api.KV) { t.Parallel() deploymentID := testutil.BuildDeploymentID(t) + ctx := context.Background() type args struct { kv *api.KV taskID string @@ -312,7 +317,7 @@ func testconsulScalingStatusChange(t *testing.T, kv *api.KV) { ids := make([]string, 0) for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := ScalingStatusChange(tt.args.kv, deploymentID, tt.args.taskID, tt.args.status) + got, err := PublishAndLogScalingStatusChange(ctx, tt.args.kv, deploymentID, tt.args.taskID, tt.args.status) if (err != nil) != tt.wantErr { t.Errorf("ScalingStatusChange() error = %v, wantErr %v", err, tt.wantErr) return @@ -340,28 +345,30 @@ func testconsulScalingStatusChange(t *testing.T, kv *api.KV) { func testconsulWorkflowStatusChange(t *testing.T, kv *api.KV) { t.Parallel() + ctx := context.Background() deploymentID := testutil.BuildDeploymentID(t) type args struct { - kv *api.KV - taskID string - status string + kv *api.KV + taskID string + status string + workflowName string } tests := []struct { name string args args wantErr bool }{ - {"TestStatusInitial", args{kv, "1", "initial"}, false}, - {"TestStatusDepInProgress", args{kv, "1", "running"}, false}, - {"TestStatusDepDeployed", args{kv, "2", "initial"}, false}, - {"TestStatusDepDeployed", args{kv, "2", "running"}, false}, - {"TestStatusDepDeployed", args{kv, "1", "done"}, false}, - {"TestStatusDepDeployed", args{kv, "2", "done"}, false}, + {"TestStatusInitial", args{kv, "1", "initial", "install"}, false}, + {"TestStatusDepInProgress", args{kv, "1", "running", "install"}, false}, + {"TestStatusDepDeployed", args{kv, "2", "initial", "install"}, false}, + {"TestStatusDepDeployed", args{kv, "2", "running", "install"}, false}, + {"TestStatusDepDeployed", args{kv, "1", "done", "install"}, false}, + {"TestStatusDepDeployed", args{kv, "2", "done", "install"}, false}, } ids := make([]string, 0) for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := WorkflowStatusChange(tt.args.kv, deploymentID, tt.args.taskID, tt.args.status) + got, err := PublishAndLogWorkflowStatusChange(ctx, tt.args.kv, deploymentID, tt.args.taskID, tt.args.workflowName, tt.args.status) if (err != nil) != tt.wantErr { t.Errorf("WorkflowStatusChange() error = %v, wantErr %v", err, tt.wantErr) return @@ -383,6 +390,7 @@ func testconsulWorkflowStatusChange(t *testing.T, kv *api.KV) { event := toStatusChangeMap(t, string(kvp.Value)) assert.Equal(t, tc.args.status, event[infoStatus.String()]) assert.Equal(t, tc.args.taskID, event[infoAlienExecutionID.String()]) + assert.Equal(t, tc.args.workflowName, event[infoWorkflowID.String()]) } } @@ -432,7 +440,7 @@ func testconsulWorkflowStepStatusChange(t *testing.T, kv *api.KV) { event := toStatusChangeMap(t, string(kvp.Value)) assert.Equal(t, tc.args.status, event[infoStatus.String()]) assert.Equal(t, tc.args.taskID, event[infoAlienExecutionID.String()]) - assert.Equal(t, "install", event[infoWorkFlowID.String()]) + assert.Equal(t, "install", event[infoWorkflowID.String()]) assert.Equal(t, "0", event[infoInstanceID.String()]) assert.Equal(t, "step1", event[infoWorkflowStepID.String()]) assert.Equal(t, "node1", event[infoNodeID.String()]) @@ -487,7 +495,7 @@ func testconsulAlienTaskStatusChange(t *testing.T, kv *api.KV) { assert.Equal(t, tc.args.status, event[infoStatus.String()]) assert.Equal(t, tc.args.taskID, event[infoAlienExecutionID.String()]) assert.Equal(t, tc.args.taskExecutionID, event[infoAlienTaskExecutionID.String()]) - assert.Equal(t, "install", event[infoWorkFlowID.String()]) + assert.Equal(t, "install", event[infoWorkflowID.String()]) assert.Equal(t, "0", event[infoInstanceID.String()]) assert.Equal(t, "step1", event[infoWorkflowStepID.String()]) assert.Equal(t, "node1", event[infoNodeID.String()]) @@ -496,21 +504,22 @@ func testconsulAlienTaskStatusChange(t *testing.T, kv *api.KV) { func testconsulGetStatusEvents(t *testing.T, kv *api.KV) { t.Parallel() + ctx := context.Background() deploymentID := testutil.BuildDeploymentID(t) ids := make([]string, 5) - id, err := InstanceStatusChange(kv, deploymentID, "node1", "1", "started") + id, err := PublishAndLogInstanceStatusChange(ctx, kv, deploymentID, "node1", "1", "started") require.Nil(t, err) ids[0] = id - id, err = DeploymentStatusChange(kv, deploymentID, "deployed") + id, err = PublishAndLogDeploymentStatusChange(ctx, kv, deploymentID, "deployed") require.Nil(t, err) ids[1] = id - id, err = ScalingStatusChange(kv, deploymentID, "t2", "failed") + id, err = PublishAndLogScalingStatusChange(ctx, kv, deploymentID, "t2", "failed") require.Nil(t, err) ids[2] = id - id, err = CustomCommandStatusChange(kv, deploymentID, "t3", "running") + id, err = PublishAndLogCustomCommandStatusChange(ctx, kv, deploymentID, "t3", "running") require.Nil(t, err) ids[3] = id - id, err = WorkflowStatusChange(kv, deploymentID, "t4", "done") + id, err = PublishAndLogWorkflowStatusChange(ctx, kv, deploymentID, "t4", "install", "done") require.Nil(t, err) ids[4] = id @@ -557,7 +566,7 @@ func testconsulGetStatusEvents(t *testing.T, kv *api.KV) { require.Equal(t, "done", events[4][infoStatus.String()]) require.Equal(t, ids[4], events[4][infoTimestamp.String()]) require.Equal(t, "t4", events[4][infoAlienExecutionID.String()]) - + require.Equal(t, "install", events[4][infoWorkflowID.String()]) } func testconsulGetLogs(t *testing.T, kv *api.KV) { diff --git a/events/structs.go b/events/structs.go index a91d91a68..5c1add694 100644 --- a/events/structs.go +++ b/events/structs.go @@ -52,7 +52,7 @@ const ( infoEventType - infoWorkFlowID + infoWorkflowID infoAlienExecutionID @@ -81,10 +81,10 @@ func (i infoType) String() string { return "timestamp" case infoEventType: return "type" - case infoWorkFlowID: - return "workFlowID" + case infoWorkflowID: + return "workflowId" case infoAlienExecutionID: - return "alienExecutionID" + return "alienExecutionId" case infoNodeID: return "nodeId" case infoInstanceID: @@ -178,8 +178,8 @@ func (e *statusChange) check() error { StatusChangeTypeCustomCommand: {infoAlienExecutionID}, StatusChangeTypeScaling: {infoAlienExecutionID}, StatusChangeTypeWorkflow: {infoAlienExecutionID}, - StatusChangeTypeWorkflowStep: {infoAlienExecutionID, infoWorkFlowID, infoNodeID, infoWorkflowStepID, infoInstanceID}, - StatusChangeTypeAlienTask: {infoAlienExecutionID, infoWorkFlowID, infoNodeID, infoWorkflowStepID, infoInstanceID, infoAlienTaskExecutionID}, + StatusChangeTypeWorkflowStep: {infoAlienExecutionID, infoWorkflowID, infoNodeID, infoWorkflowStepID, infoInstanceID}, + StatusChangeTypeAlienTask: {infoAlienExecutionID, infoWorkflowID, infoNodeID, infoWorkflowStepID, infoInstanceID, infoAlienTaskExecutionID}, } // Check mandatory info in function of status change type if mandatoryInfos, is := mandatoryMap[e.eventType]; is { diff --git a/tasks/collector.go b/tasks/collector.go index 993b3bb15..0b33f0999 100644 --- a/tasks/collector.go +++ b/tasks/collector.go @@ -113,7 +113,7 @@ func (c *Collector) registerTaskWithoutDestroyLock(targetID string, taskType Tas if err != nil { return nil, nil, taskID, err } - EmitTaskEventWithContextualLogs(nil, kv, targetID, taskID, taskType, TaskStatusINITIAL.String()) + EmitTaskEventWithContextualLogs(nil, kv, targetID, taskID, taskType, "unknown", TaskStatusINITIAL.String()) destroy := func(taskLockCreate *api.Lock, taskId, targetId string) { log.Debugf("Unlocking newly created task with id %q (target id %q)", taskId, targetId) diff --git a/tasks/collector/collector.go b/tasks/collector/collector.go index d484a0201..94ee51d46 100644 --- a/tasks/collector/collector.go +++ b/tasks/collector/collector.go @@ -106,7 +106,8 @@ func (c *Collector) ResumeTask(taskID string) error { return err } - tasks.EmitTaskEventWithContextualLogs(nil, c.consulClient.KV(), targetID, taskID, taskType, tasks.TaskStatusINITIAL.String()) + tasks.EmitTaskEventWithContextualLogs(nil, c.consulClient.KV(), targetID, taskID, taskType, workflowName, tasks.TaskStatusINITIAL.String()) + tasks.EmitTaskEventWithContextualLogs(nil, c.consulClient.KV(), targetID, taskID, taskType, workflowName, tasks.TaskStatusRUNNING.String()) return nil } @@ -174,7 +175,8 @@ func (c *Collector) registerTask(targetID string, taskType tasks.TaskType, data return "", err } - tasks.EmitTaskEventWithContextualLogs(nil, c.consulClient.KV(), targetID, taskID, taskType, tasks.TaskStatusINITIAL.String()) + tasks.EmitTaskEventWithContextualLogs(nil, c.consulClient.KV(), targetID, taskID, taskType, workflowName, tasks.TaskStatusINITIAL.String()) + tasks.EmitTaskEventWithContextualLogs(nil, c.consulClient.KV(), targetID, taskID, taskType, workflowName, tasks.TaskStatusRUNNING.String()) return taskID, nil } diff --git a/tasks/tasks.go b/tasks/tasks.go index f668f9c87..74b8dd440 100644 --- a/tasks/tasks.go +++ b/tasks/tasks.go @@ -319,11 +319,11 @@ func IsTaskRelatedNode(kv *api.KV, taskID, nodeName string) (bool, error) { // // Deprecated: use EmitTaskEventWithContextualLogs instead func EmitTaskEvent(kv *api.KV, deploymentID, taskID string, taskType TaskType, status string) (string, error) { - return EmitTaskEventWithContextualLogs(nil, kv, deploymentID, taskID, taskType, status) + return EmitTaskEventWithContextualLogs(nil, kv, deploymentID, taskID, taskType, "unknown", status) } // EmitTaskEventWithContextualLogs emits a task event based on task type -func EmitTaskEventWithContextualLogs(ctx context.Context, kv *api.KV, deploymentID, taskID string, taskType TaskType, status string) (string, error) { +func EmitTaskEventWithContextualLogs(ctx context.Context, kv *api.KV, deploymentID, taskID string, taskType TaskType, workflowName, status string) (string, error) { if ctx == nil { ctx = events.NewContext(context.Background(), events.LogOptionalFields{events.ExecutionID: taskID}) } @@ -331,7 +331,7 @@ func EmitTaskEventWithContextualLogs(ctx context.Context, kv *api.KV, deployment case TaskTypeCustomCommand: return events.PublishAndLogCustomCommandStatusChange(ctx, kv, deploymentID, taskID, strings.ToLower(status)) case TaskTypeCustomWorkflow, TaskTypeDeploy, TaskTypeUnDeploy: - return events.PublishAndLogWorkflowStatusChange(ctx, kv, deploymentID, taskID, strings.ToLower(status)) + return events.PublishAndLogWorkflowStatusChange(ctx, kv, deploymentID, taskID, workflowName, strings.ToLower(status)) case TaskTypeScaleIn, TaskTypeScaleOut: return events.PublishAndLogScalingStatusChange(ctx, kv, deploymentID, taskID, strings.ToLower(status)) } diff --git a/tasks/workflow/step.go b/tasks/workflow/step.go index 6f033eeda..ff2dd175d 100644 --- a/tasks/workflow/step.go +++ b/tasks/workflow/step.go @@ -171,13 +171,7 @@ func (s *step) run(ctx context.Context, cfg config.Configuration, kv *api.KV, de s.setStatus(tasks.TaskStepStatusDONE) return nil } - - // Let's publish initial event status for workflow step (just before run it) - eventInfo := &events.WorkflowStepInfo{WorkflowName: workflowName, NodeName: s.Target, StepName: s.Name} - events.PublishAndLogWorkflowStepStatusChange(ctx, kv, deploymentID, s.t.taskID, eventInfo, tasks.TaskStepStatusINITIAL.String()) - events.PublishAndLogAlienTaskStatusChange(ctx, kv, deploymentID, s.t.taskID, s.t.id, eventInfo, tasks.TaskStepStatusINITIAL.String()) s.setStatus(tasks.TaskStepStatusRUNNING) - logOptFields, _ := events.FromContext(ctx) // Create a new context to handle gracefully current step termination when an error occurred during another step wfCtx, cancelWf := context.WithCancel(events.NewContext(context.Background(), logOptFields)) @@ -260,6 +254,13 @@ func (s *step) runActivity(wfCtx context.Context, kv *api.KV, cfg config.Configu return err } + //// Let's publish initial event status for workflow step (just before run it) + //for _, instanceName := range instances { + // eventInfo := &events.WorkflowStepInfo{WorkflowName: workflowName, NodeName: s.Target, StepName: s.Name, InstanceName: instanceName} + // events.PublishAndLogWorkflowStepStatusChange(wfCtx, kv, deploymentID, s.t.taskID, eventInfo, tasks.TaskStepStatusINITIAL.String()) + // events.PublishAndLogAlienTaskStatusChange(wfCtx, kv, deploymentID, s.t.taskID, s.t.id, eventInfo, tasks.TaskStepStatusINITIAL.String()) + //} + eventInfo := &events.WorkflowStepInfo{WorkflowName: workflowName, NodeName: s.Target, StepName: s.Name} switch activity.Type() { case builder.ActivityTypeDelegate: @@ -275,6 +276,10 @@ func (s *step) runActivity(wfCtx context.Context, kv *api.KV, cfg config.Configu wfCtx = events.AddLogOptionalFields(wfCtx, events.LogOptionalFields{events.InterfaceName: "delegate", events.OperationName: delegateOp}) for _, instanceName := range instances { eventInfo.InstanceName = instanceName + eventInfo.OperationName = fmt.Sprintf("delegate.%s", delegateOp) + // Need to publish INITIAL status before RUNNING one with operationName set + events.PublishAndLogWorkflowStepStatusChange(wfCtx, kv, deploymentID, s.t.taskID, eventInfo, tasks.TaskStepStatusINITIAL.String()) + events.PublishAndLogAlienTaskStatusChange(wfCtx, kv, deploymentID, s.t.taskID, s.t.id, eventInfo, tasks.TaskStepStatusINITIAL.String()) events.PublishAndLogWorkflowStepStatusChange(wfCtx, kv, deploymentID, s.t.taskID, eventInfo, tasks.TaskStepStatusRUNNING.String()) events.PublishAndLogAlienTaskStatusChange(wfCtx, kv, deploymentID, s.t.taskID, s.t.id, eventInfo, tasks.TaskStepStatusRUNNING.String()) } @@ -326,6 +331,9 @@ func (s *step) runActivity(wfCtx context.Context, kv *api.KV, cfg config.Configu if op.RelOp.IsRelationshipOperation { eventInfo.TargetNodeID = op.RelOp.TargetNodeName } + // Need to publish INITIAL status before RUNNING one with operationName set + events.PublishAndLogWorkflowStepStatusChange(wfCtx, kv, deploymentID, s.t.taskID, eventInfo, tasks.TaskStepStatusINITIAL.String()) + events.PublishAndLogAlienTaskStatusChange(wfCtx, kv, deploymentID, s.t.taskID, s.t.id, eventInfo, tasks.TaskStepStatusINITIAL.String()) events.PublishAndLogWorkflowStepStatusChange(wfCtx, kv, deploymentID, s.t.taskID, eventInfo, tasks.TaskStepStatusRUNNING.String()) events.PublishAndLogAlienTaskStatusChange(wfCtx, kv, deploymentID, s.t.taskID, s.t.id, eventInfo, tasks.TaskStepStatusRUNNING.String()) } diff --git a/tasks/workflow/worker.go b/tasks/workflow/worker.go index f825b30ee..cb91f4dfa 100644 --- a/tasks/workflow/worker.go +++ b/tasks/workflow/worker.go @@ -128,7 +128,7 @@ func getOperationExecutor(kv *api.KV, deploymentID, artifact string) (prov.Opera return nil, originalErr } -func (w *worker) monitorTaskCancellation(ctx context.Context, cancelFunc context.CancelFunc, t *taskExecution) { +func (w *worker) monitorTaskCancellation(ctx context.Context, cancelFunc context.CancelFunc, t *taskExecution, workflowName string) { go func() { var lastIndex uint64 for { @@ -149,7 +149,7 @@ func (w *worker) monitorTaskCancellation(ctx context.Context, cancelFunc context if strings.ToLower(string(kvp.Value)) == "true" { log.Debugln("Task cancellation requested.") checkAndSetTaskStatus(ctx, t.kv, t.taskID, t.step, tasks.TaskStatusCANCELED) - events.PublishAndLogWorkflowStatusChange(ctx, w.consulClient.KV(), t.targetID, t.taskID, tasks.TaskStatusCANCELED.String()) + events.PublishAndLogWorkflowStatusChange(ctx, w.consulClient.KV(), t.targetID, t.taskID, workflowName, tasks.TaskStatusCANCELED.String()) cancelFunc() return } @@ -285,7 +285,7 @@ func (w *worker) handleExecution(t *taskExecution) { log.Printf("%+v", err) return } - w.monitorTaskCancellation(ctx, cancelFunc, t) + w.monitorTaskCancellation(ctx, cancelFunc, t, wfName) w.monitorTaskFailure(ctx, cancelFunc, t) defer func(t *taskExecution, start time.Time) { if taskStatus, err := t.getTaskStatus(); err != nil && taskStatus != tasks.TaskStatusRUNNING { @@ -577,7 +577,7 @@ func (w *worker) runDeploy(ctx context.Context, t *taskExecution) { log.Printf("%+v", err) w.checkAndSetDeploymentStatus(ctx, t.targetID, deployments.DEPLOYMENT_FAILED) checkAndSetTaskStatus(ctx, t.kv, t.taskID, t.step, tasks.TaskStatusFAILED) - events.PublishAndLogWorkflowStatusChange(ctx, w.consulClient.KV(), t.targetID, t.taskID, tasks.TaskStatusFAILED.String()) + events.PublishAndLogWorkflowStatusChange(ctx, w.consulClient.KV(), t.targetID, t.taskID, "install", tasks.TaskStatusFAILED.String()) return } if wfDone { @@ -587,7 +587,7 @@ func (w *worker) runDeploy(ctx context.Context, t *taskExecution) { log.Printf("%+v", err) return } - events.PublishAndLogWorkflowStatusChange(ctx, w.consulClient.KV(), t.targetID, t.taskID, tasks.TaskStatusDONE.String()) + events.PublishAndLogWorkflowStatusChange(ctx, w.consulClient.KV(), t.targetID, t.taskID, "install", tasks.TaskStatusDONE.String()) } } @@ -605,7 +605,7 @@ func (w *worker) runUndeploy(ctx context.Context, t *taskExecution) { log.Printf("%+v", err) w.checkAndSetDeploymentStatus(ctx, t.targetID, deployments.UNDEPLOYMENT_FAILED) checkAndSetTaskStatus(ctx, t.kv, t.taskID, t.step, tasks.TaskStatusFAILED) - events.PublishAndLogWorkflowStatusChange(ctx, w.consulClient.KV(), t.targetID, t.taskID, tasks.TaskStatusFAILED.String()) + events.PublishAndLogWorkflowStatusChange(ctx, w.consulClient.KV(), t.targetID, t.taskID, "uninstall", tasks.TaskStatusFAILED.String()) return } if wfDone { @@ -619,7 +619,7 @@ func (w *worker) runUndeploy(ctx context.Context, t *taskExecution) { w.runPurge(ctx, t) } } - events.PublishAndLogWorkflowStatusChange(ctx, w.consulClient.KV(), t.targetID, t.taskID, tasks.TaskStatusDONE.String()) + events.PublishAndLogWorkflowStatusChange(ctx, w.consulClient.KV(), t.targetID, t.taskID, "uninstall", tasks.TaskStatusDONE.String()) } else if t.taskType == tasks.TaskTypePurge { w.runPurge(ctx, t) } From d014730e42767108f2c147b63e404f86f65cba28 Mon Sep 17 00:00:00 2001 From: stephane benoist <29753229+stebenoist@users.noreply.github.com> Date: Wed, 5 Dec 2018 13:08:25 +0100 Subject: [PATCH 08/19] Add consul db schema upgrade with snapshot save/restore --- server/consul.go | 34 ++++--- server/upgradeschema/upgrade_pre31.go | 126 ++++++++++++++++++++++++++ 2 files changed, 148 insertions(+), 12 deletions(-) create mode 100644 server/upgradeschema/upgrade_pre31.go diff --git a/server/consul.go b/server/consul.go index 1f41b7726..1b81f07d0 100644 --- a/server/consul.go +++ b/server/consul.go @@ -21,10 +21,11 @@ import ( "github.com/ystia/yorc/helper/consulutil" "github.com/ystia/yorc/log" + "github.com/ystia/yorc/server/upgradeschema" ) var upgradeToMap = map[string]func(*api.KV, <-chan struct{}) error{ - "1.0.0": upgradeFromPre31, + "1.0.0": upgradeschema.UpgradeFromPre31, } var orderedUpgradesVersions []semver.Version @@ -86,13 +87,13 @@ func setupConsulDBSchema(client *api.Client) error { return errors.Wrap(err, consulutil.ConsulGenericErrMsg) } if len(kvps) > 0 { - return upgradeFromVersion(kv, leaderCh, "0.0.0") + return upgradeFromVersion(client, leaderCh, "0.0.0") } return setNewVersion(kv) } - return upgradeFromVersion(kv, leaderCh, string(kvp.Value)) + return upgradeFromVersion(client, leaderCh, string(kvp.Value)) } func setNewVersion(kv *api.KV) error { @@ -103,7 +104,7 @@ func setNewVersion(kv *api.KV) error { return nil } -func upgradeFromVersion(kv *api.KV, leaderCh <-chan struct{}, fromVersion string) error { +func upgradeFromVersion(client *api.Client, leaderCh <-chan struct{}, fromVersion string) error { vCurrent, err := semver.Make(fromVersion) if err != nil { return errors.Wrapf(err, "failed to parse current version of consul db schema") @@ -118,11 +119,25 @@ func upgradeFromVersion(kv *api.KV, leaderCh <-chan struct{}, fromVersion string // Same version nothing to do return nil case 1: + // Make a Consul snapshot and restore it if any error occurs + snap := client.Snapshot() + snapReader, _, err := snap.Save(nil) + if err != nil { + return errors.Wrapf(err, "failed to upgrade consul db schema to %q", err) + } + defer snapReader.Close() for _, vUp := range orderedUpgradesVersions { if vUp.GE(vCurrent) { - err = upgradeToMap[vUp.String()](kv, leaderCh) + err = upgradeToMap[vUp.String()](client.KV(), leaderCh) if err != nil { - return errors.Wrapf(err, "failed to upgrade consul db schema to %q", vUp) + // Restore Consul snapshot + restoreErr := snap.Restore(nil, snapReader) + if restoreErr != nil { + log.Printf("failed to restore consul db schema to %q due to error:%+v", fromVersion, restoreErr) + } else { + log.Printf("As any error occurred, schema has been successfully restored to version %q", fromVersion) + } + return errors.Wrapf(err, "failed to upgrade consul db schema to %q.", vUp) } } } @@ -131,10 +146,5 @@ func upgradeFromVersion(kv *api.KV, leaderCh <-chan struct{}, fromVersion string } - return setNewVersion(kv) -} - -func upgradeFromPre31(kv *api.KV, leaderch <-chan struct{}) error { - // Nothing to do right now - return nil + return setNewVersion(client.KV()) } diff --git a/server/upgradeschema/upgrade_pre31.go b/server/upgradeschema/upgrade_pre31.go new file mode 100644 index 000000000..c9800a289 --- /dev/null +++ b/server/upgradeschema/upgrade_pre31.go @@ -0,0 +1,126 @@ +// Copyright 2018 Bull S.A.S. Atos Technologies - Bull, Rue Jean Jaures, B.P.68, 78340, Les Clayes-sous-Bois, France. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package upgradeschema + +import ( + "encoding/json" + "github.com/hashicorp/consul/api" + "github.com/pkg/errors" + "github.com/ystia/yorc/events" + "github.com/ystia/yorc/helper/consulutil" + "github.com/ystia/yorc/log" + "path" + "strings" +) + +// UpgradeFromPre31 allows to upgrade Consul schema from schema version before 1.0.0 (pre 3.1 yorc version) +func UpgradeFromPre31(kv *api.KV, leaderch <-chan struct{}) error { + log.Print("Preparing upgrade database schema to 1.0.0 schema version") + return eventsChange(kv, leaderch) +} + +func eventsChange(kv *api.KV, leaderch <-chan struct{}) error { + // Events format has changed + // Need to retrieve the previous StatusUpdateType and related event information + // To store with the new format (JSON) + log.Print("Update events format change") + eventsPrefix := path.Join(consulutil.EventsPrefix) + type StatusUpdateType uint64 + const ( + // InstanceStatusChangeType is the StatusUpdate type for an instance state change event + InstanceStatusChangeType StatusUpdateType = iota + // DeploymentStatusChangeType is the StatusUpdate type for an deployment status change event + DeploymentStatusChangeType + // CustomCommandStatusChangeType is the StatusUpdate type for an custom command status change event + CustomCommandStatusChangeType + // ScalingStatusChangeType is the StatusUpdate type for an scaling status change event + ScalingStatusChangeType + // WorkflowStatusChangeType is the StatusUpdate type for an workflow status change event + WorkflowStatusChangeType + ) + + stringStatusUpdate := func(s StatusUpdateType) string { + switch s { + case InstanceStatusChangeType: + return events.StatusChangeTypeInstance.String() + case DeploymentStatusChangeType: + return events.StatusChangeTypeDeployment.String() + case CustomCommandStatusChangeType: + return events.StatusChangeTypeCustomCommand.String() + case ScalingStatusChangeType: + return events.StatusChangeTypeScaling.String() + case WorkflowStatusChangeType: + return events.StatusChangeTypeWorkflow.String() + } + return "" + } + + kvps, qm, err := kv.List(eventsPrefix, nil) + if err != nil || qm == nil { + return errors.Wrapf(err, "failed to upgrade consul database schema from 100") + } + for _, kvp := range kvps { + event := make(map[string]string) + depIDAndTimestamp := strings.Split(strings.TrimPrefix(kvp.Key, eventsPrefix+"/"), "/") + deploymentID := depIDAndTimestamp[0] + eventTimestamp := depIDAndTimestamp[1] + + values := strings.Split(string(kvp.Value), "\n") + eventType := StatusUpdateType(kvp.Flags) + switch eventType { + case InstanceStatusChangeType: + if len(values) != 3 { + return errors.Errorf("failed to upgrade consul database schema from 100: unexpected event value %q for event %q", string(kvp.Value), kvp.Key) + } + event["deploymentId"] = deploymentID + event["timestamp"] = eventTimestamp + event["type"] = stringStatusUpdate(eventType) + event["nodeId"] = values[0] + event["status"] = values[1] + event["instanceId"] = values[2] + case DeploymentStatusChangeType: + if len(values) != 1 { + return errors.Errorf("failed to upgrade consul database schema from 100: unexpected event value %q for event %q", string(kvp.Value), kvp.Key) + } + event["deploymentId"] = deploymentID + event["timestamp"] = eventTimestamp + event["type"] = stringStatusUpdate(eventType) + event["status"] = values[0] + case CustomCommandStatusChangeType, ScalingStatusChangeType, WorkflowStatusChangeType: + if len(values) != 2 { + return errors.Errorf("failed to upgrade consul database schema from 100: unexpected event value %q for event %q", string(kvp.Value), kvp.Key) + } + event["deploymentId"] = deploymentID + event["timestamp"] = eventTimestamp + event["type"] = stringStatusUpdate(eventType) + event["status"] = values[1] + event["alienExecutionId"] = values[0] + default: + return errors.Errorf("failed to upgrade consul database schema from 100: unsupported event type %d for event %q", kvp.Flags, kvp.Key) + } + + // Save new format value + log.Debugf("Convert event format with event:%+v", event) + b, err := json.Marshal(event) + if err != nil { + log.Printf("failed to upgrade consul database schema from 100: failed to marshal event [%+v]: due to error:%+v", event, err) + } + err = consulutil.StoreConsulKey(path.Join(eventsPrefix, deploymentID, event["timestamp"]), b) + if err != nil { + return errors.Wrapf(err, "failed to upgrade consul database schema from 100") + } + } + return nil +} From 4b84c5eeb0061b01198f74bde4eb7e8ba581497b Mon Sep 17 00:00:00 2001 From: stephane benoist <29753229+stebenoist@users.noreply.github.com> Date: Wed, 5 Dec 2018 15:30:52 +0100 Subject: [PATCH 09/19] make CLI events output more beautiful --- commands/deployments/dep_events.go | 82 +++++++++++++++-- events/events.go | 47 +++++----- events/events_test.go | 138 ++++++++++++++--------------- events/structs.go | 112 ++++++++++++----------- 4 files changed, 227 insertions(+), 152 deletions(-) diff --git a/commands/deployments/dep_events.go b/commands/deployments/dep_events.go index 2ae2d1b71..ef4a50475 100644 --- a/commands/deployments/dep_events.go +++ b/commands/deployments/dep_events.go @@ -27,6 +27,7 @@ import ( "github.com/pkg/errors" "github.com/spf13/cobra" "github.com/ystia/yorc/commands/httputil" + "github.com/ystia/yorc/events" "github.com/ystia/yorc/rest" ) @@ -127,11 +128,7 @@ func StreamsEvents(client *httputil.YorcClient, deploymentID string, colorize, f } lastIdx = evts.LastIndex for _, event := range evts.Events { - if colorize { - fmt.Printf("%s\n", color.MagentaString("%s", string(event))) - } else { - fmt.Printf("%s\n", string(event)) - } + fmt.Printf("%s\n", formatEvent(event, colorize)) } response.Body.Close() if stop { @@ -139,3 +136,78 @@ func StreamsEvents(client *httputil.YorcClient, deploymentID string, colorize, f } } } + +func formatEvent(event json.RawMessage, colorize bool) string { + //fmt.Printf("%q", event) + var ret string + var data map[string]string + err := json.Unmarshal(event, &data) + if err != nil { + if colorize { + ret += fmt.Sprintf("%s: ", color.MagentaString("Warning")) + } else { + ret += fmt.Sprintf("Warning: ") + } + ret += fmt.Sprintf("Failed to unmarshal json:%q", event) + return ret + } + + evType, is := data[events.EType.String()] + if !is { + if colorize { + ret += fmt.Sprintf("%s: ", color.MagentaString("Warning")) + } else { + ret += fmt.Sprintf("Warning: ") + } + ret += fmt.Sprintf("Misssing event type in json event:%q", event) + return ret + } + + ts := data[events.ETimestamp.String()] + if colorize { + ts = color.CyanString("%s", ts) + } + + statusChange, err := events.ParseStatusChangeType(evType) + if err != nil { + if colorize { + ret += fmt.Sprintf("%s: ", color.MagentaString("Warning")) + } else { + ret += fmt.Sprintf("Warning: ") + } + ret += fmt.Sprintf("Unknown event type: %q\n", evType) + return ret + } + + switch statusChange { + case events.StatusChangeTypeInstance: + ret = fmt.Sprintf("%s:\t Deployment: %s\t Node: %s\t Instance: %s\t Status: %s\n", ts, data[events.EDeploymentID.String()], data[events.ENodeID.String()], data[events.EInstanceID.String()], data[events.EStatus.String()]) + case events.StatusChangeTypeDeployment: + ret = fmt.Sprintf("%s:\t Deployment: %s\t Deployment Status: %s\n", ts, data[events.EDeploymentID.String()], data[events.EStatus.String()]) + case events.StatusChangeTypeCustomCommand: + ret = fmt.Sprintf("%s:\t Deployment: %s\t Task %q (custom command)\t Status: %s\n", ts, data[events.EDeploymentID.String()], data[events.ETaskID.String()], data[events.EStatus.String()]) + case events.StatusChangeTypeScaling: + ret = fmt.Sprintf("%s:\t Deployment: %s\t Task %q (scaling)\t Status: %s\n", ts, data[events.EDeploymentID.String()], data[events.ETaskID.String()], data[events.EStatus.String()]) + case events.StatusChangeTypeWorkflow: + ret = fmt.Sprintf("%s:\t Deployment: %s\t Task %q (workflow)\t Workflow: %s\t Status: %s\n", ts, data[events.EDeploymentID.String()], data[events.ETaskID.String()], data[events.EWorkflowID.String()], data[events.EStatus.String()]) + case events.StatusChangeTypeWorkflowStep: + ret = fmt.Sprintf("%s:\t Deployment: %s\t Task %q (workflowStep)\t Workflow: %s\t Instance: %s\t Step: %s\t Node: %s\t Operation: %s%s\t Status: %s\n", ts, data[events.EDeploymentID.String()], + data[events.ETaskID.String()], data[events.EWorkflowID.String()], data[events.EInstanceID.String()], data[events.EWorkflowStepID.String()], data[events.EOperationName.String()], data[events.ENodeID.String()], formatOptionalInfo(data), data[events.EStatus.String()]) + case events.StatusChangeTypeAlienTask: + ret = fmt.Sprintf("%s:\t Deployment: %s\t Task %q (Execution)\t Execution: %q\t Workflow: %s\t Instance: %s\t Step: %s\t Node: %s\t Operation: %s%s\t Status: %s\n", ts, data[events.EDeploymentID.String()], + data[events.ETaskID.String()], data[events.ETaskExecutionID.String()], data[events.EWorkflowID.String()], data[events.EInstanceID.String()], data[events.EWorkflowStepID.String()], data[events.EOperationName.String()], data[events.ENodeID.String()], formatOptionalInfo(data), data[events.EStatus.String()]) + } + + return ret +} + +func formatOptionalInfo(data map[string]string) string { + var ret string + if targetInst, is := data[events.ETargetInstanceID.String()]; is && targetInst != "" { + ret += fmt.Sprintf("\t TargetInstance: %s", targetInst) + } + if targetNode, is := data[events.ETargetNodeID.String()]; is && targetNode != "" { + ret += fmt.Sprintf("\t TargetNode: %s", targetNode) + } + return ret +} diff --git a/events/events.go b/events/events.go index 04bff2dfc..c49330553 100644 --- a/events/events.go +++ b/events/events.go @@ -45,8 +45,8 @@ func PublishAndLogInstanceStatusChange(ctx context.Context, kv *api.KV, deployme ctx = AddLogOptionalFields(ctx, LogOptionalFields{NodeID: nodeName, InstanceID: instance}) info := make(Info) - info[infoNodeID] = nodeName - info[infoInstanceID] = instance + info[ENodeID] = nodeName + info[EInstanceID] = instance e, err := newStatusChange(StatusChangeTypeInstance, info, deploymentID, strings.ToLower(status)) if err != nil { return "", err @@ -101,7 +101,7 @@ func PublishAndLogCustomCommandStatusChange(ctx context.Context, kv *api.KV, dep ctx = NewContext(context.Background(), LogOptionalFields{ExecutionID: taskID}) } info := make(Info) - info[infoAlienExecutionID] = taskID + info[ETaskID] = taskID e, err := newStatusChange(StatusChangeTypeCustomCommand, info, deploymentID, strings.ToLower(status)) if err != nil { return "", err @@ -131,7 +131,7 @@ func PublishAndLogScalingStatusChange(ctx context.Context, kv *api.KV, deploymen ctx = NewContext(context.Background(), LogOptionalFields{ExecutionID: taskID}) } info := make(Info) - info[infoAlienExecutionID] = taskID + info[ETaskID] = taskID e, err := newStatusChange(StatusChangeTypeScaling, info, deploymentID, strings.ToLower(status)) if err != nil { return "", err @@ -164,14 +164,14 @@ func PublishAndLogWorkflowStepStatusChange(ctx context.Context, kv *api.KV, depl return "", errors.Errorf("WorkflowStep information param must be provided") } info := make(Info) - info[infoAlienExecutionID] = taskID - info[infoInstanceID] = wfStepInfo.InstanceName - info[infoWorkflowID] = wfStepInfo.WorkflowName - info[infoNodeID] = wfStepInfo.NodeName - info[infoWorkflowStepID] = wfStepInfo.StepName - info[infoOperationName] = wfStepInfo.OperationName - info[infoTargetNodeID] = wfStepInfo.TargetNodeID - info[infoTargetInstanceID] = wfStepInfo.TargetInstanceID + info[ETaskID] = taskID + info[EInstanceID] = wfStepInfo.InstanceName + info[EWorkflowID] = wfStepInfo.WorkflowName + info[ENodeID] = wfStepInfo.NodeName + info[EWorkflowStepID] = wfStepInfo.StepName + info[EOperationName] = wfStepInfo.OperationName + info[ETargetNodeID] = wfStepInfo.TargetNodeID + info[ETargetInstanceID] = wfStepInfo.TargetInstanceID e, err := newStatusChange(StatusChangeTypeWorkflowStep, info, deploymentID, strings.ToLower(status)) if err != nil { return "", err @@ -192,16 +192,16 @@ func PublishAndLogAlienTaskStatusChange(ctx context.Context, kv *api.KV, deploym ctx = NewContext(context.Background(), LogOptionalFields{ExecutionID: taskID}) } info := make(Info) - info[infoAlienExecutionID] = taskID + info[ETaskID] = taskID // Warning: Alien task corresponds to what we call taskExecution - info[infoAlienTaskExecutionID] = taskExecutionID - info[infoWorkflowID] = wfStepInfo.WorkflowName - info[infoNodeID] = wfStepInfo.NodeName - info[infoWorkflowStepID] = wfStepInfo.StepName - info[infoInstanceID] = wfStepInfo.InstanceName - info[infoOperationName] = wfStepInfo.OperationName - info[infoTargetNodeID] = wfStepInfo.TargetNodeID - info[infoTargetInstanceID] = wfStepInfo.TargetInstanceID + info[ETaskExecutionID] = taskExecutionID + info[EWorkflowID] = wfStepInfo.WorkflowName + info[ENodeID] = wfStepInfo.NodeName + info[EWorkflowStepID] = wfStepInfo.StepName + info[EInstanceID] = wfStepInfo.InstanceName + info[EOperationName] = wfStepInfo.OperationName + info[ETargetNodeID] = wfStepInfo.TargetNodeID + info[ETargetInstanceID] = wfStepInfo.TargetInstanceID e, err := newStatusChange(StatusChangeTypeAlienTask, info, deploymentID, strings.ToLower(status)) if err != nil { return "", err @@ -222,8 +222,8 @@ func PublishAndLogWorkflowStatusChange(ctx context.Context, kv *api.KV, deployme ctx = NewContext(context.Background(), LogOptionalFields{ExecutionID: taskID}) } info := make(Info) - info[infoAlienExecutionID] = taskID - info[infoWorkflowID] = workflowID + info[ETaskID] = taskID + info[EWorkflowID] = workflowID e, err := newStatusChange(StatusChangeTypeWorkflow, info, deploymentID, strings.ToLower(status)) if err != nil { return "", err @@ -257,7 +257,6 @@ func StatusEvents(kv *api.KV, deploymentID string, waitIndex uint64, timeout tim if kvp.ModifyIndex <= waitIndex { continue } - //eventType := StatusChangeType(kvp.Flags) events = append(events, kvp.Value) } log.Debugf("Found %d events after index", len(events)) diff --git a/events/events_test.go b/events/events_test.go index be6462c85..68e7ef449 100644 --- a/events/events_test.go +++ b/events/events_test.go @@ -68,10 +68,10 @@ func testConsulPubSubStatusChange(t *testing.T, kv *api.KV) { assert.Equal(t, ids[index], strings.TrimPrefix(kvp.Key, prefix+"/")) tc := testData[index] res := toStatusChangeMap(t, string(kvp.Value)) - assert.Equal(t, tc.node, res[infoNodeID.String()], "unexpected node value for statusChange") - assert.Equal(t, tc.status, res[infoStatus.String()], "unexpected status value for statusChange") - assert.Equal(t, deploymentID, res[infoDeploymentID.String()], "unexpected deploymentID value for statusChange") - assert.Equal(t, tc.instance, res[infoInstanceID.String()], "unexpected instance value for statusChange") + assert.Equal(t, tc.node, res[ENodeID.String()], "unexpected node value for statusChange") + assert.Equal(t, tc.status, res[EStatus.String()], "unexpected status value for statusChange") + assert.Equal(t, deploymentID, res[EDeploymentID.String()], "unexpected deploymentID value for statusChange") + assert.Equal(t, tc.instance, res[EInstanceID.String()], "unexpected instance value for statusChange") } } @@ -102,9 +102,9 @@ func testConsulPubSubNewEvents(t *testing.T, kv *api.KV) { require.Len(t, events, 1) event := toStatusChangeMap(t, string(events[0])) - assert.Equal(t, event[infoNodeID.String()], nodeName) - assert.Equal(t, event[infoStatus.String()], nodeStatus) - assert.Equal(t, event[infoInstanceID.String()], instance) + assert.Equal(t, event[ENodeID.String()], nodeName) + assert.Equal(t, event[EStatus.String()], nodeStatus) + assert.Equal(t, event[EInstanceID.String()], instance) }() <-ready _, err := PublishAndLogInstanceStatusChange(ctx, kv, deploymentID, nodeName, instance, nodeStatus) @@ -151,9 +151,9 @@ func testConsulPubSubNewEventsWithIndex(t *testing.T, kv *api.KV) { require.Len(t, rawEvents, 4) for index, event := range rawEvents { evt := toStatusChangeMap(t, string(event)) - assert.Equal(t, testData[index].node, evt[infoNodeID.String()]) - assert.Equal(t, testData[index].instance, evt[infoInstanceID.String()]) - assert.Equal(t, testData[index].status, evt[infoStatus.String()]) + assert.Equal(t, testData[index].node, evt[ENodeID.String()]) + assert.Equal(t, testData[index].instance, evt[EInstanceID.String()]) + assert.Equal(t, testData[index].status, evt[EStatus.String()]) } testData = []struct { @@ -178,9 +178,9 @@ func testConsulPubSubNewEventsWithIndex(t *testing.T, kv *api.KV) { for index, rawEvent := range rawEvents { event := toStatusChangeMap(t, string(rawEvent)) - assert.Equal(t, testData[index].node, event[infoNodeID.String()]) - assert.Equal(t, testData[index].instance, event[infoInstanceID.String()]) - assert.Equal(t, testData[index].status, event[infoStatus.String()]) + assert.Equal(t, testData[index].node, event[ENodeID.String()]) + assert.Equal(t, testData[index].instance, event[EInstanceID.String()]) + assert.Equal(t, testData[index].status, event[EStatus.String()]) } } @@ -238,7 +238,7 @@ func testconsulDeploymentStatusChange(t *testing.T, kv *api.KV) { assert.Equal(t, ids[index], strings.TrimPrefix(kvp.Key, prefix+"/")) tc := tests[index] event := toStatusChangeMap(t, string(kvp.Value)) - assert.Equal(t, tc.args.status, event[infoStatus.String()]) + assert.Equal(t, tc.args.status, event[EStatus.String()]) } } @@ -287,8 +287,8 @@ func testconsulCustomCommandStatusChange(t *testing.T, kv *api.KV) { assert.Equal(t, ids[index], strings.TrimPrefix(kvp.Key, prefix+"/")) tc := tests[index] event := toStatusChangeMap(t, string(kvp.Value)) - assert.Equal(t, tc.args.status, event[infoStatus.String()]) - assert.Equal(t, tc.args.taskID, event[infoAlienExecutionID.String()]) + assert.Equal(t, tc.args.status, event[EStatus.String()]) + assert.Equal(t, tc.args.taskID, event[ETaskID.String()]) } } @@ -337,8 +337,8 @@ func testconsulScalingStatusChange(t *testing.T, kv *api.KV) { assert.Equal(t, ids[index], strings.TrimPrefix(kvp.Key, prefix+"/")) tc := tests[index] event := toStatusChangeMap(t, string(kvp.Value)) - assert.Equal(t, tc.args.status, event[infoStatus.String()]) - assert.Equal(t, tc.args.taskID, event[infoAlienExecutionID.String()]) + assert.Equal(t, tc.args.status, event[EStatus.String()]) + assert.Equal(t, tc.args.taskID, event[ETaskID.String()]) } } @@ -388,9 +388,9 @@ func testconsulWorkflowStatusChange(t *testing.T, kv *api.KV) { assert.Equal(t, ids[index], strings.TrimPrefix(kvp.Key, prefix+"/")) tc := tests[index] event := toStatusChangeMap(t, string(kvp.Value)) - assert.Equal(t, tc.args.status, event[infoStatus.String()]) - assert.Equal(t, tc.args.taskID, event[infoAlienExecutionID.String()]) - assert.Equal(t, tc.args.workflowName, event[infoWorkflowID.String()]) + assert.Equal(t, tc.args.status, event[EStatus.String()]) + assert.Equal(t, tc.args.taskID, event[ETaskID.String()]) + assert.Equal(t, tc.args.workflowName, event[EWorkflowID.String()]) } } @@ -438,12 +438,12 @@ func testconsulWorkflowStepStatusChange(t *testing.T, kv *api.KV) { assert.Equal(t, ids[index], strings.TrimPrefix(kvp.Key, prefix+"/")) tc := tests[index] event := toStatusChangeMap(t, string(kvp.Value)) - assert.Equal(t, tc.args.status, event[infoStatus.String()]) - assert.Equal(t, tc.args.taskID, event[infoAlienExecutionID.String()]) - assert.Equal(t, "install", event[infoWorkflowID.String()]) - assert.Equal(t, "0", event[infoInstanceID.String()]) - assert.Equal(t, "step1", event[infoWorkflowStepID.String()]) - assert.Equal(t, "node1", event[infoNodeID.String()]) + assert.Equal(t, tc.args.status, event[EStatus.String()]) + assert.Equal(t, tc.args.taskID, event[ETaskID.String()]) + assert.Equal(t, "install", event[EWorkflowID.String()]) + assert.Equal(t, "0", event[EInstanceID.String()]) + assert.Equal(t, "step1", event[EWorkflowStepID.String()]) + assert.Equal(t, "node1", event[ENodeID.String()]) } } @@ -492,13 +492,13 @@ func testconsulAlienTaskStatusChange(t *testing.T, kv *api.KV) { assert.Equal(t, ids[index], strings.TrimPrefix(kvp.Key, prefix+"/")) tc := tests[index] event := toStatusChangeMap(t, string(kvp.Value)) - assert.Equal(t, tc.args.status, event[infoStatus.String()]) - assert.Equal(t, tc.args.taskID, event[infoAlienExecutionID.String()]) - assert.Equal(t, tc.args.taskExecutionID, event[infoAlienTaskExecutionID.String()]) - assert.Equal(t, "install", event[infoWorkflowID.String()]) - assert.Equal(t, "0", event[infoInstanceID.String()]) - assert.Equal(t, "step1", event[infoWorkflowStepID.String()]) - assert.Equal(t, "node1", event[infoNodeID.String()]) + assert.Equal(t, tc.args.status, event[EStatus.String()]) + assert.Equal(t, tc.args.taskID, event[ETaskID.String()]) + assert.Equal(t, tc.args.taskExecutionID, event[ETaskExecutionID.String()]) + assert.Equal(t, "install", event[EWorkflowID.String()]) + assert.Equal(t, "0", event[EInstanceID.String()]) + assert.Equal(t, "step1", event[EWorkflowStepID.String()]) + assert.Equal(t, "node1", event[ENodeID.String()]) } } @@ -532,41 +532,41 @@ func testconsulGetStatusEvents(t *testing.T, kv *api.KV) { events[i] = toStatusChangeMap(t, string(rawEvent)) } - require.Equal(t, StatusChangeTypeInstance.String(), events[0][infoEventType.String()]) - require.Equal(t, "node1", events[0][infoNodeID.String()]) - require.Equal(t, "1", events[0][infoInstanceID.String()]) - require.Equal(t, "started", events[0][infoStatus.String()]) - require.Equal(t, ids[0], events[0][infoTimestamp.String()]) - require.Equal(t, "", events[0][infoAlienExecutionID.String()]) - - require.Equal(t, StatusChangeTypeDeployment.String(), events[1][infoEventType.String()]) - require.Equal(t, "", events[1][infoNodeID.String()]) - require.Equal(t, "", events[1][infoInstanceID.String()]) - require.Equal(t, "deployed", events[1][infoStatus.String()]) - require.Equal(t, ids[1], events[1][infoTimestamp.String()]) - require.Equal(t, "", events[1][infoAlienExecutionID.String()]) - - require.Equal(t, StatusChangeTypeScaling.String(), events[2][infoEventType.String()]) - require.Equal(t, "", events[2][infoNodeID.String()]) - require.Equal(t, "", events[2][infoInstanceID.String()]) - require.Equal(t, "failed", events[2][infoStatus.String()]) - require.Equal(t, ids[2], events[2][infoTimestamp.String()]) - require.Equal(t, "t2", events[2][infoAlienExecutionID.String()]) - - require.Equal(t, StatusChangeTypeCustomCommand.String(), events[3][infoEventType.String()]) - require.Equal(t, "", events[3][infoNodeID.String()]) - require.Equal(t, "", events[3][infoInstanceID.String()]) - require.Equal(t, "running", events[3][infoStatus.String()]) - require.Equal(t, ids[3], events[3][infoTimestamp.String()]) - require.Equal(t, "t3", events[3][infoAlienExecutionID.String()]) - - require.Equal(t, StatusChangeTypeWorkflow.String(), events[4][infoEventType.String()]) - require.Equal(t, "", events[4][infoNodeID.String()]) - require.Equal(t, "", events[4][infoInstanceID.String()]) - require.Equal(t, "done", events[4][infoStatus.String()]) - require.Equal(t, ids[4], events[4][infoTimestamp.String()]) - require.Equal(t, "t4", events[4][infoAlienExecutionID.String()]) - require.Equal(t, "install", events[4][infoWorkflowID.String()]) + require.Equal(t, StatusChangeTypeInstance.String(), events[0][EType.String()]) + require.Equal(t, "node1", events[0][ENodeID.String()]) + require.Equal(t, "1", events[0][EInstanceID.String()]) + require.Equal(t, "started", events[0][EStatus.String()]) + require.Equal(t, ids[0], events[0][ETimestamp.String()]) + require.Equal(t, "", events[0][ETaskID.String()]) + + require.Equal(t, StatusChangeTypeDeployment.String(), events[1][EType.String()]) + require.Equal(t, "", events[1][ENodeID.String()]) + require.Equal(t, "", events[1][EInstanceID.String()]) + require.Equal(t, "deployed", events[1][EStatus.String()]) + require.Equal(t, ids[1], events[1][ETimestamp.String()]) + require.Equal(t, "", events[1][ETaskID.String()]) + + require.Equal(t, StatusChangeTypeScaling.String(), events[2][EType.String()]) + require.Equal(t, "", events[2][ENodeID.String()]) + require.Equal(t, "", events[2][EInstanceID.String()]) + require.Equal(t, "failed", events[2][EStatus.String()]) + require.Equal(t, ids[2], events[2][ETimestamp.String()]) + require.Equal(t, "t2", events[2][ETaskID.String()]) + + require.Equal(t, StatusChangeTypeCustomCommand.String(), events[3][EType.String()]) + require.Equal(t, "", events[3][ENodeID.String()]) + require.Equal(t, "", events[3][EInstanceID.String()]) + require.Equal(t, "running", events[3][EStatus.String()]) + require.Equal(t, ids[3], events[3][ETimestamp.String()]) + require.Equal(t, "t3", events[3][ETaskID.String()]) + + require.Equal(t, StatusChangeTypeWorkflow.String(), events[4][EType.String()]) + require.Equal(t, "", events[4][ENodeID.String()]) + require.Equal(t, "", events[4][EInstanceID.String()]) + require.Equal(t, "done", events[4][EStatus.String()]) + require.Equal(t, ids[4], events[4][ETimestamp.String()]) + require.Equal(t, "t4", events[4][ETaskID.String()]) + require.Equal(t, "install", events[4][EWorkflowID.String()]) } func testconsulGetLogs(t *testing.T, kv *api.KV) { diff --git a/events/structs.go b/events/structs.go index 5c1add694..64dddfdef 100644 --- a/events/structs.go +++ b/events/structs.go @@ -38,66 +38,67 @@ import ( type StatusChangeType int // Info allows to provide custom/specific additional information for event -type Info map[infoType]interface{} +type Info map[InfoType]interface{} -// infoType represents Event status change information type -type infoType int +// InfoType represents Event status change information type +type InfoType int const ( - infoDeploymentID infoType = iota - - infoStatus - - infoTimestamp - - infoEventType - - infoWorkflowID - - infoAlienExecutionID - - infoNodeID - - infoInstanceID - - infoOperationName - - infoTargetNodeID - - infoTargetInstanceID - - infoAlienTaskExecutionID - - infoWorkflowStepID + // EDeploymentID is event information related to deploymentIS + EDeploymentID InfoType = iota + // EStatus is event information related to status + EStatus + // ETimestamp is event timestamp + ETimestamp + // EType is type event information + EType + // EWorkflowID is event information related to workflow + EWorkflowID + // ETaskID is event information related to task + ETaskID + // ENodeID is event information related to node + ENodeID + // EInstanceID is event information related to instance + EInstanceID + // EOperationName is event information related to operation + EOperationName + // ETargetNodeID is event information related to target node + ETargetNodeID + // ETargetInstanceID is event information related to target instance + ETargetInstanceID + // ETaskExecutionID is event information related to task execution + ETaskExecutionID + // EWorkflowStepID is event information related to workflow step + EWorkflowStepID ) -func (i infoType) String() string { +func (i InfoType) String() string { switch i { - case infoDeploymentID: + case EDeploymentID: return "deploymentId" - case infoStatus: + case EStatus: return "status" - case infoTimestamp: + case ETimestamp: return "timestamp" - case infoEventType: + case EType: return "type" - case infoWorkflowID: + case EWorkflowID: return "workflowId" - case infoAlienExecutionID: + case ETaskID: return "alienExecutionId" - case infoNodeID: + case ENodeID: return "nodeId" - case infoInstanceID: + case EInstanceID: return "instanceId" - case infoOperationName: + case EOperationName: return "operationName" - case infoTargetNodeID: + case ETargetNodeID: return "targetNodeId" - case infoTargetInstanceID: + case ETargetInstanceID: return "targetInstanceId" - case infoAlienTaskExecutionID: + case ETaskExecutionID: return "alienTaskId" - case infoWorkflowStepID: + case EWorkflowStepID: return "stepId" } return "" @@ -147,12 +148,15 @@ func (e *statusChange) register() (string, error) { func (e *statusChange) flat() map[string]interface{} { flat := make(map[string]interface{}) - flat[infoDeploymentID.String()] = e.deploymentID - flat[infoStatus.String()] = e.status - flat[infoTimestamp.String()] = e.timestamp - flat[infoEventType.String()] = e.eventType.String() + flat[EDeploymentID.String()] = e.deploymentID + flat[EStatus.String()] = e.status + flat[ETimestamp.String()] = e.timestamp + flat[EType.String()] = e.eventType.String() for k, v := range e.info { - flat[k.String()] = v + if v != "" { + flat[k.String()] = v + } + } return flat } @@ -173,13 +177,13 @@ func (e *statusChange) check() error { return errors.New("DeploymentID and status are mandatory parameters for EventStatusVChange") } - mandatoryMap := map[StatusChangeType][]infoType{ - StatusChangeTypeInstance: {infoNodeID, infoInstanceID}, - StatusChangeTypeCustomCommand: {infoAlienExecutionID}, - StatusChangeTypeScaling: {infoAlienExecutionID}, - StatusChangeTypeWorkflow: {infoAlienExecutionID}, - StatusChangeTypeWorkflowStep: {infoAlienExecutionID, infoWorkflowID, infoNodeID, infoWorkflowStepID, infoInstanceID}, - StatusChangeTypeAlienTask: {infoAlienExecutionID, infoWorkflowID, infoNodeID, infoWorkflowStepID, infoInstanceID, infoAlienTaskExecutionID}, + mandatoryMap := map[StatusChangeType][]InfoType{ + StatusChangeTypeInstance: {ENodeID, EInstanceID}, + StatusChangeTypeCustomCommand: {ETaskID}, + StatusChangeTypeScaling: {ETaskID}, + StatusChangeTypeWorkflow: {ETaskID}, + StatusChangeTypeWorkflowStep: {ETaskID, EWorkflowID, ENodeID, EWorkflowStepID, EInstanceID}, + StatusChangeTypeAlienTask: {ETaskID, EWorkflowID, ENodeID, EWorkflowStepID, EInstanceID, ETaskExecutionID}, } // Check mandatory info in function of status change type if mandatoryInfos, is := mandatoryMap[e.eventType]; is { From d691e674123da95c2c8ddc405bcb98b9d4ce84c1 Mon Sep 17 00:00:00 2001 From: stephane benoist <29753229+stebenoist@users.noreply.github.com> Date: Wed, 5 Dec 2018 15:48:11 +0100 Subject: [PATCH 10/19] fix issue on workflow event published too soon --- tasks/workflow/worker.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tasks/workflow/worker.go b/tasks/workflow/worker.go index cb91f4dfa..2c9efe6cc 100644 --- a/tasks/workflow/worker.go +++ b/tasks/workflow/worker.go @@ -596,6 +596,7 @@ func (w *worker) runUndeploy(ctx context.Context, t *taskExecution) { if err != nil { log.Printf("Deployment id: %q, Task id: %q, Failed to get deployment status: %+v", t.targetID, t.taskID, err) checkAndSetTaskStatus(ctx, t.kv, t.taskID, t.step, tasks.TaskStatusFAILED) + events.PublishAndLogWorkflowStatusChange(ctx, w.consulClient.KV(), t.targetID, t.taskID, "uninstall", tasks.TaskStatusFAILED.String()) return } if status != deployments.UNDEPLOYED { @@ -618,8 +619,8 @@ func (w *worker) runUndeploy(ctx context.Context, t *taskExecution) { if t.taskType == tasks.TaskTypePurge { w.runPurge(ctx, t) } + events.PublishAndLogWorkflowStatusChange(ctx, w.consulClient.KV(), t.targetID, t.taskID, "uninstall", tasks.TaskStatusDONE.String()) } - events.PublishAndLogWorkflowStatusChange(ctx, w.consulClient.KV(), t.targetID, t.taskID, "uninstall", tasks.TaskStatusDONE.String()) } else if t.taskType == tasks.TaskTypePurge { w.runPurge(ctx, t) } From 5f3c5360ef3d070121f7f548e27f379c6bfd801b Mon Sep 17 00:00:00 2001 From: stephane benoist <29753229+stebenoist@users.noreply.github.com> Date: Wed, 5 Dec 2018 16:23:15 +0100 Subject: [PATCH 11/19] cleanup some code --- events/log_entries.go | 6 +++--- tasks/workflow/step.go | 7 ------- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/events/log_entries.go b/events/log_entries.go index b25db3ee7..16b675426 100644 --- a/events/log_entries.go +++ b/events/log_entries.go @@ -42,10 +42,10 @@ type LogEntryDraft struct { additionalInfo LogOptionalFields } -// LogOptionalFields are log's additional additionalInfo +// LogOptionalFields are log's additional info type LogOptionalFields map[FieldType]interface{} -// FieldType is allowed/expected additional additionalInfo types +// FieldType is allowed/expected additional info types type FieldType int const ( @@ -220,7 +220,7 @@ func (e LogEntry) toFlatMap() map[string]interface{} { // order flatMap["timestamp"] = e.timestamp.Format(time.RFC3339Nano) - // NewLogEntry additional additionalInfo + // NewLogEntry additional info for k, v := range e.additionalInfo { flatMap[k.String()] = v } diff --git a/tasks/workflow/step.go b/tasks/workflow/step.go index ff2dd175d..21a7352f9 100644 --- a/tasks/workflow/step.go +++ b/tasks/workflow/step.go @@ -254,13 +254,6 @@ func (s *step) runActivity(wfCtx context.Context, kv *api.KV, cfg config.Configu return err } - //// Let's publish initial event status for workflow step (just before run it) - //for _, instanceName := range instances { - // eventInfo := &events.WorkflowStepInfo{WorkflowName: workflowName, NodeName: s.Target, StepName: s.Name, InstanceName: instanceName} - // events.PublishAndLogWorkflowStepStatusChange(wfCtx, kv, deploymentID, s.t.taskID, eventInfo, tasks.TaskStepStatusINITIAL.String()) - // events.PublishAndLogAlienTaskStatusChange(wfCtx, kv, deploymentID, s.t.taskID, s.t.id, eventInfo, tasks.TaskStepStatusINITIAL.String()) - //} - eventInfo := &events.WorkflowStepInfo{WorkflowName: workflowName, NodeName: s.Target, StepName: s.Name} switch activity.Type() { case builder.ActivityTypeDelegate: From 4f4f22f6aeff7a25fb26c63671e876ebb4f11263 Mon Sep 17 00:00:00 2001 From: Laurent Ganne <33217305+laurentganne@users.noreply.github.com> Date: Mon, 10 Dec 2018 14:10:20 +0100 Subject: [PATCH 12/19] Inverted node and operation in parameters --- commands/deployments/dep_events.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/commands/deployments/dep_events.go b/commands/deployments/dep_events.go index ef4a50475..f2c52d611 100644 --- a/commands/deployments/dep_events.go +++ b/commands/deployments/dep_events.go @@ -192,10 +192,10 @@ func formatEvent(event json.RawMessage, colorize bool) string { ret = fmt.Sprintf("%s:\t Deployment: %s\t Task %q (workflow)\t Workflow: %s\t Status: %s\n", ts, data[events.EDeploymentID.String()], data[events.ETaskID.String()], data[events.EWorkflowID.String()], data[events.EStatus.String()]) case events.StatusChangeTypeWorkflowStep: ret = fmt.Sprintf("%s:\t Deployment: %s\t Task %q (workflowStep)\t Workflow: %s\t Instance: %s\t Step: %s\t Node: %s\t Operation: %s%s\t Status: %s\n", ts, data[events.EDeploymentID.String()], - data[events.ETaskID.String()], data[events.EWorkflowID.String()], data[events.EInstanceID.String()], data[events.EWorkflowStepID.String()], data[events.EOperationName.String()], data[events.ENodeID.String()], formatOptionalInfo(data), data[events.EStatus.String()]) + data[events.ETaskID.String()], data[events.EWorkflowID.String()], data[events.EInstanceID.String()], data[events.EWorkflowStepID.String()], data[events.ENodeID.String()], data[events.EOperationName.String()], formatOptionalInfo(data), data[events.EStatus.String()]) case events.StatusChangeTypeAlienTask: ret = fmt.Sprintf("%s:\t Deployment: %s\t Task %q (Execution)\t Execution: %q\t Workflow: %s\t Instance: %s\t Step: %s\t Node: %s\t Operation: %s%s\t Status: %s\n", ts, data[events.EDeploymentID.String()], - data[events.ETaskID.String()], data[events.ETaskExecutionID.String()], data[events.EWorkflowID.String()], data[events.EInstanceID.String()], data[events.EWorkflowStepID.String()], data[events.EOperationName.String()], data[events.ENodeID.String()], formatOptionalInfo(data), data[events.EStatus.String()]) + data[events.ETaskID.String()], data[events.ETaskExecutionID.String()], data[events.EWorkflowID.String()], data[events.EInstanceID.String()], data[events.EWorkflowStepID.String()], data[events.ENodeID.String()], data[events.EOperationName.String()], formatOptionalInfo(data), data[events.EStatus.String()]) } return ret From ee94109e52633c840256983cec9af180c3867eb3 Mon Sep 17 00:00:00 2001 From: stephane benoist <29753229+stebenoist@users.noreply.github.com> Date: Mon, 10 Dec 2018 16:48:48 +0100 Subject: [PATCH 13/19] Set deployment status to "undeployment in progress" only if necessary --- tasks/workflow/worker.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tasks/workflow/worker.go b/tasks/workflow/worker.go index 2c9efe6cc..013eaba2b 100644 --- a/tasks/workflow/worker.go +++ b/tasks/workflow/worker.go @@ -600,7 +600,10 @@ func (w *worker) runUndeploy(ctx context.Context, t *taskExecution) { return } if status != deployments.UNDEPLOYED { - w.setDeploymentStatus(ctx, t.targetID, deployments.UNDEPLOYMENT_IN_PROGRESS) + if status != deployments.UNDEPLOYMENT_IN_PROGRESS { + w.setDeploymentStatus(ctx, t.targetID, deployments.UNDEPLOYMENT_IN_PROGRESS) + } + wfDone, err := w.runWorkflowStep(ctx, t, "uninstall", true) if err != nil { log.Printf("%+v", err) From f4b00911bc39c2d472168f0893a7beedd4cd9414 Mon Sep 17 00:00:00 2001 From: benoist-s Date: Tue, 11 Dec 2018 09:26:03 +0100 Subject: [PATCH 14/19] Add some comments related to task and execution semantic --- events/structs.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/events/structs.go b/events/structs.go index 64dddfdef..d01ab8ffb 100644 --- a/events/structs.go +++ b/events/structs.go @@ -84,6 +84,10 @@ func (i InfoType) String() string { return "type" case EWorkflowID: return "workflowId" + // Warning: A Yorc task is corresponding more or less to what Alien names an execution... + // ie a workflow execution + // But in Yorc semantic, task execution is referring to more or less what Alien names a task + // ie a workflow step execution case ETaskID: return "alienExecutionId" case ENodeID: From 7759db843ce3bae5a5a8d5f73dbfd1ccf61c49ba Mon Sep 17 00:00:00 2001 From: benoist-s Date: Tue, 11 Dec 2018 17:00:41 +0100 Subject: [PATCH 15/19] Add known issues section with BER issue --- doc/index.rst | 1 + doc/issues.rst | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+) create mode 100644 doc/issues.rst diff --git a/doc/index.rst b/doc/index.rst index 66e723f3f..897551eda 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -40,4 +40,5 @@ Yorc |release| Documentation telemetry performance upgrade + issues diff --git a/doc/issues.rst b/doc/issues.rst new file mode 100644 index 000000000..525433979 --- /dev/null +++ b/doc/issues.rst @@ -0,0 +1,35 @@ +.. + Copyright 2018 Bull S.A.S. Atos Technologies - Bull, Rue Jean Jaures, B.P.68, 78340, Les Clayes-sous-Bois, France. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + --- + +.. _yorc_issues_section: + +Known issues +============ + +.. _yorc_ber_issue_section: + +BER for SSH private key is not supported +---------------------------------------- + +Yorc uses SSH to connect to provisioned Computes or to hosts from hosts pool. + +Default behavior is to add related private keys to SSH-agent in order to handle authentication. + +But in some cases, SSH-agent can't be used and authentication must be done with private key file with the :ref:`--disable_ssh_agent ` command-line flag + +As we use Golang ssh package (https://godoc.org/golang.org/x/crypto/ssh) to parse the private key, we don't support +BER encoding format (https://github.com/golang/go/issues/14145). +This kind of format is especially used by OpenStack Liberty SSH keypair generator. From 19cbd1bd963ea9c7b0c8dbe5665e5a054a7f2265 Mon Sep 17 00:00:00 2001 From: stephane benoist <29753229+stebenoist@users.noreply.github.com> Date: Wed, 12 Dec 2018 10:57:19 +0100 Subject: [PATCH 16/19] Set unique taskExecutionID for each instance in AlienTaskEvent --- tasks/workflow/step.go | 39 +++++++++++++++++---------------------- 1 file changed, 17 insertions(+), 22 deletions(-) diff --git a/tasks/workflow/step.go b/tasks/workflow/step.go index 21a7352f9..41b682cce 100644 --- a/tasks/workflow/step.go +++ b/tasks/workflow/step.go @@ -268,13 +268,9 @@ func (s *step) runActivity(wfCtx context.Context, kv *api.KV, cfg config.Configu delegateOp := activity.Value() wfCtx = events.AddLogOptionalFields(wfCtx, events.LogOptionalFields{events.InterfaceName: "delegate", events.OperationName: delegateOp}) for _, instanceName := range instances { - eventInfo.InstanceName = instanceName eventInfo.OperationName = fmt.Sprintf("delegate.%s", delegateOp) // Need to publish INITIAL status before RUNNING one with operationName set - events.PublishAndLogWorkflowStepStatusChange(wfCtx, kv, deploymentID, s.t.taskID, eventInfo, tasks.TaskStepStatusINITIAL.String()) - events.PublishAndLogAlienTaskStatusChange(wfCtx, kv, deploymentID, s.t.taskID, s.t.id, eventInfo, tasks.TaskStepStatusINITIAL.String()) - events.PublishAndLogWorkflowStepStatusChange(wfCtx, kv, deploymentID, s.t.taskID, eventInfo, tasks.TaskStepStatusRUNNING.String()) - events.PublishAndLogAlienTaskStatusChange(wfCtx, kv, deploymentID, s.t.taskID, s.t.id, eventInfo, tasks.TaskStepStatusRUNNING.String()) + s.publishInstanceRelatedEvents(wfCtx, kv, deploymentID, instanceName, eventInfo, tasks.TaskStepStatusINITIAL, tasks.TaskStepStatusRUNNING) } err = func() error { defer metrics.MeasureSince(metricsutil.CleanupMetricKey([]string{"executor", "delegate", deploymentID, nodeType, delegateOp}), time.Now()) @@ -284,17 +280,13 @@ func (s *step) runActivity(wfCtx context.Context, kv *api.KV, cfg config.Configu if err != nil { metrics.IncrCounter(metricsutil.CleanupMetricKey([]string{"executor", "delegate", deploymentID, nodeType, delegateOp, "failures"}), 1) for _, instanceName := range instances { - eventInfo.InstanceName = instanceName - events.PublishAndLogWorkflowStepStatusChange(wfCtx, kv, deploymentID, s.t.taskID, eventInfo, tasks.TaskStepStatusERROR.String()) - events.PublishAndLogAlienTaskStatusChange(wfCtx, kv, deploymentID, s.t.taskID, s.t.id, eventInfo, tasks.TaskStepStatusERROR.String()) + s.publishInstanceRelatedEvents(wfCtx, kv, deploymentID, instanceName, eventInfo, tasks.TaskStepStatusERROR) } return err } metrics.IncrCounter(metricsutil.CleanupMetricKey([]string{"executor", "delegate", deploymentID, nodeType, delegateOp, "successes"}), 1) for _, instanceName := range instances { - eventInfo.InstanceName = instanceName - events.PublishAndLogWorkflowStepStatusChange(wfCtx, kv, deploymentID, s.t.taskID, eventInfo, tasks.TaskStepStatusDONE.String()) - events.PublishAndLogAlienTaskStatusChange(wfCtx, kv, deploymentID, s.t.taskID, s.t.id, eventInfo, tasks.TaskStepStatusDONE.String()) + s.publishInstanceRelatedEvents(wfCtx, kv, deploymentID, instanceName, eventInfo, tasks.TaskStepStatusDONE) } case builder.ActivityTypeSetState: setNodeStatus(wfCtx, kv, s.t.taskID, deploymentID, s.Target, activity.Value()) @@ -319,16 +311,13 @@ func (s *step) runActivity(wfCtx context.Context, kv *api.KV, cfg config.Configu } wfCtx = operations.SetOperationLogFields(wfCtx, op) for _, instanceName := range instances { + // Check for specific info about relationships eventInfo.OperationName = op.Name - eventInfo.InstanceName = instanceName if op.RelOp.IsRelationshipOperation { eventInfo.TargetNodeID = op.RelOp.TargetNodeName } // Need to publish INITIAL status before RUNNING one with operationName set - events.PublishAndLogWorkflowStepStatusChange(wfCtx, kv, deploymentID, s.t.taskID, eventInfo, tasks.TaskStepStatusINITIAL.String()) - events.PublishAndLogAlienTaskStatusChange(wfCtx, kv, deploymentID, s.t.taskID, s.t.id, eventInfo, tasks.TaskStepStatusINITIAL.String()) - events.PublishAndLogWorkflowStepStatusChange(wfCtx, kv, deploymentID, s.t.taskID, eventInfo, tasks.TaskStepStatusRUNNING.String()) - events.PublishAndLogAlienTaskStatusChange(wfCtx, kv, deploymentID, s.t.taskID, s.t.id, eventInfo, tasks.TaskStepStatusRUNNING.String()) + s.publishInstanceRelatedEvents(wfCtx, kv, deploymentID, instanceName, eventInfo, tasks.TaskStepStatusINITIAL, tasks.TaskStepStatusRUNNING) } // In function of the operation, the execution is sync or async if s.Async { @@ -362,18 +351,14 @@ func (s *step) runActivity(wfCtx context.Context, kv *api.KV, cfg config.Configu if err != nil { metrics.IncrCounter(metricsutil.CleanupMetricKey([]string{"executor", "operation", deploymentID, nodeType, op.Name, "failures"}), 1) for _, instanceName := range instances { - eventInfo.InstanceName = instanceName - events.PublishAndLogWorkflowStepStatusChange(wfCtx, kv, deploymentID, s.t.taskID, eventInfo, tasks.TaskStepStatusERROR.String()) - events.PublishAndLogAlienTaskStatusChange(wfCtx, kv, deploymentID, s.t.taskID, s.t.id, eventInfo, tasks.TaskStepStatusERROR.String()) + s.publishInstanceRelatedEvents(wfCtx, kv, deploymentID, instanceName, eventInfo, tasks.TaskStepStatusERROR) } return err } metrics.IncrCounter(metricsutil.CleanupMetricKey([]string{"executor", "operation", deploymentID, nodeType, op.Name, "successes"}), 1) if !s.Async { for _, instanceName := range instances { - eventInfo.InstanceName = instanceName - events.PublishAndLogWorkflowStepStatusChange(wfCtx, kv, deploymentID, s.t.taskID, eventInfo, tasks.TaskStepStatusDONE.String()) - events.PublishAndLogAlienTaskStatusChange(wfCtx, kv, deploymentID, s.t.taskID, s.t.id, eventInfo, tasks.TaskStepStatusDONE.String()) + s.publishInstanceRelatedEvents(wfCtx, kv, deploymentID, instanceName, eventInfo, tasks.TaskStepStatusDONE) } } case builder.ActivityTypeInline: @@ -382,3 +367,13 @@ func (s *step) runActivity(wfCtx context.Context, kv *api.KV, cfg config.Configu } return nil } + +func (s *step) publishInstanceRelatedEvents(ctx context.Context, kv *api.KV, deploymentID, instanceName string, eventInfo *events.WorkflowStepInfo, statuses ...tasks.TaskStepStatus) { + // taskExecutionID has to be unique for each instance, so we concat it to instanceName + eventInfo.InstanceName = instanceName + instanceTaskExecutionID := fmt.Sprintf("%s-%s", s.t.id, instanceName) + for _, status := range statuses { + events.PublishAndLogWorkflowStepStatusChange(ctx, kv, deploymentID, s.t.taskID, eventInfo, status.String()) + events.PublishAndLogAlienTaskStatusChange(ctx, kv, deploymentID, s.t.taskID, instanceTaskExecutionID, eventInfo, status.String()) + } +} From 485f5d920f2d097a7dcac73e7caaa5bf5827ad5e Mon Sep 17 00:00:00 2001 From: stephane benoist <29753229+stebenoist@users.noreply.github.com> Date: Wed, 12 Dec 2018 14:09:09 +0100 Subject: [PATCH 17/19] Add taskExecution log field and handle it for instances --- events/events.go | 2 +- events/log_entries.go | 16 ++++++++++++++-- prov/ansible/execution.go | 1 + tasks/workflow/step.go | 2 +- 4 files changed, 17 insertions(+), 4 deletions(-) diff --git a/events/events.go b/events/events.go index c49330553..9deddef0a 100644 --- a/events/events.go +++ b/events/events.go @@ -189,7 +189,7 @@ func PublishAndLogWorkflowStepStatusChange(ctx context.Context, kv *api.KV, depl // PublishAndLogAlienTaskStatusChange returns the published event id func PublishAndLogAlienTaskStatusChange(ctx context.Context, kv *api.KV, deploymentID, taskID, taskExecutionID string, wfStepInfo *WorkflowStepInfo, status string) (string, error) { if ctx == nil { - ctx = NewContext(context.Background(), LogOptionalFields{ExecutionID: taskID}) + ctx = NewContext(context.Background(), LogOptionalFields{ExecutionID: taskID, TaskExecutionID: taskExecutionID}) } info := make(Info) info[ETaskID] = taskID diff --git a/events/log_entries.go b/events/log_entries.go index 16b675426..1102686fa 100644 --- a/events/log_entries.go +++ b/events/log_entries.go @@ -69,6 +69,9 @@ const ( // TypeID is the field type representing the type ID in log entry TypeID + + // TaskExecutionID is the field type representing the task execution ID in log entry + TaskExecutionID ) // String allows to stringify the field type enumeration in JSON standard @@ -88,6 +91,8 @@ func (ft FieldType) String() string { return "operationName" case TypeID: return "type" + case TaskExecutionID: + return "alienTaskId" } return "" } @@ -153,6 +158,13 @@ func (e LogEntry) Register(content []byte) { // Get the timestamp e.timestamp = time.Now() + // Handle TaskExecutionID field which needs to be enrich with instanceID + inst, existInst := e.additionalInfo[InstanceID] + id, existID := e.additionalInfo[TaskExecutionID] + if existInst && existID { + e.additionalInfo[TaskExecutionID] = fmt.Sprintf("%s-%s", id, inst) + } + // Get the value to store and the flat log entry representation to log entry val, flat := e.generateValue() err := consulutil.StoreConsulKey(e.generateKey(), val) @@ -227,10 +239,10 @@ func (e LogEntry) toFlatMap() map[string]interface{} { return flatMap } -// FormatLog allows to format the flat map log representation in the following format :[Timestamp][Level][DeploymentID][WorkflowID][ExecutionID][NodeID][InstanceID][InterfaceName][OperationName][TypeID]Content +// FormatLog allows to format the flat map log representation in the following format :[Timestamp][Level][DeploymentID][WorkflowID][ExecutionID][TaskExecutionID][NodeID][InstanceID][InterfaceName][OperationName][TypeID]Content func FormatLog(flat map[string]interface{}) string { var str string - sliceOfKeys := []string{"timestamp", "level", "deploymentId", WorkFlowID.String(), ExecutionID.String(), NodeID.String(), InstanceID.String(), InterfaceName.String(), OperationName.String(), TypeID.String(), "content"} + sliceOfKeys := []string{"timestamp", "level", "deploymentId", WorkFlowID.String(), ExecutionID.String(), TaskExecutionID.String(), NodeID.String(), InstanceID.String(), InterfaceName.String(), OperationName.String(), TypeID.String(), "content"} for _, k := range sliceOfKeys { if val, ok := flat[k].(string); ok { if k != "content" { diff --git a/prov/ansible/execution.go b/prov/ansible/execution.go index b5804fd8d..d3d2467df 100644 --- a/prov/ansible/execution.go +++ b/prov/ansible/execution.go @@ -712,6 +712,7 @@ func (e *executionCommon) execute(ctx context.Context, retry bool) error { for _, instanceID := range instances { instanceName := operations.GetInstanceName(nodeName, instanceID) log.Debugf("Executing operation %q, on node %q, with current instance %q", e.operation.Name, e.NodeName, instanceName) + ctx = events.AddLogOptionalFields(ctx, events.LogOptionalFields{events.InstanceID: instanceID}) err := e.executeWithCurrentInstance(ctx, retry, instanceName) if err != nil { return err diff --git a/tasks/workflow/step.go b/tasks/workflow/step.go index 41b682cce..1f5f7c0b9 100644 --- a/tasks/workflow/step.go +++ b/tasks/workflow/step.go @@ -161,7 +161,7 @@ func (s *step) isRunnable() (bool, error) { // run allows to execute a workflow step func (s *step) run(ctx context.Context, cfg config.Configuration, kv *api.KV, deploymentID string, bypassErrors bool, workflowName string, w *worker) error { // Fill log optional fields for log registration - ctx = events.AddLogOptionalFields(ctx, events.LogOptionalFields{events.WorkFlowID: workflowName, events.NodeID: s.Target}) + ctx = events.AddLogOptionalFields(ctx, events.LogOptionalFields{events.WorkFlowID: workflowName, events.NodeID: s.Target, events.TaskExecutionID: s.t.id}) // First: we check if Step is runnable if runnable, err := s.isRunnable(); err != nil { return err From e2fdb586534d3fc81f27eb9e01821ddaed44fe2c Mon Sep 17 00:00:00 2001 From: benoist-s Date: Thu, 13 Dec 2018 09:02:04 +0100 Subject: [PATCH 18/19] Add event statusChange for customCommand and done status --- tasks/workflow/worker.go | 44 ++++++++++++++++------------------------ 1 file changed, 18 insertions(+), 26 deletions(-) diff --git a/tasks/workflow/worker.go b/tasks/workflow/worker.go index 013eaba2b..8e8b91ed4 100644 --- a/tasks/workflow/worker.go +++ b/tasks/workflow/worker.go @@ -299,7 +299,14 @@ func (w *worker) handleExecution(t *taskExecution) { case tasks.TaskTypeUnDeploy, tasks.TaskTypePurge: w.runUndeploy(ctx, t) case tasks.TaskTypeCustomCommand: - w.runCustomCommand(ctx, t) + err := w.runCustomCommand(ctx, t) + if err != nil { + log.Printf("Deployment id: %q, Task id: %q, Failed to get Custom command name: %+v", t.targetID, t.taskID, err) + checkAndSetTaskStatus(ctx, t.kv, t.taskID, t.step, tasks.TaskStatusFAILED) + events.PublishAndLogCustomCommandStatusChange(ctx, t.kv, t.targetID, t.taskID, tasks.TaskStatusFAILED.String()) + return + } + events.PublishAndLogCustomCommandStatusChange(ctx, t.kv, t.targetID, t.taskID, tasks.TaskStatusDONE.String()) case tasks.TaskTypeScaleOut: w.runScaleOut(ctx, t) case tasks.TaskTypeScaleIn: @@ -317,30 +324,22 @@ func (w *worker) handleExecution(t *taskExecution) { } } -func (w *worker) runCustomCommand(ctx context.Context, t *taskExecution) { +func (w *worker) runCustomCommand(ctx context.Context, t *taskExecution) error { kv := w.consulClient.KV() commandName, err := tasks.GetTaskData(kv, t.taskID, "commandName") if err != nil { - log.Printf("Deployment id: %q, Task id: %q, Failed to get Custom command name: %+v", t.targetID, t.taskID, err) - checkAndSetTaskStatus(ctx, t.kv, t.taskID, t.step, tasks.TaskStatusFAILED) - return + return err } interfaceNameKv, _, err := kv.Get(path.Join(consulutil.TasksPrefix, t.taskID, "interfaceName"), nil) if err != nil { - log.Printf("Deployment id: %q, Task id: %q, Failed to get Custom command name: %+v", t.targetID, t.taskID, err) - checkAndSetTaskStatus(ctx, t.kv, t.taskID, t.step, tasks.TaskStatusFAILED) - return + return err } nodes, err := tasks.GetTaskRelatedNodes(kv, t.taskID) if err != nil { - log.Printf("Deployment id: %q, Task id: %q, Failed to get Custom command node: %+v", t.targetID, t.taskID, err) - checkAndSetTaskStatus(ctx, t.kv, t.taskID, t.step, tasks.TaskStatusFAILED) - return + return err } if len(nodes) != 1 { - log.Printf("Deployment id: %q, Task id: %q, Expecting custom command TaskExecution to be related to \"1\" node while it is actually related to \"%d\" nodes", t.targetID, t.taskID, len(nodes)) - checkAndSetTaskStatus(ctx, t.kv, t.taskID, t.step, tasks.TaskStatusFAILED) - return + return errors.Errorf("Deployment id: %q, Task id: %q, Expecting custom command TaskExecution to be related to \"1\" node while it is actually related to \"%d\" nodes", t.targetID, t.taskID, len(nodes)) } nodeName := nodes[0] interfaceName := "custom" @@ -349,29 +348,23 @@ func (w *worker) runCustomCommand(ctx context.Context, t *taskExecution) { } nodeType, err := deployments.GetNodeType(w.consulClient.KV(), t.targetID, nodeName) if err != nil { - log.Printf("Deployment id: %q, Task id: %q, Failed to get Custom command node type: %+v", t.targetID, t.taskID, err) - checkAndSetTaskStatus(ctx, t.kv, t.taskID, t.step, tasks.TaskStatusFAILED) - return + return err } op, err := operations.GetOperation(ctx, kv, t.targetID, nodeName, interfaceName+"."+commandName, "", "") if err != nil { - log.Printf("Deployment id: %q, Task id: %q, Command TaskExecution failed for node %q: %+v", t.targetID, t.taskID, nodeName, err) err = setNodeStatus(ctx, t.kv, t.taskID, t.targetID, nodeName, tosca.NodeStateError.String()) if err != nil { log.Printf("Deployment id: %q, Task id: %q, Failed to set status for node %q: %+v", t.targetID, t.taskID, nodeName, err) } - checkAndSetTaskStatus(ctx, t.kv, t.taskID, t.step, tasks.TaskStatusFAILED) - return + return errors.Wrapf(err, "Command TaskExecution failed for node %q", nodeName) } exec, err := getOperationExecutor(kv, t.targetID, op.ImplementationArtifact) if err != nil { - log.Printf("Deployment id: %q, Task id: %q, Command TaskExecution failed for node %q: %+v", t.targetID, t.taskID, nodeName, err) err = setNodeStatus(ctx, t.kv, t.taskID, t.targetID, nodeName, tosca.NodeStateError.String()) if err != nil { log.Printf("Deployment id: %q, Task id: %q, Failed to set status for node %q: %+v", t.targetID, t.taskID, nodeName, err) } - checkAndSetTaskStatus(ctx, t.kv, t.taskID, t.step, tasks.TaskStatusFAILED) - return + return errors.Wrapf(err, "Command TaskExecution failed for node %q", nodeName) } ctx = operations.SetOperationLogFields(ctx, op) @@ -383,16 +376,15 @@ func (w *worker) runCustomCommand(ctx context.Context, t *taskExecution) { }() if err != nil { metrics.IncrCounter(metricsutil.CleanupMetricKey([]string{"executor", "operation", t.targetID, nodeType, op.Name, "failures"}), 1) - log.Printf("Deployment id: %q, Task id: %q, Command TaskExecution failed for node %q: %+v", t.targetID, t.taskID, nodeName, err) err = setNodeStatus(ctx, t.kv, t.taskID, t.targetID, nodeName, tosca.NodeStateError.String()) if err != nil { log.Printf("Deployment id: %q, Task id: %q, Failed to set status for node %q: %+v", t.targetID, t.taskID, nodeName, err) } - checkAndSetTaskStatus(ctx, t.kv, t.taskID, t.step, tasks.TaskStatusFAILED) - return + return errors.Wrapf(err, "Command TaskExecution failed for node %q", nodeName) } checkAndSetTaskStatus(ctx, t.kv, t.taskID, t.step, tasks.TaskStatusDONE) metrics.IncrCounter(metricsutil.CleanupMetricKey([]string{"executor", "operation", t.targetID, nodeType, op.Name, "successes"}), 1) + return nil } func (w *worker) endAction(ctx context.Context, action *prov.Action, actionErr error) { From 7cc2cb7230714fa2e5832781700a9db7fe939378 Mon Sep 17 00:00:00 2001 From: benoist-s Date: Thu, 13 Dec 2018 10:52:43 +0100 Subject: [PATCH 19/19] Update changelog --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e6fec3418..8e7a22da2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ ## UNRELEASED +### ENHANCEMENTS + +* Generate Alien 2.1-compatible events ([GH-148](https://github.com/ystia/yorc/issues/148)) + ## 3.1.0-M7 (December 07, 2018) ### DEPENDENCIES