Quick fix for deployment updates (#1975)

* Quick fix for deploymeny updates

* Fix test

* Move stringset function to pkg/stringset

Co-authored-by: Nick Angelou <[email protected]>

internal/log_analysis/datacatalog_updater/process/create_tables.go

@@ -32,6 +32,7 @@ import (
 	"github.com/panther-labs/panther/internal/log_analysis/awsglue"
 	"github.com/panther-labs/panther/internal/log_analysis/gluetables"
 	"github.com/panther-labs/panther/internal/log_analysis/log_processor/logtypes"
+	"github.com/panther-labs/panther/pkg/stringset"
 )
 
 // CreateTablesMessage is the event that triggers the creation of Glue tables/views for logtypes.
@@ -67,8 +68,23 @@ func (m CreateTablesMessage) Send(sqsClient sqsiface.SQSAPI, queueURL string) er
 }
 
 func HandleCreateTablesMessage(ctx context.Context, msg *CreateTablesMessage) error {
+	syncLogTypes := msg.LogTypes
+	// This is a quick fix for the sync issues
+	if msg.Sync {
+		// update the views with the new tables
+		availableLogTypes, err := listAvailableLogTypes(ctx)
+		if err != nil {
+			return err
+		}
+		deployedLogTypes, err := gluetables.DeployedLogTypes(ctx, glueClient, availableLogTypes)
+		if err != nil {
+			return err
+		}
+		syncLogTypes = stringset.Concat(msg.LogTypes, deployedLogTypes)
+	}
+
 	// create/update all tables associated with logTypes
-	for _, logType := range msg.LogTypes {
+	for _, logType := range syncLogTypes {
 		entry, err := logtypesResolver.Resolve(ctx, logType)
 		if err != nil {
 			return err
@@ -112,7 +128,7 @@ func HandleCreateTablesMessage(ctx context.Context, msg *CreateTablesMessage) er
 				awsglue.RuleMatchDatabaseName,
 				awsglue.RuleErrorsDatabaseName,
 			},
-			LogTypes: msg.LogTypes,
+			LogTypes: deployedLogTypes,
 			DryRun:   false,
 		})
 		if err != nil {

internal/log_analysis/datacatalog_updater/process/create_tables_test.go

@@ -100,7 +100,7 @@ func TestSQS_CreateTablesWithSync(t *testing.T) {
 	// Here comes the mocking
 	mockGlueClient.On("CreateTable", mock.Anything).Return(&glue.CreateTableOutput{}, nil)
 	// below called once for each database
-	mockGlueClient.On("GetTablesPagesWithContext", mock.Anything, mock.Anything, mock.Anything).Return(nil).Twice()
+	mockGlueClient.On("GetTablesPagesWithContext", mock.Anything, mock.Anything, mock.Anything).Return(nil).Times(4)
 	mockAthenaClient := &testutils.AthenaMock{}
 	athenaClient = mockAthenaClient
 	mockAthenaClient.On("StartQueryExecution", mock.Anything).Return(&athena.StartQueryExecutionOutput{

pkg/stringset/stringset.go

@@ -0,0 +1,65 @@
+package stringset
+
+/**
+ * Panther is a Cloud-Native SIEM for the Modern Security Team.
+ * Copyright (C) 2020 Panther Labs Inc
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+// New creates a new set of unique string values
+func New(values ...string) []string {
+	if values == nil {
+		return nil
+	}
+	return Append(make([]string, 0, len(values)), values...)
+}
+
+// Dedup de-duplicates a set of values in-place
+func Dedup(values []string) []string {
+	if values == nil {
+		return nil
+	}
+	return Append(values[:0], values...)
+}
+
+// Concat concatenates all parts into a single set of distinct values
+func Concat(parts ...[]string) []string {
+	// Collect the max required size
+	size := 0
+	for _, part := range parts {
+		size += len(part)
+	}
+	// Allocate a big enough slice for all values
+	union := make([]string, 0, size)
+	// Add all parts omitting duplicate values
+	for _, part := range parts {
+		union = Append(union, part...)
+	}
+	return union
+}
+
+// Append appends src to dst skipping duplicate values
+func Append(dst []string, values ...string) []string {
+loopValues:
+	for _, value := range values {
+		for _, duplicate := range dst {
+			if duplicate == value {
+				continue loopValues
+			}
+		}
+		dst = append(dst, value)
+	}
+	return dst
+}

pkg/stringset/stringset_test.go

@@ -0,0 +1,152 @@
+package stringset
+
+/**
+ * Panther is a Cloud-Native SIEM for the Modern Security Team.
+ * Copyright (C) 2020 Panther Labs Inc
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+import (
+	"strconv"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestConcat(t *testing.T) {
+	type testCase struct {
+		Args   [][]string
+		Expect []string
+	}
+	for i, tc := range []testCase{
+		{
+			Args: [][]string{
+				{"foo", "bar"},
+				{"foo", "baz"},
+			},
+			Expect: []string{"foo", "bar", "baz"},
+		},
+		{
+			Args: [][]string{
+				{"foo", "bar"},
+			},
+			Expect: []string{"foo", "bar"},
+		},
+		{
+			Args: [][]string{
+				{"foo", "foo"},
+			},
+			Expect: []string{"foo"},
+		},
+		{
+			Args: [][]string{
+				{},
+				nil,
+			},
+			Expect: []string{},
+		},
+		{
+			Args: [][]string{
+				{"foo", "bar"},
+				{"foo", "baz"},
+				{"bar", "baz", "qux"},
+			},
+			Expect: []string{"foo", "bar", "baz", "qux"},
+		},
+	} {
+		tc := tc
+		t.Run(strconv.Itoa(i), func(t *testing.T) {
+			actual := Concat(tc.Args...)
+			assert.Equal(t, tc.Expect, actual)
+		})
+	}
+}
+
+func TestNew(t *testing.T) {
+	type testCase struct {
+		Args   []string
+		Expect []string
+	}
+	for i, tc := range []testCase{
+		{
+			Args:   []string{"foo", "bar", "baz"},
+			Expect: []string{"foo", "bar", "baz"},
+		},
+		{
+			Args:   []string{"foo", "bar", "baz", "foo"},
+			Expect: []string{"foo", "bar", "baz"},
+		},
+		{
+			Args:   nil,
+			Expect: nil,
+		},
+		{
+			Args:   []string{},
+			Expect: []string{},
+		},
+	} {
+		tc := tc
+		t.Run(strconv.Itoa(i), func(t *testing.T) {
+			actual := New(tc.Args...)
+			assert.Equal(t, tc.Expect, actual)
+
+			if len(tc.Args) > 0 {
+				// Ensure the result is a new slice
+				for i := range tc.Args {
+					tc.Args[i] += "foo"
+				}
+				assert.NotEqual(t, actual, tc.Args)
+			}
+		})
+	}
+}
+
+func TestDedup(t *testing.T) {
+	type testCase struct {
+		Args   []string
+		Expect []string
+	}
+	for i, tc := range []testCase{
+		{
+			Args:   []string{"foo", "bar", "baz"},
+			Expect: []string{"foo", "bar", "baz"},
+		},
+		{
+			Args:   []string{"foo", "bar", "baz", "foo"},
+			Expect: []string{"foo", "bar", "baz"},
+		},
+		{
+			Args:   nil,
+			Expect: nil,
+		},
+		{
+			Args:   []string{},
+			Expect: []string{},
+		},
+	} {
+		tc := tc
+		t.Run(strconv.Itoa(i), func(t *testing.T) {
+			actual := Dedup(tc.Args)
+			assert.Equal(t, tc.Expect, actual)
+			if len(tc.Args) > 0 {
+				// Ensure the result is the same slice
+				for i := range tc.Args {
+					tc.Args[i] += "foo"
+				}
+				assert.Equal(t, actual, tc.Args[:len(actual)])
+			}
+		})
+	}
+}