Create codeql-analysis.yml (flyteorg#374)

* Scale out with propeller manager and workflow sharding (flyteorg#351) * added 'manager' command Signed-off-by: Daniel Rammer <[email protected]> * using go routine and timer for manager loop Signed-off-by: Daniel Rammer <[email protected]> * moved manager loop out of cmd and into pkg directory Signed-off-by: Daniel Rammer <[email protected]> * detecting missing replicas Signed-off-by: Daniel Rammer <[email protected]> * moved extracting replica from pod name to new function Signed-off-by: Daniel Rammer <[email protected]> * creating managed flytepropeller pods Signed-off-by: Daniel Rammer <[email protected]> * refactored configuration Signed-off-by: Daniel Rammer <[email protected]> * removed regex parsing for replica - checking for existance with fully qualified pod name Signed-off-by: Daniel Rammer <[email protected]> * mocked out shard strategy abstraction Signed-off-by: Daniel Rammer <[email protected]> * adding arguments to podspec for ConsistentHashingShardStrategy Signed-off-by: Daniel Rammer <[email protected]> * updated import naming Signed-off-by: Daniel Rammer <[email protected]> * moved manager to a top-level package Signed-off-by: Daniel Rammer <[email protected]> * added shard strategy to manager configuration Signed-off-by: Daniel Rammer <[email protected]> * setting shard key label selector on managed propeller instances Signed-off-by: Daniel Rammer <[email protected]> * fixed random lint issues Signed-off-by: Daniel Rammer <[email protected]> * split pod name generate to separate function to ease future auto-scaler implementation Signed-off-by: Daniel Rammer <[email protected]> * cleaned up pod label selector Signed-off-by: Daniel Rammer <[email protected]> * delete pods on shutdown Signed-off-by: Daniel Rammer <[email protected]> * added prometheus metric reporting Signed-off-by: Daniel Rammer <[email protected]> * updated manager run loop to use k8s wait.UntilWithContext Signed-off-by: Daniel Rammer <[email protected]> * moved getKubeConfig into a shared package Signed-off-by: Daniel Rammer <[email protected]> * assigning shard and namespace labels on FlyteWorkflow Signed-off-by: Daniel Rammer <[email protected]> * implement NamespaceShardStrategy Signed-off-by: Daniel Rammer <[email protected]> * implemented NamespaceShardStrategy Signed-off-by: Daniel Rammer <[email protected]> * fixed shard label Signed-off-by: Daniel Rammer <[email protected]> * added comments Signed-off-by: Daniel Rammer <[email protected]> * checking for existing pods on startup Signed-off-by: Daniel Rammer <[email protected]> * handling delete of non-existent pod Signed-off-by: Daniel Rammer <[email protected]> * changes ConsistentHashing name to Random - because that's what it really is Signed-off-by: Daniel Rammer <[email protected]> * implemented EnableUncoveredReplica configuration option Signed-off-by: Daniel Rammer <[email protected]> * added leader election to manager using existing propeller config Signed-off-by: Daniel Rammer <[email protected]> * fixed disable leader election in managed propeller pods Signed-off-by: Daniel Rammer <[email protected]> * removed listPods function Signed-off-by: Daniel Rammer <[email protected]> * added leader election to mitigate concurrent modification issues Signed-off-by: Daniel Rammer <[email protected]> * enabled pprof to profile resource metrics Signed-off-by: Daniel Rammer <[email protected]> * added 'manager' target to Makefile to start manager in development mode (similar to existing server) Signed-off-by: Daniel Rammer <[email protected]> * added shard strategy test for computing key ranges Signed-off-by: Daniel Rammer <[email protected]> * fixed key range computation Signed-off-by: Daniel Rammer <[email protected]> * implemented project and domain shard types Signed-off-by: Daniel Rammer <[email protected]> * returning error on out of range podIndex during UpdatePodSpec call on shard strategy Signed-off-by: Daniel Rammer <[email protected]> * fixed random lint issues Signed-off-by: Daniel Rammer <[email protected]> * added manager tests Signed-off-by: Daniel Rammer <[email protected]> * fixed lint issues Signed-off-by: Daniel Rammer <[email protected]> * added doc comments on exported types and functions Signed-off-by: Daniel Rammer <[email protected]> * exporting ComputeKeyRange function and changed adding addLabelSelector function name to addLabelSelectorIfExists to better reflect functionality Signed-off-by: Daniel Rammer <[email protected]> * adding pod template resource version and shard config hash annotations to fuel automatic pod management on updates Signed-off-by: Daniel Rammer <[email protected]> * removed pod deletion on manager shutdown Signed-off-by: Daniel Rammer <[email protected]> * cleaned up unit tests and lint Signed-off-by: Daniel Rammer <[email protected]> * updated getContainer function to retrive flytepropeller container from pod spec using container name instead of command Signed-off-by: Daniel Rammer <[email protected]> * removed addLabelSelectorIfExists function call Signed-off-by: Daniel Rammer <[email protected]> * changed bytes.Buffer from a var to declaring with new Signed-off-by: Daniel Rammer <[email protected]> * created a new shardstrategy package Signed-off-by: Daniel Rammer <[email protected]> * generating mocks for ShardStrategy to decouple manager package tests from shardstrategy package tests Signed-off-by: Daniel Rammer <[email protected]> * fixed lint issues Signed-off-by: Daniel Rammer <[email protected]> * changed shard configuration defintions and added support for wildcard id in EnvironmentShardStrategy Signed-off-by: Daniel Rammer <[email protected]> * updated documentation Signed-off-by: Daniel Rammer <[email protected]> * fixed lint issues Signed-off-by: Daniel Rammer <[email protected]> * setting managed pod owner references Signed-off-by: Daniel Rammer <[email protected]> * updated documentation Signed-off-by: Daniel Rammer <[email protected]> * fixed a few nits Signed-off-by: Daniel Rammer <[email protected]> * delete pods with failed state Signed-off-by: Daniel Rammer <[email protected]> * changed ShardType type to int instead of string Signed-off-by: Daniel Rammer <[email protected]> * removed default values in manager config Signed-off-by: Daniel Rammer <[email protected]> * updated config_flags with pflags generation Signed-off-by: Daniel Rammer <[email protected]> Signed-off-by: Haytham Abuelfutuh <[email protected]> * Create codeql-analysis.yml Signed-off-by: Haytham Abuelfutuh <[email protected]> * Handle code quality issue Signed-off-by: Haytham Abuelfutuh <[email protected]> * check boundaries Signed-off-by: Haytham Abuelfutuh <[email protected]> * 0 is ok Signed-off-by: Haytham Abuelfutuh <[email protected]> * Use ParseUint instead Signed-off-by: Haytham Abuelfutuh <[email protected]> * bump for DCO Signed-off-by: Haytham Abuelfutuh <[email protected]> Co-authored-by: Dan Rammer <[email protected]>
eapolinario · Jan 7, 2022 · 472621f · 472621f
1 parent b135183
commit 472621f
Show file tree

Hide file tree

Showing 4 changed files with 90 additions and 6 deletions.
diff --git a/flytepropeller/.github/workflows/codeql-analysis.yml b/flytepropeller/.github/workflows/codeql-analysis.yml
@@ -0,0 +1,70 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [ master ]
+  schedule:
+    - cron: '23 3 * * 6'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'go' ]
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
+        # Learn more about CodeQL language support at https://git.io/codeql-language-support
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v1
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+        # queries: ./path/to/local/query, your-org/your-repo/queries@main
+
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v1
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 https://git.io/JvXDl
+
+    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
+    #    and modify them (or add more) to build your code if your project
+    #    uses a compiled language
+
+    #- run: |
+    #   make bootstrap
+    #   make release
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v1
diff --git a/flytepropeller/pkg/controller/nodes/task/catalog/datacatalog/datacatalog.go b/flytepropeller/pkg/controller/nodes/task/catalog/datacatalog/datacatalog.go
@@ -3,6 +3,7 @@ package datacatalog
 import (
 	"context"
 	"crypto/x509"
+	"fmt"
 	"time"
 
 	"github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core"
@@ -124,7 +125,13 @@ func (m *CatalogClient) Get(ctx context.Context, key catalog.Key) (catalog.Entry
 		// TODO should we look through all the tags to find the relevant one?
 		relevantTag = artifact.GetTags()[0]
 	}
-	md := EventCatalogMetadata(dataset.GetId(), relevantTag, GetSourceFromMetadata(dataset.GetMetadata(), artifact.GetMetadata(), key.Identifier))
+
+	source, err := GetSourceFromMetadata(dataset.GetMetadata(), artifact.GetMetadata(), key.Identifier)
+	if err != nil {
+		return catalog.Entry{}, fmt.Errorf("failed to get source from metadata. Error: %w", err)
+	}
+
+	md := EventCatalogMetadata(dataset.GetId(), relevantTag, source)
 
 	outputs, err := GenerateTaskOutputsFromArtifact(key.Identifier, key.TypedInterface, artifact)
 	if err != nil {

diff --git a/flytepropeller/pkg/controller/nodes/task/catalog/datacatalog/transformer.go b/flytepropeller/pkg/controller/nodes/task/catalog/datacatalog/transformer.go
@@ -199,18 +199,24 @@ func GetArtifactMetadataForSource(taskExecutionID *core.TaskExecutionIdentifier)
 	}
 }
 
-// Returns the Source TaskExecutionIdentifier from the catalog metadata
+// GetSourceFromMetadata returns the Source TaskExecutionIdentifier from the catalog metadata
 // For all the information not available it returns Unknown. This is because as of July-2020 Catalog does not have all
 // the information. After the first deployment of this code, it will have this and the "unknown's" can be phased out
-func GetSourceFromMetadata(datasetMd, artifactMd *datacatalog.Metadata, currentID core.Identifier) *core.TaskExecutionIdentifier {
+func GetSourceFromMetadata(datasetMd, artifactMd *datacatalog.Metadata, currentID core.Identifier) (*core.TaskExecutionIdentifier, error) {
 	if datasetMd == nil || datasetMd.KeyMap == nil {
 		datasetMd = &datacatalog.Metadata{KeyMap: map[string]string{}}
 	}
 	if artifactMd == nil || artifactMd.KeyMap == nil {
 		artifactMd = &datacatalog.Metadata{KeyMap: map[string]string{}}
 	}
+
 	// Jul-06-2020 DataCatalog stores only wfExecutionKey & taskVersionKey So we will default the project / domain to the current dataset's project domain
-	attempt, _ := strconv.Atoi(GetOrDefault(artifactMd.KeyMap, execTaskAttemptKey, "0"))
+	val := GetOrDefault(artifactMd.KeyMap, execTaskAttemptKey, "0")
+	attempt, err := strconv.ParseUint(val, 10, 32)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse [%v] to integer. Error: %w", val, err)
+	}
+
 	return &core.TaskExecutionIdentifier{
 		TaskId: &core.Identifier{
 			ResourceType: currentID.ResourceType,
@@ -228,7 +234,7 @@ func GetSourceFromMetadata(datasetMd, artifactMd *datacatalog.Metadata, currentI
 				Name:    GetOrDefault(artifactMd.KeyMap, execNameKey, "unknown"),
 			},
 		},
-	}
+	}, nil
 }
 
 // Given the Catalog Information (returned from a Catalog call), returns the CatalogMetadata that is populated in the event.

diff --git a/flytepropeller/pkg/controller/nodes/task/catalog/datacatalog/transformer_test.go b/flytepropeller/pkg/controller/nodes/task/catalog/datacatalog/transformer_test.go
@@ -279,8 +279,9 @@ func TestGetSourceFromMetadata(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			if got := GetSourceFromMetadata(&datacatalog.Metadata{KeyMap: tt.args.datasetMd}, &datacatalog.Metadata{KeyMap: tt.args.artifactMd}, tt.args.currentID); !reflect.DeepEqual(got, tt.want) {
+			if got, err := GetSourceFromMetadata(&datacatalog.Metadata{KeyMap: tt.args.datasetMd}, &datacatalog.Metadata{KeyMap: tt.args.artifactMd}, tt.args.currentID); !reflect.DeepEqual(got, tt.want) {
 				t.Errorf("GetSourceFromMetadata() = %v, want %v", got, tt.want)
+				assert.NoError(t, err)
 			}
 		})
 	}