diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml new file mode 100644 index 000000000..fd6b11af7 --- /dev/null +++ b/.github/workflows/codeql-analysis.yml @@ -0,0 +1,70 @@ +# For most projects, this workflow file will not need changing; you simply need +# to commit it to your repository. +# +# You may wish to alter this file to override the set of languages analyzed, +# or to provide custom queries or build logic. +# +# ******** NOTE ******** +# We have attempted to detect the languages in your repository. Please check +# the `language` matrix defined below to confirm you have the correct set of +# supported CodeQL languages. +# +name: "CodeQL" + +on: + push: + branches: [ master ] + pull_request: + # The branches below must be a subset of the branches above + branches: [ master ] + schedule: + - cron: '23 3 * * 6' + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write + + strategy: + fail-fast: false + matrix: + language: [ 'go' ] + # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] + # Learn more about CodeQL language support at https://git.io/codeql-language-support + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v1 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + # queries: ./path/to/local/query, your-org/your-repo/queries@main + + # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@v1 + + # ℹī¸ Command-line programs to run using the OS shell. + # 📚 https://git.io/JvXDl + + # ✏ī¸ If the Autobuild fails above, remove it and uncomment the following three lines + # and modify them (or add more) to build your code if your project + # uses a compiled language + + #- run: | + # make bootstrap + # make release + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v1 diff --git a/pkg/controller/nodes/task/catalog/datacatalog/datacatalog.go b/pkg/controller/nodes/task/catalog/datacatalog/datacatalog.go index d91936553..f65dd33c9 100644 --- a/pkg/controller/nodes/task/catalog/datacatalog/datacatalog.go +++ b/pkg/controller/nodes/task/catalog/datacatalog/datacatalog.go @@ -3,6 +3,7 @@ package datacatalog import ( "context" "crypto/x509" + "fmt" "time" "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core" @@ -124,7 +125,13 @@ func (m *CatalogClient) Get(ctx context.Context, key catalog.Key) (catalog.Entry // TODO should we look through all the tags to find the relevant one? relevantTag = artifact.GetTags()[0] } - md := EventCatalogMetadata(dataset.GetId(), relevantTag, GetSourceFromMetadata(dataset.GetMetadata(), artifact.GetMetadata(), key.Identifier)) + + source, err := GetSourceFromMetadata(dataset.GetMetadata(), artifact.GetMetadata(), key.Identifier) + if err != nil { + return catalog.Entry{}, fmt.Errorf("failed to get source from metadata. Error: %w", err) + } + + md := EventCatalogMetadata(dataset.GetId(), relevantTag, source) outputs, err := GenerateTaskOutputsFromArtifact(key.Identifier, key.TypedInterface, artifact) if err != nil { diff --git a/pkg/controller/nodes/task/catalog/datacatalog/transformer.go b/pkg/controller/nodes/task/catalog/datacatalog/transformer.go index 655ffe4b0..e8dbcb476 100644 --- a/pkg/controller/nodes/task/catalog/datacatalog/transformer.go +++ b/pkg/controller/nodes/task/catalog/datacatalog/transformer.go @@ -199,18 +199,24 @@ func GetArtifactMetadataForSource(taskExecutionID *core.TaskExecutionIdentifier) } } -// Returns the Source TaskExecutionIdentifier from the catalog metadata +// GetSourceFromMetadata returns the Source TaskExecutionIdentifier from the catalog metadata // For all the information not available it returns Unknown. This is because as of July-2020 Catalog does not have all // the information. After the first deployment of this code, it will have this and the "unknown's" can be phased out -func GetSourceFromMetadata(datasetMd, artifactMd *datacatalog.Metadata, currentID core.Identifier) *core.TaskExecutionIdentifier { +func GetSourceFromMetadata(datasetMd, artifactMd *datacatalog.Metadata, currentID core.Identifier) (*core.TaskExecutionIdentifier, error) { if datasetMd == nil || datasetMd.KeyMap == nil { datasetMd = &datacatalog.Metadata{KeyMap: map[string]string{}} } if artifactMd == nil || artifactMd.KeyMap == nil { artifactMd = &datacatalog.Metadata{KeyMap: map[string]string{}} } + // Jul-06-2020 DataCatalog stores only wfExecutionKey & taskVersionKey So we will default the project / domain to the current dataset's project domain - attempt, _ := strconv.Atoi(GetOrDefault(artifactMd.KeyMap, execTaskAttemptKey, "0")) + val := GetOrDefault(artifactMd.KeyMap, execTaskAttemptKey, "0") + attempt, err := strconv.ParseUint(val, 10, 32) + if err != nil { + return nil, fmt.Errorf("failed to parse [%v] to integer. Error: %w", val, err) + } + return &core.TaskExecutionIdentifier{ TaskId: &core.Identifier{ ResourceType: currentID.ResourceType, @@ -228,7 +234,7 @@ func GetSourceFromMetadata(datasetMd, artifactMd *datacatalog.Metadata, currentI Name: GetOrDefault(artifactMd.KeyMap, execNameKey, "unknown"), }, }, - } + }, nil } // Given the Catalog Information (returned from a Catalog call), returns the CatalogMetadata that is populated in the event. diff --git a/pkg/controller/nodes/task/catalog/datacatalog/transformer_test.go b/pkg/controller/nodes/task/catalog/datacatalog/transformer_test.go index d4575874d..4d2485f27 100644 --- a/pkg/controller/nodes/task/catalog/datacatalog/transformer_test.go +++ b/pkg/controller/nodes/task/catalog/datacatalog/transformer_test.go @@ -279,8 +279,9 @@ func TestGetSourceFromMetadata(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got := GetSourceFromMetadata(&datacatalog.Metadata{KeyMap: tt.args.datasetMd}, &datacatalog.Metadata{KeyMap: tt.args.artifactMd}, tt.args.currentID); !reflect.DeepEqual(got, tt.want) { + if got, err := GetSourceFromMetadata(&datacatalog.Metadata{KeyMap: tt.args.datasetMd}, &datacatalog.Metadata{KeyMap: tt.args.artifactMd}, tt.args.currentID); !reflect.DeepEqual(got, tt.want) { t.Errorf("GetSourceFromMetadata() = %v, want %v", got, tt.want) + assert.NoError(t, err) } }) }