Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support schema error results to be output in JSON format including custom format flags #40

Merged
merged 28 commits into from
Jun 26, 2023
Merged
Show file tree
Hide file tree
Changes from 17 commits
Commits
Show all changes
28 commits
Select commit Hold shift + click to select a range
47bbdcc
Create a framework for validation error special case handling
mrutkows Jun 16, 2023
2688048
Create a framework for validation error special case handling
mrutkows Jun 16, 2023
8369472
Adjust JSON output formatting as an array
mrutkows Jun 19, 2023
bfa6985
Use an ordered map to control JSON output marshaling order
mrutkows Jun 19, 2023
7776c34
Use an ordered map to control JSON output marshaling order
mrutkows Jun 19, 2023
09bf70a
Use an ordered map to control JSON output marshaling order
mrutkows Jun 19, 2023
fd9cba3
Use an ordered map to control JSON output marshaling order
mrutkows Jun 19, 2023
bc89b66
Separate format related functions into their own file
mrutkows Jun 20, 2023
086f09d
Separate format related functions into their own file
mrutkows Jun 20, 2023
55810a6
Format value for unique item error
mrutkows Jun 21, 2023
54449f2
Consolidate validation flags and use on top-level API call
mrutkows Jun 21, 2023
e3bf5c4
Adjust JSON error result output prefix and indent
mrutkows Jun 21, 2023
7d23529
Add validation test case for bad iri-format
mrutkows Jun 21, 2023
38511c1
Add validation test case for bad iri-format
mrutkows Jun 21, 2023
292a82d
Consolidate persistent command flags into a struct
mrutkows Jun 21, 2023
4ab94f6
represent array type, index and item as a map in json error results
mrutkows Jun 21, 2023
5b57e38
Support flag true|false on validate command
mrutkows Jun 21, 2023
a630b7e
Fix even more Sonatype errors that seem to chnage every time I touch …
mrutkows Jun 21, 2023
16f8ce6
Adjust help for validate given new formats/flags
mrutkows Jun 22, 2023
ff1c278
Update README to show validate JSON output and new flags
mrutkows Jun 22, 2023
7258d9e
buffer JSON output for unit tests
mrutkows Jun 22, 2023
577e945
Update the text format logic to mirror new json formatting
mrutkows Jun 23, 2023
b02cebc
Update the text format logic to mirror new json formatting
mrutkows Jun 23, 2023
bf72268
Update the text format logic to mirror new json formatting
mrutkows Jun 23, 2023
ac503d5
Streamline json and text formatting paths
mrutkows Jun 23, 2023
d646e3e
Adjust colorized indent to match normal indent
mrutkows Jun 26, 2023
62deaed
Add additional test assertions to validate # errs and error conext
mrutkows Jun 26, 2023
64af463
Assure forced schema file tests reset to default schema
mrutkows Jun 26, 2023
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ jobs:
github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }}
goarch: ${{ matrix.goarch }}
extra_files: LICENSE config.json license.json custom.json ${{env.SBOM_NAME}}
extra_files: LICENSE README.md config.json license.json custom.json ${{env.SBOM_NAME}}
# "auto" will use ZIP for Windows, otherwise default is TAR
compress_assets: auto
# NOTE: This verbose flag may be removed
Expand Down
6 changes: 5 additions & 1 deletion .vscode/settings.json
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,7 @@
"multimap",
"myservices",
"NOASSERTION",
"nolint",
"nosec",
"NTIA",
"Nyffenegger",
Expand Down Expand Up @@ -102,5 +103,8 @@
],
"files.watcherExclude": {
"**/target": true
}
},
"cSpell.ignoreWords": [
"iancoleman"
]
}
41 changes: 20 additions & 21 deletions cmd/diff.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@ package cmd
import (
"encoding/json"
"fmt"
"io/ioutil"
"os"
"strings"

Expand Down Expand Up @@ -51,7 +50,7 @@ func NewCommandDiff() *cobra.Command {
command.Use = CMD_USAGE_DIFF
command.Short = "Report on differences between two BOM files using RFC 6902 format"
command.Long = "Report on differences between two BOM files using RFC 6902 format"
command.Flags().StringVarP(&utils.GlobalFlags.OutputFormat, FLAG_FILE_OUTPUT_FORMAT, "", FORMAT_TEXT,
command.Flags().StringVarP(&utils.GlobalFlags.PersistentFlags.OutputFormat, FLAG_FILE_OUTPUT_FORMAT, "", FORMAT_TEXT,
FLAG_DIFF_OUTPUT_FORMAT_HELP+DIFF_OUTPUT_SUPPORTED_FORMATS)
command.Flags().StringVarP(&utils.GlobalFlags.DiffFlags.RevisedFile,
FLAG_DIFF_FILENAME_REVISION,
Expand All @@ -75,7 +74,7 @@ func preRunTestForFiles(cmd *cobra.Command, args []string) error {
getLogger().Tracef("args: %v", args)

// Make sure the base (input) file is present and exists
baseFilename := utils.GlobalFlags.InputFile
baseFilename := utils.GlobalFlags.PersistentFlags.InputFile
if baseFilename == "" {
return getLogger().Errorf("Missing required argument(s): %s", FLAG_FILENAME_INPUT)
} else if _, err := os.Stat(baseFilename); err != nil {
Expand All @@ -98,7 +97,8 @@ func diffCmdImpl(cmd *cobra.Command, args []string) (err error) {
defer getLogger().Exit()

// Create output writer
outputFile, writer, err := createOutputFile(utils.GlobalFlags.OutputFile)
outputFilename := utils.GlobalFlags.PersistentFlags.OutputFile
outputFile, writer, err := createOutputFile(outputFilename)
getLogger().Tracef("outputFile: `%v`; writer: `%v`", outputFile, writer)

// use function closure to assure consistent error output based upon error type
Expand All @@ -109,7 +109,7 @@ func diffCmdImpl(cmd *cobra.Command, args []string) (err error) {
if err != nil {
return
}
getLogger().Infof("Closed output file: `%s`", utils.GlobalFlags.OutputFile)
getLogger().Infof("Closed output file: `%s`", utils.GlobalFlags.PersistentFlags.OutputFile)
}
}()

Expand All @@ -123,11 +123,11 @@ func Diff(flags utils.CommandFlags) (err error) {
defer getLogger().Exit()

// create locals
format := utils.GlobalFlags.OutputFormat
baseFilename := utils.GlobalFlags.InputFile
outputFilename := utils.GlobalFlags.OutputFile
outputFormat := utils.GlobalFlags.OutputFormat
deltaFilename := utils.GlobalFlags.DiffFlags.RevisedFile
format := utils.GlobalFlags.PersistentFlags.OutputFormat
inputFilename := utils.GlobalFlags.PersistentFlags.InputFile
outputFilename := utils.GlobalFlags.PersistentFlags.OutputFile
outputFormat := utils.GlobalFlags.PersistentFlags.OutputFormat
revisedFilename := utils.GlobalFlags.DiffFlags.RevisedFile
deltaColorize := utils.GlobalFlags.DiffFlags.Colorize

// Create output writer
Expand All @@ -138,31 +138,31 @@ func Diff(flags utils.CommandFlags) (err error) {
// always close the output file
if outputFile != nil {
err = outputFile.Close()
getLogger().Infof("Closed output file: `%s`", utils.GlobalFlags.OutputFile)
getLogger().Infof("Closed output file: `%s`", outputFilename)
}
}()

getLogger().Infof("Reading file (--input-file): `%s` ...", baseFilename)
getLogger().Infof("Reading file (--input-file): `%s` ...", inputFilename)
// #nosec G304 (suppress warning)
bBaseData, errReadBase := ioutil.ReadFile(baseFilename)
bBaseData, errReadBase := os.ReadFile(inputFilename)
if errReadBase != nil {
getLogger().Debugf("%v", bBaseData[:255])
err = getLogger().Errorf("Failed to ReadFile '%s': %s\n", utils.GlobalFlags.InputFile, err.Error())
err = getLogger().Errorf("Failed to ReadFile '%s': %s\n", inputFilename, err.Error())
return
}

getLogger().Infof("Reading file (--input-revision): `%s` ...", deltaFilename)
getLogger().Infof("Reading file (--input-revision): `%s` ...", revisedFilename)
// #nosec G304 (suppress warning)
bRevisedData, errReadDelta := ioutil.ReadFile(deltaFilename)
bRevisedData, errReadDelta := os.ReadFile(revisedFilename)
if errReadDelta != nil {
getLogger().Debugf("%v", bRevisedData[:255])
err = getLogger().Errorf("Failed to ReadFile '%s': %s\n", utils.GlobalFlags.InputFile, err.Error())
err = getLogger().Errorf("Failed to ReadFile '%s': %s\n", inputFilename, err.Error())
return
}

// Compare the base with the revision
differ := diff.New()
getLogger().Infof("Comparing files: `%s` (base) to `%s` (revised) ...", baseFilename, deltaFilename)
getLogger().Infof("Comparing files: `%s` (base) to `%s` (revised) ...", inputFilename, revisedFilename)
d, err := differ.Compare(bBaseData, bRevisedData)
if err != nil {
err = getLogger().Errorf("Failed to Compare data: %s\n", err.Error())
Expand All @@ -178,7 +178,7 @@ func Diff(flags utils.CommandFlags) (err error) {
err = json.Unmarshal(bBaseData, &aJson)

if err != nil {
err = getLogger().Errorf("json.Unmarshal() failed '%s': %s\n", utils.GlobalFlags.InputFile, err.Error())
err = getLogger().Errorf("json.Unmarshal() failed '%s': %s\n", inputFilename, err.Error())
return
}

Expand All @@ -201,8 +201,7 @@ func Diff(flags utils.CommandFlags) (err error) {

} else {
getLogger().Infof("No deltas found. baseFilename: `%s`, revisedFilename=`%s` match.",
utils.GlobalFlags.InputFile,
utils.GlobalFlags.DiffFlags.RevisedFile)
inputFilename, revisedFilename)
}

return
Expand Down
6 changes: 3 additions & 3 deletions cmd/diff_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -48,15 +48,15 @@ func innerDiffError(t *testing.T, baseFilename string, revisedFilename string, f
defer getLogger().Exit()

// Copy the test filename to the command line flags where the code looks for it
utils.GlobalFlags.OutputFormat = format
utils.GlobalFlags.InputFile = baseFilename
utils.GlobalFlags.PersistentFlags.OutputFormat = format
utils.GlobalFlags.PersistentFlags.InputFile = baseFilename
utils.GlobalFlags.DiffFlags.RevisedFile = revisedFilename
utils.GlobalFlags.DiffFlags.Colorize = true

actualError = Diff(utils.GlobalFlags)

getLogger().Tracef("baseFilename: `%s`, revisedFilename=`%s`, actualError=`%T`",
utils.GlobalFlags.InputFile,
utils.GlobalFlags.PersistentFlags.InputFile,
utils.GlobalFlags.DiffFlags.RevisedFile,
actualError)

Expand Down
16 changes: 9 additions & 7 deletions cmd/document.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,26 +28,28 @@ func LoadInputSbomFileAndDetectSchema() (document *schema.Sbom, err error) {
getLogger().Enter()
defer getLogger().Exit()

inputFile := utils.GlobalFlags.PersistentFlags.InputFile

// check for required fields on command
getLogger().Tracef("utils.Flags.InputFile: `%s`", utils.GlobalFlags.InputFile)
if utils.GlobalFlags.InputFile == "" {
return nil, fmt.Errorf("invalid input file (-%s): `%s` ", FLAG_FILENAME_INPUT_SHORT, utils.GlobalFlags.InputFile)
getLogger().Tracef("utils.Flags.InputFile: `%s`", inputFile)
if inputFile == "" {
return nil, fmt.Errorf("invalid input file (-%s): `%s` ", FLAG_FILENAME_INPUT_SHORT, inputFile)
}

// Construct an Sbom object around the input file
document = schema.NewSbom(utils.GlobalFlags.InputFile)
document = schema.NewSbom(inputFile)

// Load the raw, candidate SBOM (file) as JSON data
getLogger().Infof("Attempting to load and unmarshal file `%s`...", utils.GlobalFlags.InputFile)
getLogger().Infof("Attempting to load and unmarshal file `%s`...", inputFile)
err = document.UnmarshalSBOMAsJsonMap() // i.e., utils.Flags.InputFile
if err != nil {
return
}
getLogger().Infof("Successfully unmarshalled data from: `%s`", utils.GlobalFlags.InputFile)
getLogger().Infof("Successfully unmarshalled data from: `%s`", inputFile)

// Search the document keys/values for known SBOM formats and schema in the config. file
getLogger().Infof("Determining file's SBOM format and version...")
err = document.FindFormatAndSchema()
err = document.FindFormatAndSchema(utils.GlobalFlags.PersistentFlags.InputFile)
if err != nil {
return
}
Expand Down
3 changes: 2 additions & 1 deletion cmd/errors.go
Original file line number Diff line number Diff line change
Expand Up @@ -116,10 +116,11 @@ func (err BaseError) Error() string {
return formattedMessage
}

//nolint:all
func (base BaseError) AppendMessage(addendum string) {
// Ignore (invalid) static linting message:
// "ineffective assignment to field (SA4005)"
base.Message += addendum
base.Message += addendum //nolint:staticcheck
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

SA4005: ineffective assignment to field BaseError.Message


ℹ️ Expand to see all @sonatype-lift commands

You can reply with the following commands. For example, reply with @sonatype-lift ignoreall to leave out all findings.

Command Usage
@sonatype-lift ignore Leave out the above finding from this PR
@sonatype-lift ignoreall Leave out all the existing findings from this PR
@sonatype-lift exclude <file|issue|path|tool> Exclude specified file|issue|path|tool from Lift findings by updating your config.toml file

Note: When talking to LiftBot, you need to refresh the page to see its response.
Click here to add LiftBot to another repo.

}

type UtilityError struct {
Expand Down
11 changes: 6 additions & 5 deletions cmd/license_list.go
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ const (
MSG_OUTPUT_NO_LICENSES_ONLY_NOASSERTION = "no valid licenses found in BOM document (only licenses marked NOASSERTION)"
)

//"Type", "ID/Name/Expression", "Component(s)", "BOM ref.", "Document location"
// "Type", "ID/Name/Expression", "Component(s)", "BOM ref.", "Document location"
// filter keys
const (
LICENSE_FILTER_KEY_USAGE_POLICY = "usage-policy"
Expand Down Expand Up @@ -106,7 +106,7 @@ func NewCommandList() *cobra.Command {
command.Use = CMD_USAGE_LICENSE_LIST
command.Short = "List licenses found in the BOM input file"
command.Long = "List licenses and associated policies found in the BOM input file"
command.Flags().StringVarP(&utils.GlobalFlags.OutputFormat, FLAG_FILE_OUTPUT_FORMAT, "", "",
command.Flags().StringVarP(&utils.GlobalFlags.PersistentFlags.OutputFormat, FLAG_FILE_OUTPUT_FORMAT, "", "",
FLAG_LICENSE_LIST_OUTPUT_FORMAT_HELP+
LICENSE_LIST_SUPPORTED_FORMATS+
LICENSE_LIST_SUMMARY_SUPPORTED_FORMATS)
Expand Down Expand Up @@ -162,22 +162,23 @@ func listCmdImpl(cmd *cobra.Command, args []string) (err error) {
defer getLogger().Exit()

// Create output writer
outputFile, writer, err := createOutputFile(utils.GlobalFlags.OutputFile)
outputFilename := utils.GlobalFlags.PersistentFlags.OutputFile
outputFile, writer, err := createOutputFile(outputFilename)

// use function closure to assure consistent error output based upon error type
defer func() {
// always close the output file
if outputFile != nil {
err = outputFile.Close()
getLogger().Infof("Closed output file: `%s`", utils.GlobalFlags.OutputFile)
getLogger().Infof("Closed output file: `%s`", outputFilename)
}
}()

// process filters supplied on the --where command flag
whereFilters, err := processWhereFlag(cmd)

if err == nil {
err = ListLicenses(writer, utils.GlobalFlags.OutputFormat, whereFilters, utils.GlobalFlags.LicenseFlags.Summary)
err = ListLicenses(writer, utils.GlobalFlags.PersistentFlags.OutputFormat, whereFilters, utils.GlobalFlags.LicenseFlags.Summary)
}

return
Expand Down
10 changes: 5 additions & 5 deletions cmd/license_policy.go
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,7 @@ func NewCommandPolicy() *cobra.Command {
command.Use = CMD_USAGE_LICENSE_POLICY
command.Short = "List policies associated with known licenses"
command.Long = "List caller-supplied, \"allow/deny\"-style policies associated with known software, hardware or data licenses"
command.Flags().StringVarP(&utils.GlobalFlags.OutputFormat, FLAG_FILE_OUTPUT_FORMAT, "", FORMAT_TEXT,
command.Flags().StringVarP(&utils.GlobalFlags.PersistentFlags.OutputFormat, FLAG_FILE_OUTPUT_FORMAT, "", FORMAT_TEXT,
FLAG_POLICY_OUTPUT_FORMAT_HELP+LICENSE_POLICY_SUPPORTED_FORMATS)
command.Flags().BoolVarP(
&utils.GlobalFlags.LicenseFlags.Summary, // re-use license flag
Expand Down Expand Up @@ -161,14 +161,14 @@ func policyCmdImpl(cmd *cobra.Command, args []string) (err error) {
getLogger().Enter(args)
defer getLogger().Exit()

outputFile, writer, err := createOutputFile(utils.GlobalFlags.OutputFile)
outputFile, writer, err := createOutputFile(utils.GlobalFlags.PersistentFlags.OutputFile)

// use function closure to assure consistent error output based upon error type
defer func() {
// always close the output file
if outputFile != nil {
err = outputFile.Close()
getLogger().Infof("Closed output file: `%s`", utils.GlobalFlags.OutputFile)
getLogger().Infof("Closed output file: `%s`", utils.GlobalFlags.PersistentFlags.OutputFile)
}
}()

Expand Down Expand Up @@ -211,7 +211,7 @@ func ListLicensePolicies(writer io.Writer, whereFilters []WhereFilter, flags uti
}

// default output (writer) to standard out
switch utils.GlobalFlags.OutputFormat {
switch utils.GlobalFlags.PersistentFlags.OutputFormat {
case FORMAT_DEFAULT:
// defaults to text if no explicit `--format` parameter
err = DisplayLicensePoliciesTabbedText(writer, filteredMap, flags)
Expand All @@ -224,7 +224,7 @@ func ListLicensePolicies(writer io.Writer, whereFilters []WhereFilter, flags uti
default:
// default to text format for anything else
getLogger().Warningf("Unsupported format: `%s`; using default format.",
utils.GlobalFlags.OutputFormat)
utils.GlobalFlags.PersistentFlags.OutputFormat)
err = DisplayLicensePoliciesTabbedText(writer, filteredMap, flags)
}
return
Expand Down
16 changes: 9 additions & 7 deletions cmd/license_policy_config.go
Original file line number Diff line number Diff line change
Expand Up @@ -42,16 +42,18 @@ var VALID_USAGE_POLICIES = []string{POLICY_ALLOW, POLICY_DENY, POLICY_NEEDS_REVI
var ALL_USAGE_POLICIES = []string{POLICY_ALLOW, POLICY_DENY, POLICY_NEEDS_REVIEW, POLICY_UNDEFINED, POLICY_CONFLICT}

// Note: the SPDX spec. does not provide regex for an SPDX ID, but provides the following in ABNF:
// string = 1*(ALPHA / DIGIT / "-" / "." )
//
// string = 1*(ALPHA / DIGIT / "-" / "." )
//
// Currently, the regex below tests composition of of only
// alphanum, "-", and "." characters and disallows empty strings
// TODO:
// - First and last chars are not "-" or "."
// - Enforce reasonable min/max lengths
// In theory, we can check overall length with positive lookahead
// (e.g., min 3 max 128): (?=.{3,128}$)
// However, this does not appear to be supported in `regexp` package
// or perhaps it must be a compiled expression TBD
// - First and last chars are not "-" or "."
// - Enforce reasonable min/max lengths
// In theory, we can check overall length with positive lookahead
// (e.g., min 3 max 128): (?=.{3,128}$)
// However, this does not appear to be supported in `regexp` package
// or perhaps it must be a compiled expression TBD
const (
REGEX_VALID_SPDX_ID = "^[a-zA-Z0-9.-]+$"
)
Expand Down
8 changes: 4 additions & 4 deletions cmd/license_policy_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -81,8 +81,8 @@ func innerTestLicensePolicyListCustomAndBuffered(t *testing.T, testInfo *License
}

// Use the test data to set the BOM input file and output format
utils.GlobalFlags.InputFile = testInfo.InputFile
utils.GlobalFlags.OutputFormat = testInfo.ListFormat
utils.GlobalFlags.PersistentFlags.InputFile = testInfo.InputFile
utils.GlobalFlags.PersistentFlags.OutputFormat = testInfo.ListFormat
utils.GlobalFlags.LicenseFlags.Summary = testInfo.ListSummary

// TODO: pass GlobalConfig to every Command to allow per-instance changes for tests
Expand Down Expand Up @@ -118,9 +118,9 @@ func innerTestLicensePolicyList(t *testing.T, testInfo *LicenseTestInfo) (output
return
}

//-----------------------------------
// -----------------------------------
// Usage Policy: allowed value tests
//-----------------------------------
// -----------------------------------
func TestLicensePolicyUsageValueAllow(t *testing.T) {
value := POLICY_ALLOW
if !IsValidUsagePolicy(value) {
Expand Down
6 changes: 3 additions & 3 deletions cmd/license_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@ func innerTestLicenseListBuffered(t *testing.T, testInfo *LicenseTestInfo, where
defer outputWriter.Flush()

// Use a test input SBOM formatted in SPDX
utils.GlobalFlags.InputFile = testInfo.InputFile
utils.GlobalFlags.PersistentFlags.InputFile = testInfo.InputFile

// Invoke the actual List command (API)
err = ListLicenses(outputWriter, testInfo.ListFormat, whereFilters, testInfo.ListSummary)
Expand Down Expand Up @@ -288,9 +288,9 @@ func TestLicenseListPolicyCdx14InvalidLicenseName(t *testing.T) {
innerTestLicenseList(t, lti)
}

//---------------------------
// ---------------------------
// Where filter tests
//---------------------------
// ---------------------------
func TestLicenseListSummaryTextCdx13WhereUsageNeedsReview(t *testing.T) {
lti := NewLicenseTestInfoBasic(TEST_LICENSE_LIST_CDX_1_3, FORMAT_TEXT, true)
lti.WhereClause = "usage-policy=needs-review"
Expand Down
Loading