Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Exclude Polyspace defects with status "Not a Defect" or "Justified"; Add substitution of env variables for min/max possible for Robot and Polyspace #143

Merged
merged 24 commits into from
Jul 25, 2024
Merged
Changes from 22 commits
Commits
Show all changes
24 commits
Select commit Hold shift + click to select a range
2369e6f
Exclude defect when the status is "Not a defect" or "Justified"
JokeWaumans Jul 25, 2024
f3ddb51
Indent .format
JokeWaumans Jul 25, 2024
5668a5d
Fix column number
JokeWaumans Jul 25, 2024
e248258
Fix typo; col is used instead of column
JokeWaumans Jul 25, 2024
10b1b16
Change some status to "Justified" or "Not a defect"; 5 less defects i…
JokeWaumans Jul 25, 2024
f03cfb1
Change test code quality to use the right column numbers
JokeWaumans Jul 25, 2024
ada326c
Add explanation about the excluded statuses in readme
JokeWaumans Jul 25, 2024
c13c003
Delete default status "No Action Planned" from the excluded statuses
JokeWaumans Jul 25, 2024
0b4eb2a
Make substitution of environment variables possible for min and max o…
JokeWaumans Jul 25, 2024
deecdd1
Fix over-indented
JokeWaumans Jul 25, 2024
dc0ca29
Fix bullet list
JokeWaumans Jul 25, 2024
7b57dc7
Update README.rst typo correction
pietvr Jul 25, 2024
b86dbb6
Rephrase
JokeWaumans Jul 25, 2024
bd2ee46
Rephrase README
JokeWaumans Jul 25, 2024
6e8cb8d
Revert "Fix over-indented"
JasperCraeghs Jul 25, 2024
ff64d74
Revert "Make substitution of environment variables possible for min a…
JasperCraeghs Jul 25, 2024
59b1209
Revert "Revert "Make substitution of environment variables possible f…
JasperCraeghs Jul 25, 2024
4923b47
Revert implementation part of 0b4eb2a649dc84a2dd2b6dd2b6b61f2482ce8b48
JasperCraeghs Jul 25, 2024
364686b
Update test suite after 4923b47e5ea7ac5c9e54863dbdf8259d299b43d5
JasperCraeghs Jul 25, 2024
47c1f70
Improve readability of error msg
JasperCraeghs Jul 25, 2024
5b79619
Substitute envvars in WarningsChecker.substitute_envvar
JasperCraeghs Jul 25, 2024
feb0553
Merge branch 'filter-poly-defects' of https://github.com/melexis/warn…
JasperCraeghs Jul 25, 2024
1911694
Remove redundant newline
JasperCraeghs Jul 25, 2024
cd52002
Delete minimum and maximum of docstring
JokeWaumans Jul 25, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions README.rst
Original file line number Diff line number Diff line change
@@ -324,6 +324,17 @@ For example, "run-time check" is the family of Code Prover and "defect" is the f
The value of that key is a list, which contains the name of the column to check as a key and
the value of that column to check together with ``min`` and ``max`` values.

All results with one of the following statuses are discarded altogether:

- Justified
- Not a Defect
JasperCraeghs marked this conversation as resolved.
Show resolved Hide resolved

These statuses indicate that you have given due consideration and justified that result, as described
in `Polyspace's documentation about results`_.
The status "No Action Planned" is not treated differently because this is the default status for annotations.

.. _`Polyspace's documentation about results`: https://nl.mathworks.com/help/polyspace_access/ug/fix-or-comment-polyspace-results-web-browser.html

Example Checks
^^^^^^^^^^^^^^

28 changes: 16 additions & 12 deletions src/mlx/warnings/polyspace_checker.py
Original file line number Diff line number Diff line change
@@ -154,7 +154,7 @@ def parse_config(self, config):
"'Family' column. These dicts need to consist 3 key-value pairs (Note: if 'min' or "
"'max' is not defined, it will get the default value of 0):\n"
"{\n <column_name>: <value_to_check>,\n min: <number>,\n max: <number>\n};"
f"got {column_name} as column_name and {check_value} as value_to_check"
f"got {column_name!r} as column_name and {check_value!r} as value_to_check"
)
for checker in self.checkers:
checker.cq_enabled = self.cq_enabled
@@ -243,7 +243,7 @@ def add_code_quality_finding(self, row):
if "line" in row:
finding["location"]["positions"]["begin"]["line"] = row["line"]
if "col" in row:
finding["location"]["positions"]["begin"]["column"] = row["line"]
finding["location"]["positions"]["begin"]["column"] = row["col"]
finding["description"] = description
exclude = ("new", "status", "severity", "comment", "key")
row_without_key = [value for key, value in row.items() if key not in exclude]
@@ -258,13 +258,17 @@ def check(self, content):
content (dict): The row of the TSV file
'''
if content[self.column_name].lower() == self.check_value:
tab_sep_string = "\t".join(content.values())
if not self._is_excluded(tab_sep_string):
self.count = self.count + 1
self.counted_warnings.append('family: {} -> {}: {}'.format(
self.family_value,
self.column_name,
self.check_value
))
if self.cq_enabled and content["color"].lower() != "green":
self.add_code_quality_finding(content)
if content["status"].lower() in ["not a defect", "justified"]:
self.print_when_verbose("Excluded row {!r} because the status is 'Not a defect' or 'Justified'"
.format(content))
else:
tab_sep_string = "\t".join(content.values())
if not self._is_excluded(tab_sep_string):
self.count = self.count + 1
self.counted_warnings.append('family: {} -> {}: {}'.format(
self.family_value,
self.column_name,
self.check_value
))
if self.cq_enabled and content["color"].lower() != "green":
self.add_code_quality_finding(content)
22 changes: 0 additions & 22 deletions src/mlx/warnings/warnings.py
Original file line number Diff line number Diff line change
@@ -5,12 +5,10 @@
import errno
import glob
import json
import os
import subprocess
import sys
from importlib.metadata import distribution
from pathlib import Path
from string import Template

from ruamel.yaml import YAML

@@ -23,25 +21,6 @@
__version__ = distribution('mlx.warnings').version


def substitute_envvar(checker_config, keys):
"""Modifies configuration for checker in-place, resolving any environment variables for ``keys``
Args:
checker_config (dict): Configuration for a specific WarningsChecker
keys (set): Set of keys to process the value of
Raises:
WarningsConfigError: Failed to find an environment variable
"""
for key in keys:
if key in checker_config and isinstance(checker_config[key], str):
template_obj = Template(checker_config[key])
try:
checker_config[key] = template_obj.substitute(os.environ)
except KeyError as err:
raise WarningsConfigError(f"Failed to find environment variable {err} for configuration value {key!r}")\
from None


class WarningsPlugin:

@@ -229,7 +208,6 @@ def config_parser(self, config):
try:
checker_config = config[checker.name]
if bool(checker_config['enabled']):
substitute_envvar(checker_config, {'min', 'max'})
self.activate_checker(checker)
checker.parse_config(checker_config)
print("Config parsing for {name} completed".format(name=checker.name))
21 changes: 21 additions & 0 deletions src/mlx/warnings/warnings_checker.py
Original file line number Diff line number Diff line change
@@ -6,6 +6,26 @@
from .exceptions import WarningsConfigError


def substitute_envvar(checker_config, keys):
"""Modifies configuration for checker inplace, resolving any environment variables for ``keys``
Args:
checker_config (dict): Configuration for a specific WarningsChecker
keys (set): Set of keys to process the value of
Raises:
WarningsConfigError: Failed to find an environment variable
"""
for key in keys:
if key in checker_config and isinstance(checker_config[key], str):
template_obj = Template(checker_config[key])
try:
checker_config[key] = template_obj.substitute(os.environ)
except KeyError as err:
raise WarningsConfigError(f"Failed to find environment variable {err} for configuration value {key!r}")\
from None


class WarningsChecker:
name = 'checker'

@@ -156,6 +176,7 @@ def print_when_verbose(self, message):
print(message)

def parse_config(self, config):
substitute_envvar(config, {'min', 'max'})
self.maximum = int(config['max'])
self.minimum = int(config['min'])
self.add_patterns(config.get("exclude"), self.exclude_patterns)
4 changes: 2 additions & 2 deletions tests/test_in/config_example_polyspace_exclude.json
Original file line number Diff line number Diff line change
@@ -36,8 +36,8 @@
"run-time check": [
{
"color": "red",
"min": 0,
"max": 0
"min": "$MIN_POLY_WARNINGS",
"max": "${MAX_POLY_WARNINGS}"
},
{
"color": "orange",
4 changes: 2 additions & 2 deletions tests/test_in/config_example_robot.json
Original file line number Diff line number Diff line change
@@ -45,8 +45,8 @@
},
{
"name": "b4d su1te name",
"min": 0,
"max": 0
"min": "$MIN_ROBOT_WARNINGS",
"max": "${MAX_ROBOT_WARNINGS}"
}
]
},
12 changes: 6 additions & 6 deletions tests/test_in/polyspace.tsv
Original file line number Diff line number Diff line change
@@ -25,28 +25,28 @@ ID Family Group Color New Check Information Function File Status Severity Commen
22036 Run-time Check Numerical Green no Overflow dummy_function() dummy_file_name.c Unreviewed Unset C4992326D32B4698C491 6 4
22065 Run-time Check Data flow Green no Non-initialized local variable dummy_function() dummy_file_name.c Unreviewed Unset C49953E693A45698C489 5 5
22035 Run-time Check Data flow Green no Non-initialized local variable dummy_function() dummy_file_name.c Unreviewed Unset C499232693A45698C491 5 6
21989 Run-time Check Numerical Green no Overflow dummy_function() dummy_file_name.c Unreviewed Unset C4997327D32B4698C881 6 7
21989 Run-time Check Numerical Green no Overflow dummy_function() dummy_file_name.c Not a defect Unset C4997327D32B4698C881 6 7
19928 Run-time Check Numerical Orange no Overflow dummy_function() dummy_file_name.c Unreviewed Unset 62C8B1F4CA9126336A 6 8
19962 Run-time Check Numerical Orange no Overflow dummy_function() dummy_file_name.c Unreviewed Unset 62D8A9F4CA91263472 7 9
19962 Run-time Check Numerical Orange no Overflow dummy_function() dummy_file_name.c Not a defect Unset 62D8A9F4CA91263472 7 9
19526 Run-time Check Numerical Orange no Overflow dummy_function() dummy_file_name.c Unreviewed Unset 66C899F4CA9126316A 7 10
19424 Run-time Check Numerical Orange no Overflow dummy_function() dummy_file_name.h Unreviewed Unset 62E489F4CA91263262 6 11
19429 Run-time Check Numerical Orange no Overflow dummy_function() dummy_file_name.h Unreviewed Unset 62E4A1F4CA91263162 6 2
19429 Run-time Check Numerical Orange no Overflow dummy_function() dummy_file_name.h Justified Unset 62E4A1F4CA91263162 6 2
19442 Run-time Check Numerical Orange no Overflow dummy_function() dummy_file_name.h Unreviewed Unset 62C091F4CA91263170 7 3
19450 Run-time Check Numerical Orange no Overflow dummy_function() dummy_file_name.h Unreviewed Unset 62C091F4CA91263170 7 3
19450 Run-time Check Numerical Orange no Overflow dummy_function() dummy_file_name.h Justified Unset 62C091F4CA91263170 7 3
19375 Run-time Check Numerical Orange no Overflow dummy_function() dummy_file_name.c Unreviewed Unset 66C089F4CA9126326C 8 4
19378 Run-time Check Numerical Orange no Overflow dummy_function() dummy_file_name.c Unreviewed Unset 66C091F4CA9126316E 8 5
19377 Run-time Check Numerical Orange no Overflow dummy_function() dummy_file_name.c Unreviewed Unset 66C091F4CA9126336A 7 6
19357 Run-time Check Numerical Orange no Overflow dummy_function() dummy_file_name.c Unreviewed Unset 66C8A9F4CA91263368 7 7
19352 Run-time Check Numerical Orange no Overflow dummy_function() dummy_file_name.c Unreviewed Unset 66C8A9F4CA91263760 8 8
19351 Run-time Check Numerical Orange no Overflow dummy_function() dummy_file_name.c Unreviewed Unset 66C8A9F4CA91263862 8 9
19351 Run-time Check Numerical Orange no Overflow dummy_function() dummy_file_name.c Justified Unset 66C8A9F4CA91263862 8 9
19355 Run-time Check Numerical Orange no Overflow dummy_function() dummy_file_name.c Unreviewed Unset 66C8A9F4CA91263962 9 10
19354 Run-time Check Numerical Orange no Overflow dummy_function() dummy_file_name.c Unreviewed Unset CC9153E995234C62C091 9 11
19358 Run-time Check Numerical Orange no Overflow Origin: Path related issue dummy_function() dummy_file_name.c Unreviewed Unset CC9153E995234C62C0C1 8 3
19360 Run-time Check Numerical Orange no Overflow dummy_function() dummy_file_name.c Unreviewed Unset 66C8B1F4CA91263262 8 4
19349 Run-time Check Numerical Orange no Overflow dummy_function() dummy_file_name.c Unreviewed Unset 66C8C1F4CA91263262 9 5
19345 Run-time Check Numerical Orange no Overflow dummy_function() dummy_file_name.c Unreviewed Unset 66C8C1F4CA91263470 9 6
19344 Run-time Check Numerical Orange no Overflow dummy_function() dummy_file_name.c Unreviewed Unset 66C8C1F4CA91263572 10 7
19339 Run-time Check Numerical Orange no Overflow dummy_function() dummy_file_name.c Unreviewed Unset 66CC91F4CA91263464 10 8
19339 Run-time Check Numerical Orange no Overflow dummy_function() dummy_file_name.c Not a defect Unset 66CC91F4CA91263464 10 8
19338 Run-time Check Numerical Orange no Overflow dummy_function() dummy_file_name.c Unreviewed Unset 66CC91F4CA91263468 9 9
19336 Run-time Check Numerical Orange no Overflow dummy_function() dummy_file_name.c Unreviewed Unset 66CC91F4CA91263970 9 10
19335 Run-time Check Numerical Orange no Overflow dummy_function() dummy_file_name.c Unreviewed Unset CC9923E995234C62C0C9 10 11
212 changes: 71 additions & 141 deletions tests/test_in/polyspace_code_quality.json

Large diffs are not rendered by default.

116 changes: 44 additions & 72 deletions tests/test_in/polyspace_code_quality_exclude.json

Large diffs are not rendered by default.

10 changes: 10 additions & 0 deletions tests/test_integration.py
Original file line number Diff line number Diff line change
@@ -198,6 +198,8 @@ def test_robot_verbose(self):
)

def test_robot_config(self):
os.environ['MIN_ROBOT_WARNINGS'] = '0'
os.environ['MAX_ROBOT_WARNINGS'] = '0'
with patch('sys.stdout', new=StringIO()) as fake_out:
retval = warnings_wrapper([
'--config',
@@ -225,6 +227,9 @@ def test_robot_config(self):
stdout_log
)
self.assertEqual(2, retval)
for var in ('MIN_ROBOT_WARNINGS', 'MAX_ROBOT_WARNINGS'):
if var in os.environ:
del os.environ[var]

def test_robot_config_check_names(self):
self.maxDiff = None
@@ -281,6 +286,8 @@ def test_output_file_robot_basic(self):
self.assertTrue(filecmp.cmp(out_file, ref_file), '{} differs from {}'.format(out_file, ref_file))

def test_output_file_robot_config(self):
os.environ['MIN_ROBOT_WARNINGS'] = '0'
os.environ['MAX_ROBOT_WARNINGS'] = '0'
filename = 'robot_double_fail_config_summary.txt'
out_file = str(TEST_OUT_DIR / filename)
ref_file = str(TEST_IN_DIR / filename)
@@ -291,6 +298,9 @@ def test_output_file_robot_config(self):
])
self.assertEqual(2, retval)
self.assertTrue(filecmp.cmp(out_file, ref_file), '{} differs from {}'.format(out_file, ref_file))
for var in ('MIN_ROBOT_WARNINGS', 'MAX_ROBOT_WARNINGS'):
if var in os.environ:
del os.environ[var]

def test_output_file_junit(self):
filename = 'junit_double_fail_summary.txt'
14 changes: 7 additions & 7 deletions tests/test_polyspace.py
Original file line number Diff line number Diff line change
@@ -36,7 +36,7 @@ def test_code_prover_tsv_file(self):
stdout_log
)
self.assertEqual(count, count_sum)
self.assertEqual(count, 24)
self.assertEqual(count, 19)


class TestBugFinderWarnings(unittest.TestCase):
@@ -66,11 +66,11 @@ def test_bug_finder_tsv_file(self):

class TestPolyspaceWarnings(unittest.TestCase):
def setUp(self):
os.environ['MIN_SPHINX_WARNINGS'] = '0'
os.environ['MAX_SPHINX_WARNINGS'] = '0'
os.environ['MIN_POLY_WARNINGS'] = '0'
os.environ['MAX_POLY_WARNINGS'] = '0'

def tearDown(self):
for var in ('MIN_SPHINX_WARNINGS', 'MAX_SPHINX_WARNINGS'):
for var in ('MIN_POLY_WARNINGS', 'MAX_POLY_WARNINGS'):
if var in os.environ:
del os.environ[var]

@@ -79,7 +79,7 @@ def test_config_file(self):
'--config', str(TEST_IN_DIR / 'config_example_polyspace.yml'),
str(TEST_IN_DIR / 'polyspace.tsv')
])
self.assertEqual(66, retval)
self.assertEqual(61, retval)

def test_code_quality(self):
filename = 'polyspace_code_quality.json'
@@ -90,7 +90,7 @@ def test_code_quality(self):
'--config', str(TEST_IN_DIR / 'config_example_polyspace.yml'),
str(TEST_IN_DIR / 'polyspace.tsv'),
])
self.assertEqual(66, retval)
self.assertEqual(61, retval)
self.assertTrue(filecmp.cmp(out_file, ref_file))

def test_code_quality_no_green(self):
@@ -101,7 +101,7 @@ def test_code_quality_no_green(self):
'--config', str(TEST_IN_DIR / 'config_example_polyspace_green.yml'),
str(TEST_IN_DIR / 'polyspace.tsv'),
])
self.assertEqual(66, retval)
self.assertEqual(61, retval)
self.assertTrue(filecmp.cmp(out_file, ref_file))

def test_exclude_yaml_config(self):