Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feat: omit development dependencies from SBOM results #534

Merged
merged 11 commits into from
Mar 20, 2023
3 changes: 3 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,9 @@ optional arguments:
Use with -i to specify absolute path to a
`requirements.txt` you wish to use, else we'll look
for one in the current working directory.
-omit OMIT, --omit OMIT
jkowalleck marked this conversation as resolved.
Show resolved Hide resolved
Omit specified items when using Poetry or PipEnv
(currently supported is dev)
-X Enable debug output

Input Method:
Expand Down
6 changes: 6 additions & 0 deletions cyclonedx_py/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -235,6 +235,10 @@ def get_arg_parser(*, prog: Optional[str] = None) -> argparse.ArgumentParser:
'-pb', '--purl-bom-ref', action='store_true', dest='use_purl_bom_ref',
help="Use a component's PURL for the bom-ref value, instead of a random UUID"
)
arg_parser.add_argument(
"--omit", dest="omit", action="append",
help="Omit specified items when using Poetry or PipEnv (currently supported is dev)",
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

❗ help text does not declare allowed values properly.
Let's mark this as an open topic and concentrate on the implementations first.

FYI: CLI and argparse might change to another implementation soon.
don't waste time now, we can discuss this at the feature finalization phase.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

My assumption was that currently supported is dev is enough to get started.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

lets leave this open.
if we go with Click8 then the help text generator will take over here.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

we will fix this during the CLI/click8/subcomamnd rework

)

arg_parser.add_argument('-X', action='store_true', help='Enable debug output', dest='debug_enabled')

Expand Down Expand Up @@ -302,12 +306,14 @@ def _get_input_parser(self) -> BaseParser:
elif self._arguments.input_from_pip:
return PipEnvParser(
pipenv_contents=input_data,
omit_category=self._arguments.omit,
use_purl_bom_ref=self._arguments.use_purl_bom_ref,
debug_message=lambda m, *a, **k: self._debug_message(f'PipEnvParser {m}', *a, **k)
)
elif self._arguments.input_from_poetry:
return PoetryParser(
poetry_lock_contents=input_data,
omit_category=self._arguments.omit,
use_purl_bom_ref=self._arguments.use_purl_bom_ref,
debug_message=lambda m, *a, **k: self._debug_message(f'PoetryParser {m}', *a, **k)
)
Expand Down
36 changes: 30 additions & 6 deletions cyclonedx_py/parser/pipenv.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,10 @@
# Copyright (c) OWASP Foundation. All Rights Reserved.

import json
from typing import Any, Dict
from enum import Enum, unique
from typing import Any, Dict, Optional, Set

from cyclonedx.model import ExternalReference, ExternalReferenceType, HashType, XsUri
from cyclonedx.model import ExternalReference, ExternalReferenceType, HashType, Property, XsUri
from cyclonedx.model.component import Component
from cyclonedx.parser import BaseParser

Expand All @@ -30,11 +31,19 @@
from ._debug import DebugMessageCallback, quiet


@unique
class OmitCategory(str, Enum):
"""Supported values for omit_category."""
DEV = "dev"


class PipEnvParser(BaseParser):

def __init__(
self, pipenv_contents: str, use_purl_bom_ref: bool = False,
self, pipenv_contents: str,
use_purl_bom_ref: bool = False,
*,
omit_category: Optional[Set[OmitCategory]],
debug_message: DebugMessageCallback = quiet
) -> None:
super().__init__()
Expand All @@ -43,14 +52,25 @@ def __init__(
debug_message('loading pipenv_contents')
pipfile_lock_contents = json.loads(pipenv_contents)
pipfile_default: Dict[str, Dict[str, Any]] = pipfile_lock_contents.get('default') or {}
self._process_items(pipfile_default, 'default', use_purl_bom_ref, debug_message=debug_message)

if not omit_category or (OmitCategory.DEV not in omit_category):
pipfile_develop: Dict[str, Dict[str, Any]] = pipfile_lock_contents.get('develop') or {}
self._process_items(pipfile_develop, 'develop', use_purl_bom_ref, debug_message=debug_message)

def _process_items(self, items: Dict[str, Dict[str, Any]], group: str,
use_purl_bom_ref: bool,
debug_message: DebugMessageCallback) -> None:
debug_message('processing pipfile_default')
for (package_name, package_data) in pipfile_default.items():
for (package_name, package_data) in items.items():
debug_message('processing package: {!r} {!r}', package_name, package_data)
version = str(package_data.get('version') or 'unknown').lstrip('=')
purl = PackageURL(type='pypi', name=package_name, version=version)
bom_ref = purl.to_string() if use_purl_bom_ref else None
c = Component(name=package_name, bom_ref=bom_ref, version=version, purl=purl)
c.properties.add(Property(
name='cdx:pipenv:package:category',
value=group))
if isinstance(package_data.get('hashes'), list):
# Add download location with hashes stored in Pipfile.lock
for pip_hash in package_data['hashes']:
Expand All @@ -68,13 +88,17 @@ def __init__(
class PipEnvFileParser(PipEnvParser):

def __init__(
self, pipenv_lock_filename: str, use_purl_bom_ref: bool = False,
self, pipenv_lock_filename: str,
use_purl_bom_ref: bool = False,
*,
omit_category: Optional[Set[OmitCategory]],
debug_message: DebugMessageCallback = quiet
) -> None:
debug_message('open file: {}', pipenv_lock_filename)
with open(pipenv_lock_filename) as plf:
super(PipEnvFileParser, self).__init__(
pipenv_contents=plf.read(), use_purl_bom_ref=use_purl_bom_ref,
pipenv_contents=plf.read(),
use_purl_bom_ref=use_purl_bom_ref,
omit_category=omit_category,
debug_message=debug_message
)
32 changes: 28 additions & 4 deletions cyclonedx_py/parser/poetry.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,11 @@
# SPDX-License-Identifier: Apache-2.0
# Copyright (c) OWASP Foundation. All Rights Reserved.

from enum import Enum, unique
from typing import Optional, Set

from cyclonedx.exception.model import CycloneDxModelException
from cyclonedx.model import ExternalReference, ExternalReferenceType, HashType, XsUri
from cyclonedx.model import ExternalReference, ExternalReferenceType, HashType, Property, XsUri
from cyclonedx.model.component import Component
from cyclonedx.parser import BaseParser

Expand All @@ -29,11 +32,19 @@
from ._debug import DebugMessageCallback, quiet


@unique
class OmitCategory(str, Enum):
madpah marked this conversation as resolved.
Show resolved Hide resolved
"""Supported values for omit_category."""
DEV = "dev"


class PoetryParser(BaseParser):

def __init__(
self, poetry_lock_contents: str, use_purl_bom_ref: bool = False,
self, poetry_lock_contents: str,
use_purl_bom_ref: bool = False,
*,
omit_category: Optional[Set[OmitCategory]],
debug_message: DebugMessageCallback = quiet
) -> None:
super().__init__()
Expand All @@ -52,12 +63,21 @@ def __init__(
debug_message('processing poetry_lock')
for package in poetry_lock['package']:
debug_message('processing package: {!r}', package)

if omit_category and (OmitCategory.DEV in omit_category) and package['category'] == OmitCategory.DEV:
if "dev" in omit_category:
jkowalleck marked this conversation as resolved.
Show resolved Hide resolved
debug_message("Ignoring development package!")
continue

purl = PackageURL(type='pypi', name=package['name'], version=package['version'])
bom_ref = purl.to_string() if use_purl_bom_ref else None
component = Component(
name=package['name'], bom_ref=bom_ref, version=package['version'],
purl=purl
)
component.properties.add(Property(
name='cdx:poetry:package:group',
value=package['category']))
debug_message('detecting package_files')
package_files = package['files'] \
if poetry_lock_version >= (2,) \
Expand All @@ -82,13 +102,17 @@ def __init__(
class PoetryFileParser(PoetryParser):

def __init__(
self, poetry_lock_filename: str, use_purl_bom_ref: bool = False,
self, poetry_lock_filename: str,
use_purl_bom_ref: bool = False,
*,
omit_category: Optional[Set[OmitCategory]],
debug_message: DebugMessageCallback = quiet
) -> None:
debug_message('open file: {}', poetry_lock_filename)
with open(poetry_lock_filename) as plf:
super(PoetryFileParser, self).__init__(
poetry_lock_contents=plf.read(), use_purl_bom_ref=use_purl_bom_ref,
poetry_lock_contents=plf.read(),
use_purl_bom_ref=use_purl_bom_ref,
omit_category=omit_category,
debug_message=debug_message
)
3 changes: 3 additions & 0 deletions docs/usage.rst
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,9 @@ The full documentation can be issued by running with ``--help``:
Use with -i to specify absolute path to a
`requirements.txt` you wish to use, else we'll look
for one in the current working directory.
--omit OMIT
Omit specified items when using Poetry or PipEnv
(currently supported is dev)
-X Enable debug output

Input Method:
Expand Down
38 changes: 38 additions & 0 deletions tests/fixtures/pipfile-lock-dev-dep.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
{
"_meta": {
"hash": {
"sha256": "8ca3da46acf801a7780c6781bed1d6b7012664226203447640cda114b13aa8aa"
},
"pipfile-spec": 6,
"requires": {
"python_version": "3.9"
},
"sources": [
{
"name": "pypi",
"url": "https://pypi.org/simple",
"verify_ssl": true
}
]
},
"default": {
"toml": {
"hashes": [
"sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b",
"sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"
],
"index": "pypi",
"version": "==0.10.2"
}
},
"develop": {
"flake8": {
"hashes": [
"sha256:45681a117ecc81e870cbf1262835ae4af5e7a8b08e40b944a8a6e6b895914cfb",
"sha256:49356e766643ad15072a789a20915d3c91dc89fd313ccd71802303fd67e4deca"
],
"index": "pip_conf_index_global",
"version": "==3.7.9"
}
}
}
30 changes: 30 additions & 0 deletions tests/fixtures/poetry-dev-dep.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
# This file is automatically @generated by Poetry and should not be changed by hand.
jkowalleck marked this conversation as resolved.
Show resolved Hide resolved

[[package]]
name = "toml"
version = "0.10.2"
description = "Python Library for Tom's Obvious, Minimal Language"
category = "main"
optional = false
python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*"
files = [
{file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"},
{file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"},
]

[[package]]
name = "ddt"
version = "1.6.0"
description = "Data-Driven/Decorated Tests"
category = "dev"
optional = false
python-versions = "*"
files = [
{file = "ddt-1.6.0-py2.py3-none-any.whl", hash = "sha256:e3c93b961a108b4f4d5a6c7f2263513d928baf3bb5b32af8e1c804bfb041141d"},
{file = "ddt-1.6.0.tar.gz", hash = "sha256:f71b348731b8c78c3100bffbd951a769fbd439088d1fdbb3841eee019af80acd"},
]

[metadata]
lock-version = "2.0"
python-versions = "^3.9"
content-hash = "b97d7a3bc03286e93fd688187cbdcd469fe0e5108cdc7936c432995f983f478c"
52 changes: 48 additions & 4 deletions tests/test_parser_pipenv.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,15 +20,17 @@
import os
from unittest import TestCase

from cyclonedx_py.parser.pipenv import PipEnvFileParser
from cyclonedx_py.parser.pipenv import OmitCategory, PipEnvFileParser


class TestPipEnvParser(TestCase):

def test_simple(self) -> None:
tests_pipfile_lock = os.path.join(os.path.dirname(__file__), 'fixtures/pipfile-lock-simple.txt')

parser = PipEnvFileParser(pipenv_lock_filename=tests_pipfile_lock)
parser = PipEnvFileParser(
pipenv_lock_filename=tests_pipfile_lock,
omit_category=set())
self.assertEqual(1, parser.component_count())
c_toml = next(filter(lambda c: c.name == 'toml', parser.get_components()), None)
self.assertIsNotNone(c_toml)
Expand All @@ -41,7 +43,10 @@ def test_simple(self) -> None:
def test_simple_use_purl_bom_ref(self) -> None:
tests_pipfile_lock = os.path.join(os.path.dirname(__file__), 'fixtures/pipfile-lock-simple.txt')

parser = PipEnvFileParser(pipenv_lock_filename=tests_pipfile_lock, use_purl_bom_ref=True)
parser = PipEnvFileParser(
pipenv_lock_filename=tests_pipfile_lock,
omit_category=set(),
use_purl_bom_ref=True)
self.assertEqual(1, parser.component_count())
c_toml = next(filter(lambda c: c.name == 'toml', parser.get_components()), None)
self.assertIsNotNone(c_toml)
Expand All @@ -54,7 +59,9 @@ def test_simple_use_purl_bom_ref(self) -> None:
def test_with_multiple_and_no_index(self) -> None:
tests_pipfile_lock = os.path.join(os.path.dirname(__file__), 'fixtures/pipfile-lock-no-index-example.txt')

parser = PipEnvFileParser(pipenv_lock_filename=tests_pipfile_lock)
parser = PipEnvFileParser(
pipenv_lock_filename=tests_pipfile_lock,
omit_category=set())
self.assertEqual(2, parser.component_count())

c_anyio = next(filter(lambda c: c.name == 'anyio', parser.get_components()), None)
Expand All @@ -71,3 +78,40 @@ def test_with_multiple_and_no_index(self) -> None:
self.assertEqual('0.10.2', c_toml.version)
self.assertEqual(2, len(c_toml.external_references), f'{c_toml.external_references}')
self.assertEqual(1, len(c_toml.external_references.pop().hashes))

def test_simple_ignore_dev_dep(self) -> None:
tests_pipfile_lock = os.path.join(os.path.dirname(__file__), 'fixtures/pipfile-lock-dev-dep.txt')

parser = PipEnvFileParser(
pipenv_lock_filename=tests_pipfile_lock,
omit_category=set(),
use_purl_bom_ref=True)
self.assertEqual(2, parser.component_count())
c_toml = next(filter(lambda c: c.name == 'toml', parser.get_components()), None)
self.assertIsNotNone(c_toml)
self.assertEqual('toml', c_toml.name)
self.assertEqual(c_toml.purl.to_string(), c_toml.bom_ref.value)
self.assertEqual('0.10.2', c_toml.version)
self.assertEqual(2, len(c_toml.external_references), f'{c_toml.external_references}')
self.assertEqual(1, len(c_toml.external_references.pop().hashes))

c_flake8 = next(filter(lambda c: c.name == 'flake8', parser.get_components()), None)
self.assertIsNotNone(c_flake8)
self.assertEqual('flake8', c_flake8.name)
self.assertEqual(c_flake8.purl.to_string(), c_flake8.bom_ref.value)
self.assertEqual('3.7.9', c_flake8.version)
self.assertEqual(2, len(c_flake8.external_references), f'{c_flake8.external_references}')
self.assertEqual(1, len(c_flake8.external_references.pop().hashes))

parser = PipEnvFileParser(
pipenv_lock_filename=tests_pipfile_lock,
omit_category={OmitCategory.DEV},
use_purl_bom_ref=True)
self.assertEqual(1, parser.component_count())
c_toml = next(filter(lambda c: c.name == 'toml', parser.get_components()), None)
self.assertIsNotNone(c_toml)
self.assertEqual('toml', c_toml.name)
self.assertEqual(c_toml.purl.to_string(), c_toml.bom_ref.value)
self.assertEqual('0.10.2', c_toml.version)
self.assertEqual(2, len(c_toml.external_references), f'{c_toml.external_references}')
self.assertEqual(1, len(c_toml.external_references.pop().hashes))
Loading