fix: input file encoding

cyclonedx_py/client.py

@@ -25,6 +25,7 @@
 from datetime import datetime
 from typing import Any, Optional
 
+from chardet import detect as chardetect  # type:ignore[import]
 from cyclonedx.model import Tool
 from cyclonedx.model.bom import Bom
 from cyclonedx.output import BaseOutput, OutputFormat, SchemaVersion, get_instance as get_output_instance
@@ -270,11 +271,13 @@ def _get_input_parser(self) -> BaseParser:
                     raise CycloneDxCmdNoInputFileSupplied(
                         'When using input from Conda JSON, you need to pipe input via STDIN')
                 elif self._arguments.input_from_pip:
-                    self._arguments.input_source = open(os.path.join(current_directory, 'Pipfile.lock'), 'r')
+                    self._arguments.input_source = open(os.path.join(current_directory, 'Pipfile.lock'),
+                                                        'rt', encoding="UTF-8")
                 elif self._arguments.input_from_poetry:
-                    self._arguments.input_source = open(os.path.join(current_directory, 'poetry.lock'), 'r')
+                    self._arguments.input_source = open(os.path.join(current_directory, 'poetry.lock'),
+                                                        'rt', encoding="UTF-8")
                 elif self._arguments.input_from_requirements:
-                    self._arguments.input_source = open(os.path.join(current_directory, 'requirements.txt'), 'r')
+                    self._arguments.input_source = open(os.path.join(current_directory, 'requirements.txt'), 'rb')
                 else:
                     raise CycloneDxCmdException('Parser type could not be determined.')
             except FileNotFoundError as error:
@@ -285,6 +288,11 @@ def _get_input_parser(self) -> BaseParser:
         input_data_fh = self._arguments.input_source
         with input_data_fh:
             input_data = input_data_fh.read()
+            if isinstance(input_data, bytes):
+                input_encoding = chardetect(input_data)['encoding'].replace(
+                    # replace Windows-encoding with code-page
+                    'Windows-', 'cp')
+                input_data = input_data.decode(input_encoding)
             input_data_fh.close()
 
         if self._arguments.input_from_conda_explicit:

deps.lowest.r

@@ -5,6 +5,7 @@ cyclonedx-python-lib == 2.0.0
 packageurl-python == 0.9.0
 importlib-metadata == 3.4.0 # ; python_version < '3.8'
 pip-requirements-parser == 32.0.0
+chardet == 5.0.0
 setuptools == 47.0.0
 types-setuptools == 57.0.0
 toml == 0.10.0

pyproject.toml

@@ -50,6 +50,7 @@ importlib-metadata = { version = ">= 3.4", python = "< 3.8" }
 pip-requirements-parser = "^32.0.0"
 setuptools = ">= 47.0.0"
 toml = "^0.10.0"
+chardet = "^5.0"
 
 [tool.poetry.dev-dependencies]
 autopep8 = "^1.6.0"

tests/fixtures/.editorconfig

@@ -0,0 +1,4 @@
+
+[*.cp1252.txt.bin]
+charset = unset
+end_of_line = crlf

tests/fixtures/.gitattributes

@@ -0,0 +1,3 @@
+*.bin       binary
+*.txt.bin   binary diff=text
+

tests/fixtures/requirements-regression-issue448.cp1252.txt.bin

@@ -0,0 +1,8 @@
+# This is a regression test for https://github.com/CycloneDX/cyclonedx-python/issues/448
+# This file is in `cp1252` encoding - on purpose
+# This file uses CRLF for line endings - on purpose
+# some encoding specific characters: ���������������������������
+
+packageurl-python>=0.9.4
+requirements_parser>=0.2.0
+setuptools>=50.3.2