diff --git a/CHANGES.md b/CHANGES.md index 1c313b6c..21435758 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -6,7 +6,10 @@ The released versions correspond to PyPi releases. #### New Features * add support for the `buffering` parameter in `open` (see [#549](../../issues/549)) - + * add possibility to patch `io.open_code` using the new argument + `patch_open_code` (since Python 3.8) + (see [#554](../../issues/554)) + #### Fixes * do not call fake `open` if called from skipped module (see [#552](../../issues/552)) diff --git a/docs/usage.rst b/docs/usage.rst index 1383d8e2..1341fdd7 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -386,6 +386,22 @@ the default value of ``use_known_patches`` should be used, but it is present to allow users to disable this patching in case it causes any problems. It may be removed or replaced by more fine-grained arguments in future releases. +patch_open_code +~~~~~~~~~~~~~~~ +Since Python 3.8, the ``io`` module has the function ``open_code``, which +opens a file read-only and is used to open Python code files. By default, this +function is not patched, because the files it opens usually belong to the +executed library code and are not present in the fake file system. +Under some circumstances, this may not be the case, and the opened file +lives in the fake filesystem. For these cases, you can set ``patch_open_code`` +to ``True``. + +.. note:: There is no possibility to change this setting based on affected + files. Depending on the upcoming use cases, this may be changed in future + versions of ``pyfakefs``, and this argument may be changed or removed in a + later version. + + Using convenience methods ------------------------- While ``pyfakefs`` can be used just with the standard Python file system diff --git a/pyfakefs/fake_filesystem.py b/pyfakefs/fake_filesystem.py index 40ec71a5..873f37dc 100644 --- a/pyfakefs/fake_filesystem.py +++ b/pyfakefs/fake_filesystem.py @@ -888,6 +888,8 @@ def __init__(self, path_separator=os.path.sep, total_size=None, self.add_mount_point(self.root.name, total_size) self._add_standard_streams() self.dev_null = FakeNullFile(self) + # set from outside if needed + self.patch_open_code = False @property def is_linux(self): @@ -3438,7 +3440,7 @@ def dir(): """Return the list of patched function names. Used for patching functions imported from the module. """ - dir = [ + _dir = [ 'access', 'chdir', 'chmod', 'chown', 'close', 'fstat', 'fsync', 'getcwd', 'lchmod', 'link', 'listdir', 'lstat', 'makedirs', 'mkdir', 'mknod', 'open', 'read', 'readlink', 'remove', @@ -3446,13 +3448,13 @@ def dir(): 'unlink', 'utime', 'walk', 'write', 'getcwdb', 'replace' ] if sys.platform.startswith('linux'): - dir += [ + _dir += [ 'fdatasync', 'getxattr', 'listxattr', 'removexattr', 'setxattr' ] if use_scandir: - dir += ['scandir'] - return dir + _dir += ['scandir'] + return _dir def __init__(self, filesystem): """Also exposes self.path (to fake os.path). @@ -4468,7 +4470,10 @@ def dir(): """Return the list of patched function names. Used for patching functions imported from the module. """ - return 'open', + _dir = ['open'] + if sys.version_info >= (3, 8): + _dir.append('open_code') + return _dir def __init__(self, filesystem): """ @@ -4498,6 +4503,21 @@ def open(self, file, mode='r', buffering=-1, encoding=None, return fake_open(file, mode, buffering, encoding, errors, newline, closefd, opener) + if sys.version_info >= (3, 8): + def open_code(self, path): + """Redirect the call to open. Note that the behavior of the real + function may be overridden by an earlier call to the + PyFile_SetOpenCodeHook(). This behavior is not reproduced here. + """ + if not isinstance(path, str): + raise TypeError( + "open_code() argument 'path' must be str, not int") + if not self.filesystem.patch_open_code: + # mostly this is used for compiled code - + # don't patch these, as the files are probably in the real fs + return self._io_module.open_code(path) + return self.open(path, mode='rb') + def __getattr__(self, name): """Forwards any unfaked calls to the standard io module.""" return getattr(self._io_module, name) diff --git a/pyfakefs/fake_filesystem_unittest.py b/pyfakefs/fake_filesystem_unittest.py index 398c94cb..3d210566 100644 --- a/pyfakefs/fake_filesystem_unittest.py +++ b/pyfakefs/fake_filesystem_unittest.py @@ -78,7 +78,8 @@ def patchfs(_func=None, *, modules_to_reload=None, modules_to_patch=None, allow_root_user=True, - use_known_patches=True): + use_known_patches=True, + patch_open_code=False): """Convenience decorator to use patcher with additional parameters in a test function. @@ -101,7 +102,8 @@ def wrapped(*args, **kwargs): modules_to_reload=modules_to_reload, modules_to_patch=modules_to_patch, allow_root_user=allow_root_user, - use_known_patches=use_known_patches) as p: + use_known_patches=use_known_patches, + patch_open_code=patch_open_code) as p: kwargs['fs'] = p.fs return f(*args, **kwargs) @@ -123,7 +125,8 @@ def load_doctests(loader, tests, ignore, module, modules_to_reload=None, modules_to_patch=None, allow_root_user=True, - use_known_patches=True): # pylint: disable=unused-argument + use_known_patches=True, + patch_open_code=False): # pylint: disable=unused-argument """Load the doctest tests for the specified module into unittest. Args: loader, tests, ignore : arguments passed in from `load_tests()` @@ -136,7 +139,8 @@ def load_doctests(loader, tests, ignore, module, modules_to_reload=modules_to_reload, modules_to_patch=modules_to_patch, allow_root_user=allow_root_user, - use_known_patches=use_known_patches) + use_known_patches=use_known_patches, + patch_open_code=patch_open_code) globs = _patcher.replace_globs(vars(module)) tests.addTests(doctest.DocTestSuite(module, globs=globs, @@ -162,8 +166,6 @@ class TestCaseMixin: modules_to_patch: A dictionary of fake modules mapped to the fully qualified patched module names. Can be used to add patching of modules not provided by `pyfakefs`. - use_known_patches: If True (the default), some patches for commonly - used packges are applied which make them usable with pyfakes. If you specify some of these attributes here and you have DocTests, consider also specifying the same arguments to :py:func:`load_doctests`. @@ -200,7 +202,8 @@ def setUpPyfakefs(self, modules_to_reload=None, modules_to_patch=None, allow_root_user=True, - use_known_patches=True): + use_known_patches=True, + patch_open_code=False): """Bind the file-related modules to the :py:class:`pyfakefs` fake file system instead of the real file system. Also bind the fake `open()` function. @@ -223,7 +226,8 @@ def setUpPyfakefs(self, modules_to_reload=modules_to_reload, modules_to_patch=modules_to_patch, allow_root_user=allow_root_user, - use_known_patches=use_known_patches + use_known_patches=use_known_patches, + patch_open_code=patch_open_code ) self._stubber.setUp() @@ -257,9 +261,7 @@ class TestCase(unittest.TestCase, TestCaseMixin): def __init__(self, methodName='runTest', additional_skip_names=None, modules_to_reload=None, - modules_to_patch=None, - allow_root_user=True, - use_known_patches=True): + modules_to_patch=None): """Creates the test class instance and the patcher used to stub out file system related modules. @@ -272,8 +274,6 @@ def __init__(self, methodName='runTest', self.additional_skip_names = additional_skip_names self.modules_to_reload = modules_to_reload self.modules_to_patch = modules_to_patch - self.allow_root_user = allow_root_user - self.use_known_patches = use_known_patches @Deprecator('add_real_file') def copyRealFile(self, real_file_path, fake_file_path=None, @@ -358,8 +358,32 @@ class Patcher: def __init__(self, additional_skip_names=None, modules_to_reload=None, modules_to_patch=None, - allow_root_user=True, use_known_patches=True): - """For a description of the arguments, see TestCase.__init__""" + allow_root_user=True, use_known_patches=True, + patch_open_code=False): + """ + Args: + additional_skip_names: names of modules inside of which no module + replacement shall be performed, in addition to the names in + :py:attr:`fake_filesystem_unittest.Patcher.SKIPNAMES`. + Instead of the module names, the modules themselves + may be used. + modules_to_reload: A list of modules that need to be reloaded + to be patched dynamically; may be needed if the module + imports file system modules under an alias + + .. caution:: Reloading modules may have unwanted side effects. + modules_to_patch: A dictionary of fake modules mapped to the + fully qualified patched module names. Can be used to add + patching of modules not provided by `pyfakefs`. + allow_root_user: If True (default), if the test is run as root + user, the user in the fake file system is also considered a + root user, otherwise it is always considered a regular user. + use_known_patches: If True (the default), some patches for commonly + used packages are applied which make them usable with pyfakefs. + patch_open_code: If True, `io.open_code` is patched. The default + is not to patch it, as it mostly is used to load compiled + modules that are not in the fake file system. + """ if not allow_root_user: # set non-root IDs even if the real user is root @@ -370,6 +394,7 @@ def __init__(self, additional_skip_names=None, # save the original open function for use in pytest plugin self.original_open = open self.fake_open = None + self.patch_open_code = patch_open_code if additional_skip_names is not None: skip_names = [m.__name__ if inspect.ismodule(m) else m @@ -589,6 +614,7 @@ def _refresh(self): self._stubs = mox3_stubout.StubOutForTesting() self.fs = fake_filesystem.FakeFilesystem(patcher=self) + self.fs.patch_open_code = self.patch_open_code for name in self._fake_module_classes: self.fake_modules[name] = self._fake_module_classes[name](self.fs) if hasattr(self.fake_modules[name], 'skip_names'): diff --git a/pyfakefs/tests/fake_open_test.py b/pyfakefs/tests/fake_open_test.py index 966060a9..5974ac19 100644 --- a/pyfakefs/tests/fake_open_test.py +++ b/pyfakefs/tests/fake_open_test.py @@ -21,15 +21,24 @@ import locale import os import stat +import sys import time import unittest from pyfakefs import fake_filesystem -from pyfakefs.fake_filesystem import is_root, PERM_READ +from pyfakefs.fake_filesystem import is_root, PERM_READ, FakeIoModule from pyfakefs.tests.test_utils import RealFsTestCase class FakeFileOpenTestBase(RealFsTestCase): + def setUp(self): + super(FakeFileOpenTestBase, self).setUp() + if self.use_real_fs(): + self.open = io.open + else: + self.fake_io_module = FakeIoModule(self.filesystem) + self.open = self.fake_io_module.open + def path_separator(self): return '!' @@ -89,7 +98,7 @@ def test_unicode_contents(self): contents = f.read() self.assertEqual(contents, text_fractions) - def test_byte_contents_py3(self): + def test_byte_contents(self): file_path = self.make_path('foo') byte_fractions = b'\xe2\x85\x93 \xe2\x85\x94 \xe2\x85\x95 \xe2\x85\x96' with self.open(file_path, 'wb') as f: @@ -110,15 +119,6 @@ def test_write_str_read_bytes(self): self.assertEqual(str_contents, contents.decode( locale.getpreferredencoding(False))) - def test_byte_contents(self): - file_path = self.make_path('foo') - byte_fractions = b'\xe2\x85\x93 \xe2\x85\x94 \xe2\x85\x95 \xe2\x85\x96' - with self.open(file_path, 'wb') as f: - f.write(byte_fractions) - with self.open(file_path, 'rb') as f: - contents = f.read() - self.assertEqual(contents, byte_fractions) - def test_open_valid_file(self): contents = [ 'I am he as\n', @@ -926,6 +926,83 @@ def use_real_fs(self): return True +@unittest.skipIf(sys.version_info < (3, 8), + 'open_code only present since Python 3.8') +class FakeFilePatchedOpenCodeTest(FakeFileOpenTestBase): + + def setUp(self): + super(FakeFilePatchedOpenCodeTest, self).setUp() + if self.use_real_fs(): + self.open_code = io.open_code + else: + self.filesystem.patch_open_code = True + self.open_code = self.fake_io_module.open_code + + def tearDown(self): + if not self.use_real_fs(): + self.filesystem.patch_open_code = False + super(FakeFilePatchedOpenCodeTest, self).tearDown() + + def test_invalid_path(self): + with self.assertRaises(TypeError): + self.open_code(4) + + def test_byte_contents_open_code(self): + byte_fractions = b'\xe2\x85\x93 \xe2\x85\x94 \xe2\x85\x95 \xe2\x85\x96' + file_path = self.make_path('foo') + self.create_file(file_path, contents=byte_fractions) + with self.open_code(file_path) as f: + contents = f.read() + self.assertEqual(contents, byte_fractions) + + def test_open_code_in_real_fs(self): + self.skip_real_fs() + file_path = __file__ + with self.assertRaises(OSError): + self.open_code(file_path) + + +class RealPatchedFileOpenCodeTest(FakeFilePatchedOpenCodeTest): + def use_real_fs(self): + return True + + +@unittest.skipIf(sys.version_info < (3, 8), + 'open_code only present since Python 3.8') +class FakeFileUnpatchedOpenCodeTest(FakeFileOpenTestBase): + + def setUp(self): + super(FakeFileUnpatchedOpenCodeTest, self).setUp() + if self.use_real_fs(): + self.open_code = io.open_code + else: + self.open_code = self.fake_io_module.open_code + + def test_invalid_path(self): + with self.assertRaises(TypeError): + self.open_code(4) + + def test_open_code_in_real_fs(self): + file_path = __file__ + + with self.open_code(file_path) as f: + contents = f.read() + self.assertTrue(len(contents) > 100) + + +class RealUnpatchedFileOpenCodeTest(FakeFileUnpatchedOpenCodeTest): + def use_real_fs(self): + return True + + def test_byte_contents_open_code(self): + byte_fractions = b'\xe2\x85\x93 \xe2\x85\x94 \xe2\x85\x95 \xe2\x85\x96' + file_path = self.make_path('foo') + self.create_file(file_path, contents=byte_fractions) + with self.open_code(file_path) as f: + contents = f.read() + self.assertEqual(contents, byte_fractions) + + class BufferingModeTest(FakeFileOpenTestBase): def test_no_buffering(self): file_path = self.make_path("buffertest.bin") @@ -1168,10 +1245,6 @@ class OpenFileWithEncodingTest(FakeFileOpenTestBase): def setUp(self): super(OpenFileWithEncodingTest, self).setUp() - if self.use_real_fs(): - self.open = io.open - else: - self.open = fake_filesystem.FakeFileOpen(self.filesystem) self.file_path = self.make_path('foo') def test_write_str_read_bytes(self): @@ -1451,10 +1524,6 @@ def use_real_fs(self): class FakeFileOpenLineEndingWithEncodingTest(FakeFileOpenTestBase): def setUp(self): super(FakeFileOpenLineEndingWithEncodingTest, self).setUp() - if self.use_real_fs(): - self.open = io.open - else: - self.open = fake_filesystem.FakeFileOpen(self.filesystem) def test_read_standard_newline_mode(self): file_path = self.make_path('some_file')