From 601e702248ecc323a0f763d8f654212a890b55dc Mon Sep 17 00:00:00 2001 From: Shantanu Jain Date: Tue, 10 Sep 2024 17:57:55 -0700 Subject: [PATCH] add error for shadowing --- Include/internal/pycore_moduleobject.h | 1 + Lib/test/test_import/__init__.py | 233 +++++++++++++++---------- Objects/moduleobject.c | 14 +- Python/ceval.c | 88 ++++++++-- 4 files changed, 217 insertions(+), 119 deletions(-) diff --git a/Include/internal/pycore_moduleobject.h b/Include/internal/pycore_moduleobject.h index 1cd4745b23cd290..dacc00dba544951 100644 --- a/Include/internal/pycore_moduleobject.h +++ b/Include/internal/pycore_moduleobject.h @@ -12,6 +12,7 @@ extern void _PyModule_Clear(PyObject *); extern void _PyModule_ClearDict(PyObject *); extern int _PyModuleSpec_IsInitializing(PyObject *); extern int _PyModuleSpec_GetFileOrigin(PyObject *, PyObject **); +extern int _PyModule_IsPossiblyShadowing(PyObject *); extern int _PyModule_IsExtension(PyObject *obj); diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index 3d89d69955bb076..a5f144e49f515de 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -786,104 +786,133 @@ def test_issue105979(self): str(cm.exception)) def test_script_shadowing_stdlib(self): - with os_helper.temp_dir() as tmp: - with open(os.path.join(tmp, "fractions.py"), "w", encoding='utf-8') as f: - f.write("import fractions\nfractions.Fraction") - - expected_error = ( - rb"AttributeError: module 'fractions' has no attribute 'Fraction' " - rb"\(consider renaming '.*fractions.py' since it has the " - rb"same name as the standard library module named 'fractions' " - rb"and the import system gives it precedence\)" + script_errors = [ + ( + "import fractions\nfractions.Fraction", + rb"AttributeError: module 'fractions' has no attribute 'Fraction'" + ), + ( + "from fractions import Fraction", + rb"ImportError: cannot import name 'Fraction' from 'fractions'" ) + ] + for script, error in script_errors: + with os_helper.temp_dir() as tmp: + with open(os.path.join(tmp, "fractions.py"), "w", encoding='utf-8') as f: + f.write(script) + + expected_error = error + ( + rb" \(consider renaming '.*fractions.py' since it has the " + rb"same name as the standard library module named 'fractions' " + rb"and the import system gives it precedence\)" + ) - popen = script_helper.spawn_python(os.path.join(tmp, "fractions.py"), cwd=tmp) - stdout, stderr = popen.communicate() - self.assertRegex(stdout, expected_error) + popen = script_helper.spawn_python(os.path.join(tmp, "fractions.py"), cwd=tmp) + stdout, stderr = popen.communicate() + self.assertRegex(stdout, expected_error) - popen = script_helper.spawn_python('-m', 'fractions', cwd=tmp) - stdout, stderr = popen.communicate() - self.assertRegex(stdout, expected_error) + popen = script_helper.spawn_python('-m', 'fractions', cwd=tmp) + stdout, stderr = popen.communicate() + self.assertRegex(stdout, expected_error) - popen = script_helper.spawn_python('-c', 'import fractions', cwd=tmp) - stdout, stderr = popen.communicate() - self.assertRegex(stdout, expected_error) + popen = script_helper.spawn_python('-c', 'import fractions', cwd=tmp) + stdout, stderr = popen.communicate() + self.assertRegex(stdout, expected_error) - # and there's no error at all when using -P - popen = script_helper.spawn_python('-P', 'fractions.py', cwd=tmp) - stdout, stderr = popen.communicate() - self.assertEqual(stdout, b'') + # and there's no error at all when using -P + popen = script_helper.spawn_python('-P', 'fractions.py', cwd=tmp) + stdout, stderr = popen.communicate() + self.assertEqual(stdout, b'') - tmp_child = os.path.join(tmp, "child") - os.mkdir(tmp_child) + tmp_child = os.path.join(tmp, "child") + os.mkdir(tmp_child) - # test the logic with different cwd - popen = script_helper.spawn_python(os.path.join(tmp, "fractions.py"), cwd=tmp_child) - stdout, stderr = popen.communicate() - self.assertRegex(stdout, expected_error) + # test the logic with different cwd + popen = script_helper.spawn_python(os.path.join(tmp, "fractions.py"), cwd=tmp_child) + stdout, stderr = popen.communicate() + self.assertRegex(stdout, expected_error) - popen = script_helper.spawn_python('-m', 'fractions', cwd=tmp_child) - stdout, stderr = popen.communicate() - self.assertEqual(stdout, b'') # no error + popen = script_helper.spawn_python('-m', 'fractions', cwd=tmp_child) + stdout, stderr = popen.communicate() + self.assertEqual(stdout, b'') # no error - popen = script_helper.spawn_python('-c', 'import fractions', cwd=tmp_child) - stdout, stderr = popen.communicate() - self.assertEqual(stdout, b'') # no error + popen = script_helper.spawn_python('-c', 'import fractions', cwd=tmp_child) + stdout, stderr = popen.communicate() + self.assertEqual(stdout, b'') # no error def test_package_shadowing_stdlib_module(self): - with os_helper.temp_dir() as tmp: - os.mkdir(os.path.join(tmp, "fractions")) - with open(os.path.join(tmp, "fractions", "__init__.py"), "w", encoding='utf-8') as f: - f.write("shadowing_module = True") - with open(os.path.join(tmp, "main.py"), "w", encoding='utf-8') as f: - f.write(""" -import fractions -fractions.shadowing_module -fractions.Fraction -""") - - expected_error = ( - rb"AttributeError: module 'fractions' has no attribute 'Fraction' " - rb"\(consider renaming '.*fractions.__init__.py' since it has the " - rb"same name as the standard library module named 'fractions' " - rb"and the import system gives it precedence\)" + script_errors = [ + ( + "fractions.Fraction", + rb"AttributeError: module 'fractions' has no attribute 'Fraction'" + ), + ( + "from fractions import Fraction", + rb"ImportError: cannot import name 'Fraction' from 'fractions'" ) + ] + for script, error in script_errors: + with os_helper.temp_dir() as tmp: + os.mkdir(os.path.join(tmp, "fractions")) + with open( + os.path.join(tmp, "fractions", "__init__.py"), "w", encoding='utf-8' + ) as f: + f.write("shadowing_module = True") + with open(os.path.join(tmp, "main.py"), "w", encoding='utf-8') as f: + f.write("import fractions; fractions.shadowing_module\n") + f.write(script) + + expected_error = error + ( + rb" \(consider renaming '.*fractions.__init__.py' since it has the " + rb"same name as the standard library module named 'fractions' " + rb"and the import system gives it precedence\)" + ) - popen = script_helper.spawn_python(os.path.join(tmp, "main.py"), cwd=tmp) - stdout, stderr = popen.communicate() - self.assertRegex(stdout, expected_error) + popen = script_helper.spawn_python(os.path.join(tmp, "main.py"), cwd=tmp) + stdout, stderr = popen.communicate() + self.assertRegex(stdout, expected_error) - popen = script_helper.spawn_python('-m', 'main', cwd=tmp) - stdout, stderr = popen.communicate() - self.assertRegex(stdout, expected_error) + popen = script_helper.spawn_python('-m', 'main', cwd=tmp) + stdout, stderr = popen.communicate() + self.assertRegex(stdout, expected_error) - # and there's no shadowing at all when using -P - popen = script_helper.spawn_python('-P', 'main.py', cwd=tmp) - stdout, stderr = popen.communicate() - self.assertRegex(stdout, b"module 'fractions' has no attribute 'shadowing_module'") + # and there's no shadowing at all when using -P + popen = script_helper.spawn_python('-P', 'main.py', cwd=tmp) + stdout, stderr = popen.communicate() + self.assertRegex(stdout, b"module 'fractions' has no attribute 'shadowing_module'") def test_script_shadowing_third_party(self): + script_errors = [ + ( + "import numpy\nnumpy.array", + rb"AttributeError: module 'numpy' has no attribute 'array'" + ), + ( + "from numpy import array", + rb"ImportError: cannot import name 'array' from 'numpy'" + ) + ] with os_helper.temp_dir() as tmp: - with open(os.path.join(tmp, "numpy.py"), "w", encoding='utf-8') as f: - f.write("import numpy\nnumpy.array") + for script, error in script_errors: + with open(os.path.join(tmp, "numpy.py"), "w", encoding='utf-8') as f: + f.write(script) - expected_error = ( - rb"AttributeError: module 'numpy' has no attribute 'array' " - rb"\(consider renaming '.*numpy.py' if it has the " - rb"same name as a third-party module you intended to import\)\s+\Z" - ) + expected_error = error + ( + rb" \(consider renaming '.*numpy.py' if it has the " + rb"same name as a third-party module you intended to import\)\s+\Z" + ) - popen = script_helper.spawn_python(os.path.join(tmp, "numpy.py")) - stdout, stderr = popen.communicate() - self.assertRegex(stdout, expected_error) + popen = script_helper.spawn_python(os.path.join(tmp, "numpy.py")) + stdout, stderr = popen.communicate() + self.assertRegex(stdout, expected_error) - popen = script_helper.spawn_python('-m', 'numpy', cwd=tmp) - stdout, stderr = popen.communicate() - self.assertRegex(stdout, expected_error) + popen = script_helper.spawn_python('-m', 'numpy', cwd=tmp) + stdout, stderr = popen.communicate() + self.assertRegex(stdout, expected_error) - popen = script_helper.spawn_python('-c', 'import numpy', cwd=tmp) - stdout, stderr = popen.communicate() - self.assertRegex(stdout, expected_error) + popen = script_helper.spawn_python('-c', 'import numpy', cwd=tmp) + stdout, stderr = popen.communicate() + self.assertRegex(stdout, expected_error) def test_script_maybe_not_shadowing_third_party(self): with os_helper.temp_dir() as tmp: @@ -893,11 +922,17 @@ def test_script_maybe_not_shadowing_third_party(self): expected_error = ( rb"AttributeError: module 'numpy' has no attribute 'attr'\s+\Z" ) - popen = script_helper.spawn_python('-c', 'import numpy; numpy.attr', cwd=tmp) stdout, stderr = popen.communicate() self.assertRegex(stdout, expected_error) + expected_error = ( + rb"ImportError: cannot import name 'attr' from 'numpy' \(.*\)\s+\Z" + ) + popen = script_helper.spawn_python('-c', 'from numpy import attr', cwd=tmp) + stdout, stderr = popen.communicate() + self.assertRegex(stdout, expected_error) + def test_script_shadowing_stdlib_edge_cases(self): with os_helper.temp_dir() as tmp: with open(os.path.join(tmp, "fractions.py"), "w", encoding='utf-8') as f: @@ -983,28 +1018,32 @@ class substr(str): ) def test_script_shadowing_stdlib_sys_path_modification(self): - with os_helper.temp_dir() as tmp: - with open(os.path.join(tmp, "fractions.py"), "w", encoding='utf-8') as f: - f.write("shadowing_module = True") - - expected_error = ( - rb"AttributeError: module 'fractions' has no attribute 'Fraction' " - rb"\(consider renaming '.*fractions.py' since it has the " - rb"same name as the standard library module named 'fractions' " - rb"and the import system gives it precedence\)" + script_errors = [ + ( + "import fractions\nfractions.Fraction", + rb"AttributeError: module 'fractions' has no attribute 'Fraction'" + ), + ( + "from fractions import Fraction", + rb"ImportError: cannot import name 'Fraction' from 'fractions'" ) + ] + for script, error in script_errors: + with os_helper.temp_dir() as tmp: + with open(os.path.join(tmp, "fractions.py"), "w", encoding='utf-8') as f: + f.write("shadowing_module = True") + with open(os.path.join(tmp, "main.py"), "w", encoding='utf-8') as f: + f.write('import sys; sys.path.insert(0, "this_folder_does_not_exist")\n') + f.write(script) + expected_error = error + ( + rb" \(consider renaming '.*fractions.py' since it has the " + rb"same name as the standard library module named 'fractions' " + rb"and the import system gives it precedence\)" + ) - with open(os.path.join(tmp, "main.py"), "w", encoding='utf-8') as f: - f.write(""" -import sys -sys.path.insert(0, "this_folder_does_not_exist") -import fractions -fractions.Fraction -""") - - popen = script_helper.spawn_python("main.py", cwd=tmp) - stdout, stderr = popen.communicate() - self.assertRegex(stdout, expected_error) + popen = script_helper.spawn_python("main.py", cwd=tmp) + stdout, stderr = popen.communicate() + self.assertRegex(stdout, expected_error) @skip_if_dont_write_bytecode diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index 4f9226bdb711bab..98acd90972a2a73 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -855,8 +855,8 @@ _PyModuleSpec_GetFileOrigin(PyObject *spec, PyObject **p_origin) return 1; } -static int -_is_module_possibly_shadowing(PyObject *origin) +int +_PyModule_IsPossiblyShadowing(PyObject *origin) { // origin must be a unicode subtype // Returns 1 if the module at origin could be shadowing a module of the @@ -985,7 +985,7 @@ _Py_module_getattro_impl(PyModuleObject *m, PyObject *name, int suppress) goto done; } - int is_possibly_shadowing = _is_module_possibly_shadowing(origin); + int is_possibly_shadowing = _PyModule_IsPossiblyShadowing(origin); if (is_possibly_shadowing < 0) { goto done; } @@ -1011,7 +1011,10 @@ _Py_module_getattro_impl(PyModuleObject *m, PyObject *name, int suppress) } else { int rc = _PyModuleSpec_IsInitializing(spec); - if (rc > 0) { + if (rc < 0) { + goto done; + } + else if (rc > 0) { if (is_possibly_shadowing) { assert(origin); // For third-party modules, only mention the possibility of @@ -1037,7 +1040,8 @@ _Py_module_getattro_impl(PyModuleObject *m, PyObject *name, int suppress) mod_name, name); } } - else if (rc == 0) { + else { + assert(rc == 0); rc = _PyModuleSpec_IsUninitializedSubmodule(spec, name); if (rc > 0) { PyErr_Format(PyExc_AttributeError, diff --git a/Python/ceval.c b/Python/ceval.c index 3a85bd6b06c5e36..90855e88e0adb9d 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -2801,6 +2801,7 @@ _PyEval_ImportFrom(PyThreadState *tstate, PyObject *v, PyObject *name) mod_name_or_unknown = mod_name; } // mod_name is no longer an owned reference + assert(mod_name_or_unknown); assert(mod_name == NULL || mod_name == mod_name_or_unknown); origin = NULL; @@ -2818,27 +2819,80 @@ _PyEval_ImportFrom(PyThreadState *tstate, PyObject *v, PyObject *name) if (_PyModuleSpec_GetFileOrigin(spec, &origin) < 0) { goto done; } - if (origin == NULL) { + + int is_possibly_shadowing = _PyModule_IsPossiblyShadowing(origin); + if (is_possibly_shadowing < 0) { + goto done; + } + int is_possibly_shadowing_stdlib = 0; + if (is_possibly_shadowing) { + PyObject *stdlib_modules = PySys_GetObject("stdlib_module_names"); + if (stdlib_modules && PyAnySet_Check(stdlib_modules)) { + is_possibly_shadowing_stdlib = PySet_Contains(stdlib_modules, mod_name_or_unknown); + if (is_possibly_shadowing_stdlib < 0) { + goto done; + } + } + } + + if (is_possibly_shadowing_stdlib) { + assert(origin); errmsg = PyUnicode_FromFormat( - "cannot import name %R from %R (unknown location)", - name, mod_name_or_unknown + "cannot import name %R from %R " + "(consider renaming %R since it has the same " + "name as the standard library module named %R " + "and the import system gives it precedence)", + name, mod_name_or_unknown, origin, mod_name_or_unknown ); - goto done_with_errmsg; } - - int rc = _PyModuleSpec_IsInitializing(spec); - if (rc < 0) { - Py_DECREF(mod_name_or_unknown); - Py_DECREF(origin); - return NULL; + else { + int rc = _PyModuleSpec_IsInitializing(spec); + if (rc < 0) { + goto done; + } + else if (rc > 0) { + if (is_possibly_shadowing) { + assert(origin); + // For third-party modules, only mention the possibility of + // shadowing if the module is being initialized. + errmsg = PyUnicode_FromFormat( + "cannot import name %R from %R " + "(consider renaming %R if it has the same name " + "as a third-party module you intended to import)", + name, mod_name_or_unknown, origin + ); + } + else if (origin) { + errmsg = PyUnicode_FromFormat( + "cannot import name %R from partially initialized module %R " + "(most likely due to a circular import) (%S)", + name, mod_name_or_unknown, origin + ); + } + else { + errmsg = PyUnicode_FromFormat( + "cannot import name %R from partially initialized module %R " + "(most likely due to a circular import)", + name, mod_name_or_unknown + ); + } + } + else { + assert(rc == 0); + if (origin) { + errmsg = PyUnicode_FromFormat( + "cannot import name %R from %R (%S)", + name, mod_name_or_unknown, origin + ); + } + else { + errmsg = PyUnicode_FromFormat( + "cannot import name %R from %R (unknown location)", + name, mod_name_or_unknown + ); + } + } } - const char *fmt = - rc ? - "cannot import name %R from partially initialized module %R " - "(most likely due to a circular import) (%S)" : - "cannot import name %R from %R (%S)"; - - errmsg = PyUnicode_FromFormat(fmt, name, mod_name_or_unknown, origin); done_with_errmsg: /* NULL checks for errmsg and mod_name done by PyErr_SetImportError. */