1058 enable fix warnings (#1059)

* #1058: have Makefile use PYTHONWARNINGS=all by default for (almost) all commands

* #1058 fix linux tests warnings

* #1058: try not to use imp module

* #1058: get rid of imp module completely

* #1058: ignore unicode warnings

* #1058: ignore stderr from procsmem.py

* #1058: fix resource warning from Popen

* #1058: get rid of contextlib.nested (deprecated)

HISTORY.rst

@@ -19,6 +19,7 @@
   comprehensions.
 - 1040_: implemented full unicode support.
 - 1051_: disk_usage() on Python 3 is now able to accept bytes.
+- 1058_: test suite now enables all warnings by default.
 
 **Bug fixes**
 
@@ -41,6 +42,7 @@
 - 1046_: [Windows] disk_partitions() on Windows overrides user's SetErrorMode.
 - 1047_: [Windows] Process username(): memory leak in case exception is thrown.
 - 1048_: [Windows] users()'s host field report an invalid IP address.
+- 1058_: fixed Python warnings.
 
 **Porting notes**
 

Makefile

@@ -63,11 +63,11 @@ _:
 
 # Compile without installing.
 build: _
-	$(PYTHON) setup.py build
+	PYTHONWARNINGS=all $(PYTHON) setup.py build
 	@# copies compiled *.so files in ./psutil directory in order to allow
 	@# "import psutil" when using the interactive interpreter from within
 	@# this directory.
-	$(PYTHON) setup.py build_ext -i
+	PYTHONWARNINGS=all $(PYTHON) setup.py build_ext -i
 	rm -rf tmp
 
 # Install this package + GIT hooks. Install is done:
@@ -77,7 +77,7 @@ install:
 	${MAKE} build
 	# make sure setuptools is installed (needed for 'develop' / edit mode)
 	$(PYTHON) -c "import setuptools"
-	$(PYTHON) setup.py develop $(INSTALL_OPTS)
+	PYTHONWARNINGS=all $(PYTHON) setup.py develop $(INSTALL_OPTS)
 	rm -rf tmp
 
 # Uninstall this package via pip.
@@ -86,7 +86,7 @@ uninstall:
 
 # Install PIP (only if necessary).
 install-pip:
-	$(PYTHON) -c \
+	PYTHONWARNINGS=all $(PYTHON) -c \
 		"import sys, ssl, os, pkgutil, tempfile, atexit; \
 		sys.exit(0) if pkgutil.find_loader('pip') else None; \
 		pyexc = 'from urllib.request import urlopen' if sys.version_info[0] == 3 else 'from urllib2 import urlopen'; \
@@ -118,65 +118,65 @@ setup-dev-env:
 # Run all tests.
 test:
 	${MAKE} install
-	$(PYTHON) $(TSCRIPT)
+	PYTHONWARNINGS=all $(PYTHON) $(TSCRIPT)
 
 # Run process-related API tests.
 test-process:
 	${MAKE} install
-	$(PYTHON) -m unittest -v psutil.tests.test_process
+	PYTHONWARNINGS=all $(PYTHON) -m unittest -v psutil.tests.test_process
 
 # Run system-related API tests.
 test-system:
 	${MAKE} install
-	$(PYTHON) -m unittest -v psutil.tests.test_system
+	PYTHONWARNINGS=all $(PYTHON) -m unittest -v psutil.tests.test_system
 
 # Run miscellaneous tests.
 test-misc:
 	${MAKE} install
-	$(PYTHON) psutil/tests/test_misc.py
+	PYTHONWARNINGS=all $(PYTHON) psutil/tests/test_misc.py
 
 # Test APIs dealing with strings.
 test-unicode:
 	${MAKE} install
-	$(PYTHON) psutil/tests/test_unicode.py
+	PYTHONWARNINGS=all $(PYTHON) psutil/tests/test_unicode.py
 
 # APIs sanity tests.
 test-contracts:
 	${MAKE} install
-	$(PYTHON) psutil/tests/test_contracts.py
+	PYTHONWARNINGS=all $(PYTHON) psutil/tests/test_contracts.py
 
 # Test net_connections() and Process.connections().
 test-connections:
 	${MAKE} install
-	$(PYTHON) psutil/tests/test_connections.py
+	PYTHONWARNINGS=all $(PYTHON) psutil/tests/test_connections.py
 
 # POSIX specific tests.
 test-posix:
 	${MAKE} install
-	$(PYTHON) psutil/tests/test_posix.py
+	PYTHONWARNINGS=all $(PYTHON) psutil/tests/test_posix.py
 
 # Run specific platform tests only.
 test-platform:
 	${MAKE} install
-	$(PYTHON) psutil/tests/test_`$(PYTHON) -c 'import psutil; print([x.lower() for x in ("LINUX", "BSD", "OSX", "SUNOS", "WINDOWS") if getattr(psutil, x)][0])'`.py
+	PYTHONWARNINGS=all $(PYTHON) psutil/tests/test_`$(PYTHON) -c 'import psutil; print([x.lower() for x in ("LINUX", "BSD", "OSX", "SUNOS", "WINDOWS") if getattr(psutil, x)][0])'`.py
 
 # Memory leak tests.
 test-memleaks:
 	${MAKE} install
-	$(PYTHON) psutil/tests/test_memory_leaks.py
+	PYTHONWARNINGS=all $(PYTHON) psutil/tests/test_memory_leaks.py
 
 # Run a specific test by name, e.g.
 # make test-by-name psutil.tests.test_system.TestSystemAPIs.test_cpu_times
 test-by-name:
 	${MAKE} install
-	@$(PYTHON) -m unittest -v $(ARGS)
+	@PYTHONWARNINGS=all $(PYTHON) -m unittest -v $(ARGS)
 
 # Run test coverage.
 coverage:
 	${MAKE} install
 	# Note: coverage options are controlled by .coveragerc file
 	rm -rf .coverage htmlcov
-	$(PYTHON) -m coverage run $(TSCRIPT)
+	PYTHONWARNINGS=all $(PYTHON) -m coverage run $(TSCRIPT)
 	$(PYTHON) -m coverage report
 	@echo "writing results to htmlcov/index.html"
 	$(PYTHON) -m coverage html
@@ -197,7 +197,7 @@ flake8:
 	@git ls-files | grep \\.py$ | xargs $(PYTHON) -m flake8
 
 check-manifest:
-	$(PYTHON) -m check_manifest -v $(ARGS)
+	PYTHONWARNINGS=all $(PYTHON) -m check_manifest -v $(ARGS)
 
 # ===================================================================
 # GIT
@@ -220,22 +220,22 @@ install-git-hooks:
 # Upload source tarball on https://pypi.python.org/pypi/psutil.
 upload-src:
 	${MAKE} clean
-	$(PYTHON) setup.py sdist upload
+	PYTHONWARNINGS=all $(PYTHON) setup.py sdist upload
 
 # Download exes/wheels hosted on appveyor.
 win-download-exes:
-	$(PYTHON) scripts/internal/download_exes.py --user giampaolo --project psutil
+	PYTHONWARNINGS=all $(PYTHON) scripts/internal/download_exes.py --user giampaolo --project psutil
 
 # Upload exes/wheels in dist/* directory to PYPI.
 win-upload-exes:
-	$(PYTHON) -m twine upload dist/*.exe
-	$(PYTHON) -m twine upload dist/*.whl
+	PYTHONWARNINGS=all $(PYTHON) -m twine upload dist/*.exe
+	PYTHONWARNINGS=all $(PYTHON) -m twine upload dist/*.whl
 
 # All the necessary steps before making a release.
 pre-release:
 	${MAKE} clean
 	${MAKE} install  # to import psutil from download_exes.py
-	$(PYTHON) -c \
+	PYTHONWARNINGS=all $(PYTHON) -c \
 		"from psutil import __version__ as ver; \
 		doc = open('docs/index.rst').read(); \
 		history = open('HISTORY.rst').read(); \
@@ -244,18 +244,18 @@ pre-release:
 		assert 'XXXX' not in history; \
 		"
 	${MAKE} win-download-exes
-	$(PYTHON) setup.py sdist
+	PYTHONWARNINGS=all $(PYTHON) setup.py sdist
 
 # Create a release: creates tar.gz and exes/wheels, uploads them,
 # upload doc, git tag release.
 release:
 	${MAKE} pre-release
-	$(PYTHON) -m twine upload dist/*  # upload tar.gz, exes, wheels on PYPI
+	PYTHONWARNINGS=all $(PYTHON) -m twine upload dist/*  # upload tar.gz, exes, wheels on PYPI
 	${MAKE} git-tag-release
 
 # Print announce of new release.
 print-announce:
-	@$(PYTHON) scripts/internal/print_announce.py
+	@PYTHONWARNINGS=all $(PYTHON) scripts/internal/print_announce.py
 
 # ===================================================================
 # Misc
@@ -267,12 +267,12 @@ grep-todos:
 # run script which benchmarks oneshot() ctx manager (see #799)
 bench-oneshot:
 	${MAKE} install
-	$(PYTHON) scripts/internal/bench_oneshot.py
+	PYTHONWARNINGS=all $(PYTHON) scripts/internal/bench_oneshot.py
 
 # same as above but using perf module (supposed to be more precise)
 bench-oneshot-2:
 	${MAKE} install
-	$(PYTHON) scripts/internal/bench_oneshot_2.py
+	PYTHONWARNINGS=all $(PYTHON) scripts/internal/bench_oneshot_2.py
 
 # generate a doc.zip file and manually upload it to PYPI.
 doc:
@@ -282,4 +282,4 @@ doc:
 
 # check whether the links mentioned in some files are valid.
 check-broken-links:
-		git ls-files | grep \\.rst$ | xargs $(PYTHON) scripts/internal/check_broken_links.py
+		git ls-files | grep \\.rst$ | xargs PYTHONWARNINGS=all $(PYTHON) scripts/internal/check_broken_links.py

psutil/_compat.py

@@ -5,13 +5,12 @@
 """Module which provides compatibility with older Python versions."""
 
 import collections
-import contextlib
 import functools
 import os
 import sys
 
 __all__ = ["PY3", "long", "xrange", "unicode", "basestring", "u", "b",
-           "callable", "lru_cache", "which", "nested"]
+           "callable", "lru_cache", "which"]
 
 PY3 = sys.version_info[0] == 3
 
@@ -248,50 +247,3 @@ def _access_check(fn, mode):
                     if _access_check(name, mode):
                         return name
         return None
-
-
-# A backport of contextlib.nested for Python 3.
-nested = getattr(contextlib, "nested", None)
-if nested is None:
-    @contextlib.contextmanager
-    def nested(*managers):
-        """Support multiple context managers in a single with-statement.
-
-        Code like this:
-
-            with nested(A, B, C) as (X, Y, Z):
-                <body>
-
-        is equivalent to this:
-
-            with A as X:
-                with B as Y:
-                    with C as Z:
-                        <body>
-        """
-        exits = []
-        vars = []
-        exc = (None, None, None)
-        try:
-            for mgr in managers:
-                exit = mgr.__exit__
-                enter = mgr.__enter__
-                vars.append(enter())
-                exits.append(exit)
-            yield vars
-        except:  # NOQA
-            exc = sys.exc_info()
-        finally:
-            while exits:
-                exit = exits.pop()
-                try:
-                    if exit(*exc):
-                        exc = (None, None, None)
-                except:  # NOQA
-                    exc = sys.exc_info()
-            if exc != (None, None, None):
-                # Don't rely on sys.exc_info() still containing
-                # the right information. Another exception may
-                # have been raised and caught by an exit method
-                # exc[1] already has the __traceback__ attribute populated
-                raise exc[1]

psutil/tests/__init__.py

@@ -58,14 +58,6 @@
 else:
     enum = None
 
-if PY3:
-    import importlib
-    # python <=3.3
-    if not hasattr(importlib, 'reload'):
-        import imp as importlib
-else:
-    import imp as importlib
-
 
 __all__ = [
     # constants
@@ -98,16 +90,13 @@
     'check_connection_ntuple', 'check_net_address',
     'get_free_port', 'unix_socket_path', 'bind_socket', 'bind_unix_socket',
     'tcp_socketpair', 'unix_socketpair', 'create_sockets',
+    # compat
+    'reload_module', 'import_module_by_path',
     # others
     'warn', 'copyload_shared_lib', 'is_namedtuple',
 ]
 
 
-# Enable all warnings by default.
-if 'PYTHONWARNINGS' not in os.environ:
-    warnings.simplefilter('always')
-
-
 # ===================================================================
 # --- constants
 # ===================================================================
@@ -353,16 +342,19 @@ def pyrun(src, **kwds):
 
 
 @_cleanup_on_err
-def sh(cmd):
+def sh(cmd, **kwds):
     """run cmd in a subprocess and return its output.
     raises RuntimeError on error.
     """
     shell = True if isinstance(cmd, (str, unicode)) else False
     # Prevents subprocess to open error dialogs in case of error.
     flags = 0x8000000 if WINDOWS and shell else 0
-    p = subprocess.Popen(cmd, shell=shell, stdout=subprocess.PIPE,
-                         stderr=subprocess.PIPE, universal_newlines=True,
-                         creationflags=flags)
+    kwds.setdefault("shell", shell)
+    kwds.setdefault("stdout", subprocess.PIPE)
+    kwds.setdefault("stderr", subprocess.PIPE)
+    kwds.setdefault("universal_newlines", True)
+    kwds.setdefault("creationflags", flags)
+    p = subprocess.Popen(cmd, **kwds)
     stdout, stderr = p.communicate()
     if p.returncode != 0:
         raise RuntimeError(stderr)
@@ -975,6 +967,40 @@ def check_connection_ntuple(conn):
     assert conn.status in valids, conn
 
 
+# ===================================================================
+# --- compatibility
+# ===================================================================
+
+
+def reload_module(module):
+    """Backport of importlib.reload of Python 3.3+."""
+    try:
+        import importlib
+        if not hasattr(importlib, 'reload'):  # python <=3.3
+            raise ImportError
+    except ImportError:
+        import imp
+        return imp.reload(module)
+    else:
+        return importlib.reload(module)
+
+
+def import_module_by_path(path):
+    name = os.path.splitext(os.path.basename(path))[0]
+    if sys.version_info[0] == 2:
+        import imp
+        return imp.load_source(name, path)
+    elif sys.version_info[:2] <= (3, 4):
+        from importlib.machinery import SourceFileLoader
+        return SourceFileLoader(name, path).load_module()
+    else:
+        import importlib.util
+        spec = importlib.util.spec_from_file_location(name, path)
+        mod = importlib.util.module_from_spec(spec)
+        spec.loader.exec_module(mod)
+        return mod
+
+
 # ===================================================================
 # --- others
 # ===================================================================