From 414fbd1e149f3e9f196976c2256b03ad69386dc7 Mon Sep 17 00:00:00 2001 From: snowman2 Date: Wed, 2 Sep 2020 19:59:00 -0500 Subject: [PATCH] ENH: Improve pathlib.Path support --- docs/history.rst | 1 + pyproj/datadir.py | 35 +++++---- pyproj/sync.py | 3 +- setup.py | 69 ++++++++++-------- test/test_datadir.py | 164 ++++++++++++++++++------------------------- test/test_sync.py | 3 +- 6 files changed, 128 insertions(+), 147 deletions(-) diff --git a/docs/history.rst b/docs/history.rst index 419288fc1..769590fec 100644 --- a/docs/history.rst +++ b/docs/history.rst @@ -24,6 +24,7 @@ Change Log * BUG: Fix handling of polygon holes when calculating area in Geod (pull #686) * ENH: Added :ref:`network` (#675, #691, #695) * ENH: Added support for debugging internal PROJ (pull #696) +* ENH: Added pathlib support for data directory methods (pull #702) 2.6.1 ~~~~~ diff --git a/pyproj/datadir.py b/pyproj/datadir.py index 43317ac7d..619c9f64f 100644 --- a/pyproj/datadir.py +++ b/pyproj/datadir.py @@ -4,6 +4,8 @@ import os import sys from distutils.spawn import find_executable +from pathlib import Path +from typing import Union from pyproj._datadir import ( # noqa: F401 _global_context_set_data_dir, @@ -15,18 +17,18 @@ _VALIDATED_PROJ_DATA = None -def set_data_dir(proj_data_dir: str) -> None: +def set_data_dir(proj_data_dir: Union[str, Path]) -> None: """ Set the data directory for PROJ to use. Parameters ---------- - proj_data_dir: str + proj_data_dir: Union[str, Path] The path to the PROJ data directory. """ global _USER_PROJ_DATA global _VALIDATED_PROJ_DATA - _USER_PROJ_DATA = proj_data_dir + _USER_PROJ_DATA = str(proj_data_dir) # set to none to re-validate _VALIDATED_PROJ_DATA = None # need to reset the global PROJ context @@ -35,16 +37,16 @@ def set_data_dir(proj_data_dir: str) -> None: _global_context_set_data_dir() -def append_data_dir(proj_data_dir: str) -> None: +def append_data_dir(proj_data_dir: Union[str, Path]) -> None: """ Add an additional data directory for PROJ to use. Parameters ---------- - proj_data_dir: str + proj_data_dir: Union[str, Path] The path to the PROJ data directory. """ - set_data_dir(os.pathsep.join([get_data_dir(), proj_data_dir])) + set_data_dir(os.pathsep.join([get_data_dir(), str(proj_data_dir)])) def get_data_dir() -> str: @@ -68,15 +70,14 @@ def get_data_dir() -> str: if _VALIDATED_PROJ_DATA is not None: return _VALIDATED_PROJ_DATA global _USER_PROJ_DATA - internal_datadir = os.path.join( - os.path.dirname(os.path.abspath(__file__)), "proj_dir", "share", "proj" - ) + internal_datadir = Path(__file__).absolute().parent / "proj_dir" / "share" / "proj" proj_lib_dirs = os.environ.get("PROJ_LIB", "") - prefix_datadir = os.path.join(sys.prefix, "share", "proj") + prefix_datadir = Path(sys.prefix, "share", "proj") def valid_data_dir(potential_data_dir): - if potential_data_dir is not None and os.path.exists( - os.path.join(potential_data_dir, "proj.db") + if ( + potential_data_dir is not None + and Path(potential_data_dir, "proj.db").exists() ): return True return False @@ -92,19 +93,17 @@ def valid_data_dirs(potential_data_dirs): if valid_data_dirs(_USER_PROJ_DATA): _VALIDATED_PROJ_DATA = _USER_PROJ_DATA elif valid_data_dir(internal_datadir): - _VALIDATED_PROJ_DATA = internal_datadir + _VALIDATED_PROJ_DATA = str(internal_datadir) elif valid_data_dirs(proj_lib_dirs): _VALIDATED_PROJ_DATA = proj_lib_dirs elif valid_data_dir(prefix_datadir): - _VALIDATED_PROJ_DATA = prefix_datadir + _VALIDATED_PROJ_DATA = str(prefix_datadir) else: proj_exe = find_executable("proj") if proj_exe is not None: - system_proj_dir = os.path.join( - os.path.dirname(os.path.dirname(proj_exe)), "share", "proj" - ) + system_proj_dir = Path(proj_exe).parent.parent / "share" / "proj" if valid_data_dir(system_proj_dir): - _VALIDATED_PROJ_DATA = system_proj_dir + _VALIDATED_PROJ_DATA = str(system_proj_dir) if _VALIDATED_PROJ_DATA is None: raise DataDirError( diff --git a/pyproj/sync.py b/pyproj/sync.py index 496f68981..71c725c17 100644 --- a/pyproj/sync.py +++ b/pyproj/sync.py @@ -264,8 +264,7 @@ def _load_grid_geojson(target_directory=None) -> Dict[str, Any]: short_name="files.geojson", directory=target_directory, ) - with open(local_path, encoding="utf-8") as gridf: - return json.load(gridf) + return json.loads(local_path.read_text(encoding="utf-8")) def get_transform_grid_list( diff --git a/setup.py b/setup.py index f4dc6ff64..3c1ecf251 100644 --- a/setup.py +++ b/setup.py @@ -2,22 +2,25 @@ import subprocess import sys from distutils.spawn import find_executable -from glob import glob +from pathlib import Path +from typing import Dict, List, Optional from pkg_resources import parse_version from setuptools import Extension, setup PROJ_MIN_VERSION = parse_version("7.2.0") -CURRENT_FILE_PATH = os.path.dirname(os.path.abspath(__file__)) -BASE_INTERNAL_PROJ_DIR = "proj_dir" -INTERNAL_PROJ_DIR = os.path.join(CURRENT_FILE_PATH, "pyproj", BASE_INTERNAL_PROJ_DIR) +CURRENT_FILE_PATH = Path(__file__).absolute().parent +BASE_INTERNAL_PROJ_DIR = Path("proj_dir") +INTERNAL_PROJ_DIR = CURRENT_FILE_PATH / "pyproj" / BASE_INTERNAL_PROJ_DIR -def check_proj_version(proj_dir): +def check_proj_version(proj_dir: Path): """checks that the PROJ library meets the minimum version""" - proj = os.path.join(proj_dir, "bin", "proj") - proj_ver_bytes = subprocess.check_output(proj, stderr=subprocess.STDOUT) - proj_ver_bytes = (proj_ver_bytes.decode("ascii").split()[1]).strip(",") + proj = proj_dir / "bin" / "proj" + proj_ver_bytes = subprocess.check_output( + str(proj), stderr=subprocess.STDOUT + ).decode("ascii") + proj_ver_bytes = (proj_ver_bytes.split()[1]).strip(",") proj_version = parse_version(proj_ver_bytes) if proj_version < PROJ_MIN_VERSION: raise SystemExit( @@ -29,15 +32,18 @@ def check_proj_version(proj_dir): return proj_version -def get_proj_dir(): +def get_proj_dir() -> Path: """ This function finds the base PROJ directory. """ - proj_dir = os.environ.get("PROJ_DIR") - if proj_dir is None and os.path.exists(INTERNAL_PROJ_DIR): + proj_dir_environ = os.environ.get("PROJ_DIR") + proj_dir: Optional[Path] = None + if proj_dir_environ is not None: + proj_dir = Path(proj_dir_environ) + if proj_dir is None and INTERNAL_PROJ_DIR.exists(): proj_dir = INTERNAL_PROJ_DIR print(f"Internally compiled directory being used {INTERNAL_PROJ_DIR}.") - elif proj_dir is None and not os.path.exists(INTERNAL_PROJ_DIR): + elif proj_dir is None and not INTERNAL_PROJ_DIR.exists(): proj = find_executable("proj", path=sys.prefix) if proj is None: proj = find_executable("proj") @@ -47,8 +53,8 @@ def get_proj_dir(): "For more information see: " "https://pyproj4.github.io/pyproj/stable/installation.html" ) - proj_dir = os.path.dirname(os.path.dirname(proj)) - elif proj_dir is not None and os.path.exists(proj_dir): + proj_dir = Path(proj).parent.parent + elif proj_dir is not None and proj_dir.exists(): print("PROJ_DIR is set, using existing proj4 installation..\n") else: raise SystemExit(f"ERROR: Invalid path for PROJ_DIR {proj_dir}") @@ -58,7 +64,7 @@ def get_proj_dir(): return proj_dir -def get_proj_libdirs(proj_dir): +def get_proj_libdirs(proj_dir: Path) -> List[str]: """ This function finds the library directories """ @@ -66,12 +72,12 @@ def get_proj_libdirs(proj_dir): libdirs = [] if proj_libdir is None: libdir_search_paths = ( - os.path.join(proj_dir, "lib"), - os.path.join(proj_dir, "lib64"), + proj_dir / "lib", + proj_dir / "lib64", ) for libdir_search_path in libdir_search_paths: - if os.path.exists(libdir_search_path): - libdirs.append(libdir_search_path) + if libdir_search_path.exists(): + libdirs.append(str(libdir_search_path)) if not libdirs: raise SystemExit( "ERROR: PROJ_LIBDIR dir not found. Please set PROJ_LIBDIR." @@ -81,15 +87,15 @@ def get_proj_libdirs(proj_dir): return libdirs -def get_proj_incdirs(proj_dir): +def get_proj_incdirs(proj_dir: Path) -> List[str]: """ This function finds the include directories """ proj_incdir = os.environ.get("PROJ_INCDIR") incdirs = [] if proj_incdir is None: - if os.path.exists(os.path.join(proj_dir, "include")): - incdirs.append(os.path.join(proj_dir, "include")) + if (proj_dir / "include").exists(): + incdirs.append(str(proj_dir / "include")) else: raise SystemExit( "ERROR: PROJ_INCDIR dir not found. Please set PROJ_INCDIR." @@ -114,16 +120,16 @@ def get_cythonize_options(): return cythonize_options -def get_libraries(libdirs): +def get_libraries(libdirs: List[str]) -> List[str]: """ This function gets the libraries to cythonize with """ libraries = ["proj"] if os.name == "nt": for libdir in libdirs: - projlib = glob(os.path.join(libdir, "proj*.lib")) + projlib = list(Path(libdir).glob("proj*.lib")) if projlib: - libraries = [os.path.basename(projlib[0]).split(".lib")[0]] + libraries = [str(projlib[0].stem)] break return libraries @@ -174,18 +180,19 @@ def get_extension_modules(): ) -def get_package_data(): +def get_package_data() -> Dict[str, List[str]]: """ This function retrieves the package data """ # setup package data package_data = {"pyproj": ["*.pyi", "py.typed"]} - if os.environ.get("PROJ_WHEEL") is not None and os.path.exists(INTERNAL_PROJ_DIR): + if os.environ.get("PROJ_WHEEL") is not None and INTERNAL_PROJ_DIR.exists(): package_data["pyproj"].append( - os.path.join(BASE_INTERNAL_PROJ_DIR, "share", "proj", "*") + str(BASE_INTERNAL_PROJ_DIR / "share" / "proj" / "*") ) - if os.environ.get("PROJ_WHEEL") is not None and os.path.exists( - os.path.join(CURRENT_FILE_PATH, "pyproj", ".lib") + if ( + os.environ.get("PROJ_WHEEL") is not None + and (CURRENT_FILE_PATH / "pyproj" / ".lib").exists() ): package_data["pyproj"].append(os.path.join(".lib", "*")) return package_data @@ -195,7 +202,7 @@ def get_version(): """ retreive pyproj version information (taken from Fiona) """ - with open(os.path.join("pyproj", "__init__.py"), "r") as f: + with open(Path("pyproj", "__init__.py"), "r") as f: for line in f: if line.find("__version__") >= 0: # parse __version__ and remove surrounding " or ' diff --git a/test/test_datadir.py b/test/test_datadir.py index 5fa4a505b..da162f1a6 100644 --- a/test/test_datadir.py +++ b/test/test_datadir.py @@ -1,6 +1,7 @@ import logging import os from contextlib import contextmanager +from pathlib import Path import pytest from mock import patch @@ -50,31 +51,22 @@ def proj_logging_env(): def create_projdb(tmpdir): - with open(os.path.join(tmpdir, "proj.db"), "w") as pjdb: - pjdb.write("DUMMY proj.db") + Path(tmpdir, "proj.db").write_text("DUMMY proj.db") -_INVALID_PATH = "/invalid/path/to/nowhere" - - -def setup_os_mock(os_mock, abspath_return=_INVALID_PATH, proj_dir=None): - os_mock.path.abspath.return_value = abspath_return - os_mock.path.join = os.path.join - os_mock.path.dirname = os.path.dirname - os_mock.path.exists = os.path.exists - os_mock.pathsep = os.pathsep - if proj_dir is None: - os_mock.environ = {} - else: - os_mock.environ = {"PROJ_LIB": proj_dir} +_INVALID_PATH = Path("/invalid/path/to/nowhere") def test_get_data_dir__missing(): - with proj_env(), pytest.raises(DataDirError), patch( + with proj_env(), pytest.raises(DataDirError), patch.dict( + os.environ, {}, clear=True + ), patch("pyproj.datadir.Path.absolute", return_value=_INVALID_PATH), patch( "pyproj.datadir.find_executable", return_value=None - ), patch("pyproj.datadir.os") as os_mock, patch("pyproj.datadir.sys") as sys_mock: - sys_mock.prefix = _INVALID_PATH - setup_os_mock(os_mock) + ), patch( + "pyproj.datadir.Path.absolute", return_value=_INVALID_PATH + ), patch( + "pyproj.datadir.sys.prefix", str(_INVALID_PATH) + ): assert get_data_dir() is None @@ -85,115 +77,99 @@ def test_pyproj_global_context_initialize__datadir_missing(): _pyproj_global_context_initialize() -def test_get_data_dir__from_user(tmp_path): +@pytest.mark.parametrize("projdir_type", [str, Path]) +def test_get_data_dir__from_user(projdir_type, tmp_path): tmpdir = tmp_path / "proj" tmpdir.mkdir() - tmpdir = str(tmpdir) tmpdir_env = tmp_path / "proj_env" tmpdir_env.mkdir() - tmpdir_env = str(tmpdir_env) - with proj_env(), patch("pyproj.datadir.os") as os_mock, patch( - "pyproj.datadir.sys" - ) as sys_mock: # noqa: E501 - setup_os_mock( - os_mock, - abspath_return=os.path.join(tmpdir, "randomfilename.py"), - proj_dir=tmpdir_env, - ) - sys_mock.prefix = tmpdir_env + with proj_env(), patch.dict( + os.environ, {"PROJ_LIB": str(tmpdir_env)}, clear=True + ), patch("pyproj.datadir.Path.absolute", return_value=tmpdir / "datadir.py"), patch( + "pyproj.datadir.sys.prefix", str(tmpdir_env) + ): # noqa: E501 create_projdb(tmpdir) create_projdb(tmpdir_env) - set_data_dir(tmpdir) - internal_proj_dir = os.path.join(tmpdir, "proj_dir", "share", "proj") - os.makedirs(internal_proj_dir) + set_data_dir(projdir_type(tmpdir)) + internal_proj_dir = tmpdir / "proj_dir" / "share" / "proj" + internal_proj_dir.mkdir(parents=True) create_projdb(internal_proj_dir) - assert get_data_dir() == tmpdir + assert get_data_dir() == str(tmpdir) def test_get_data_dir__internal(tmp_path): tmpdir = tmp_path / "proj" tmpdir.mkdir() - tmpdir = str(tmpdir) tmpdir_fake = tmp_path / "proj_fake" tmpdir_fake.mkdir() - tmpdir_fake = str(tmpdir_fake) - with proj_env(), patch("pyproj.datadir.os") as os_mock, patch( - "pyproj.datadir.sys" - ) as sys_mock: - setup_os_mock( - os_mock, - abspath_return=os.path.join(tmpdir, "randomfilename.py"), - proj_dir=tmpdir_fake, - ) - sys_mock.prefix = tmpdir_fake + with proj_env(), patch.dict( + os.environ, {"PROJ_LIB": str(tmpdir_fake)}, clear=True + ), patch("pyproj.datadir.Path.absolute", return_value=tmpdir / "datadir.py"), patch( + "pyproj.datadir.sys.prefix", str(tmpdir_fake) + ): create_projdb(tmpdir) create_projdb(tmpdir_fake) - internal_proj_dir = os.path.join(tmpdir, "proj_dir", "share", "proj") - os.makedirs(internal_proj_dir) + internal_proj_dir = tmpdir / "proj_dir" / "share" / "proj" + internal_proj_dir.mkdir(parents=True) create_projdb(internal_proj_dir) - assert get_data_dir() == internal_proj_dir + assert get_data_dir() == str(internal_proj_dir) def test_get_data_dir__from_env_var(tmp_path): - tmpdir = str(tmp_path) - with proj_env(), patch("pyproj.datadir.os") as os_mock, patch( - "pyproj.datadir.sys" - ) as sys_mock: - setup_os_mock(os_mock, proj_dir=tmpdir) - sys_mock.prefix = _INVALID_PATH - create_projdb(tmpdir) - assert get_data_dir() == tmpdir + with proj_env(), patch.dict( + os.environ, {"PROJ_LIB": str(tmp_path)}, clear=True + ), patch("pyproj.datadir.Path.absolute", return_value=_INVALID_PATH), patch( + "pyproj.datadir.sys.prefix", str(_INVALID_PATH) + ): + create_projdb(tmp_path) + assert get_data_dir() == str(tmp_path) def test_get_data_dir__from_env_var__multiple(tmp_path): - tmpdir = str(tmp_path) - with proj_env(), patch("pyproj.datadir.os") as os_mock, patch( - "pyproj.datadir.sys" - ) as sys_mock: - setup_os_mock(os_mock, proj_dir=os.pathsep.join([tmpdir, tmpdir, tmpdir])) - sys_mock.prefix = _INVALID_PATH - create_projdb(tmpdir) - assert get_data_dir() == os.pathsep.join([tmpdir, tmpdir, tmpdir]) + tmpdir = os.pathsep.join([str(tmp_path) for _ in range(3)]) + with proj_env(), patch.dict(os.environ, {"PROJ_LIB": tmpdir}, clear=True), patch( + "pyproj.datadir.Path.absolute", return_value=_INVALID_PATH + ), patch("pyproj.datadir.sys.prefix", str(_INVALID_PATH)): + create_projdb(tmp_path) + assert get_data_dir() == tmpdir def test_get_data_dir__from_prefix(tmp_path): - tmpdir = str(tmp_path) - with proj_env(), patch("pyproj.datadir.os") as os_mock, patch( - "pyproj.datadir.sys" - ) as sys_mock: - setup_os_mock(os_mock) - sys_mock.prefix = tmpdir - proj_dir = os.path.join(tmpdir, "share", "proj") - os.makedirs(proj_dir) + with proj_env(), patch.dict(os.environ, {}, clear=True), patch( + "pyproj.datadir.Path.absolute", return_value=_INVALID_PATH + ), patch("pyproj.datadir.sys.prefix", str(tmp_path)): + proj_dir = tmp_path / "share" / "proj" + proj_dir.mkdir(parents=True) create_projdb(proj_dir) - assert get_data_dir() == proj_dir + assert get_data_dir() == str(proj_dir) def test_get_data_dir__from_path(tmp_path): - tmpdir = str(tmp_path) - with proj_env(), patch("pyproj.datadir.os") as os_mock, patch( - "pyproj.datadir.sys" - ) as sys_mock, patch("pyproj.datadir.find_executable") as find_exe: - setup_os_mock(os_mock) - sys_mock.prefix = _INVALID_PATH - find_exe.return_value = os.path.join(tmpdir, "bin", "proj") - proj_dir = os.path.join(tmpdir, "share", "proj") - os.makedirs(proj_dir) + with proj_env(), patch.dict(os.environ, {}, clear=True), patch( + "pyproj.datadir.Path.absolute", return_value=_INVALID_PATH + ), patch("pyproj.datadir.sys.prefix", str(_INVALID_PATH)), patch( + "pyproj.datadir.find_executable", return_value=str(tmp_path / "bin" / "proj") + ): + proj_dir = tmp_path / "share" / "proj" + proj_dir.mkdir(parents=True) create_projdb(proj_dir) - assert get_data_dir() == proj_dir + assert get_data_dir() == str(proj_dir) -def test_append_data_dir__internal(tmp_path): - tmpdir = str(tmp_path) - with proj_env(), patch("pyproj.datadir.os") as os_mock: - setup_os_mock(os_mock, os.path.join(tmpdir, "randomfilename.py")) - create_projdb(tmpdir) - internal_proj_dir = os.path.join(tmpdir, "proj_dir", "share", "proj") - os.makedirs(internal_proj_dir) +@pytest.mark.parametrize("projdir_type", [str, Path]) +def test_append_data_dir__internal(projdir_type, tmp_path): + with proj_env(), patch.dict(os.environ, {}, clear=True), patch( + "pyproj.datadir.Path.absolute", return_value=tmp_path / "datadir.py" + ), patch("pyproj.datadir.sys.prefix", str(_INVALID_PATH)): + create_projdb(tmp_path) + internal_proj_dir = tmp_path / "proj_dir" / "share" / "proj" + internal_proj_dir.mkdir(parents=True) create_projdb(internal_proj_dir) - extra_datadir = str(os.path.join(tmpdir, "extra_datumgrids")) - append_data_dir(extra_datadir) - assert get_data_dir() == os.pathsep.join([internal_proj_dir, extra_datadir]) + extra_datadir = tmp_path / "extra_datumgrids" + append_data_dir(projdir_type(extra_datadir)) + assert get_data_dir() == os.pathsep.join( + [str(internal_proj_dir), str(extra_datadir)] + ) @pytest.mark.slow diff --git a/test/test_sync.py b/test/test_sync.py index 72964a960..68a8b2cee 100644 --- a/test/test_sync.py +++ b/test/test_sync.py @@ -83,8 +83,7 @@ def test_get_transform_grid_list__area_of_use(): def test_sha256sum(tmp_path): test_file = tmp_path / "test.file" - with open(test_file, "w") as testf: - testf.write("TEST") + test_file.write_text("TEST") assert ( _sha256sum(test_file) == "94ee059335e587e501cc4bf90613e0814f00a7b08bc7c648fd865a2af6a22cc2"