diff --git a/.travis.yml b/.travis.yml index 677fc51b0..4e2d515cb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -33,6 +33,9 @@ matrix: - python: 3.6 env: - PROJSYNC=ALL + - python: 3.6 + env: + - PYPROJ_GLOBAL_CONTEXT=ON - python: 3.7.2 env: - DOC=true diff --git a/docs/advanced_examples.rst b/docs/advanced_examples.rst index a03938381..a669fdc35 100644 --- a/docs/advanced_examples.rst +++ b/docs/advanced_examples.rst @@ -141,7 +141,7 @@ Multithreading -------------- The :class:`pyproj.transformer.Transformer` and :class:`pyproj.crs.CRS` -classes each have their own PROJ context. However, contexts cannot be +classes each have their own PROJ context by default. However, contexts cannot be shared across threads. As such, it is recommended to create the object within the thread that uses it. @@ -162,3 +162,21 @@ Here is a simple demonstration: with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor: for result in executor.map(transform_point, range(5)): print(result) + + +Optimizing Single-Threaded Applications +---------------------------------------- + +If you have a single-threaded application that generates many objects, +enabling the use of the global context can provide performance enhancements. + +For information about using the global context, see: :ref:`global_context` + +Here is an example where enabling the global context can help: + +.. code-block:: python + + import pyproj + + codes = pyproj.get_codes("EPSG", pyproj.enums.PJType.PROJECTED_CRS, False) + crs_list = [pyproj.CRS.from_epsg(code) for code in codes] diff --git a/docs/api/global_context.rst b/docs/api/global_context.rst new file mode 100644 index 000000000..b1be96ce9 --- /dev/null +++ b/docs/api/global_context.rst @@ -0,0 +1,33 @@ +.. _global_context: + +Global Context +============== + +If you have a single-threaded application that generates many objects, +enabling the use of the global context can provide performance enhancements. + +.. warning:: The global context is not thread safe. +.. warning:: The global context does not autoclose the database. + +How to enable: + +- Using :func:`pyproj.set_use_global_context`. +- Using the environment variable `PYPROJ_GLOBAL_CONTEXT`. + + +pyproj.set_use_global_context +----------------------------- + +.. autofunction:: pyproj.set_use_global_context + + +pyproj.set_global_context_network +----------------------------------- + +.. autofunction:: pyproj.set_global_context_network + + +pyproj.is_global_context_network_enabled +------------------------------------------ + +.. autofunction:: pyproj.is_global_context_network_enabled diff --git a/docs/api/index.rst b/docs/api/index.rst index 5a15955d8..4ff5f33fd 100644 --- a/docs/api/index.rst +++ b/docs/api/index.rst @@ -12,6 +12,7 @@ API Documentation list datadir sync + global_context enums exceptions show_versions diff --git a/docs/history.rst b/docs/history.rst index c37177ee3..d2fe2c801 100644 --- a/docs/history.rst +++ b/docs/history.rst @@ -18,6 +18,7 @@ Change Log * ENH: Add support for coordinate systems with CRS using CF conventions (issue #536) * ENH: Use `proj_is_equivalent_to_with_ctx` in the place of `proj_is_equivalent_to` internally (issue #666) * BUG: Add support for identifying engineering/parametric/temporal datums (issue #670) +* ENH: Added ability to use global context (issue #661) 2.6.1 ~~~~~ diff --git a/pyproj/__init__.py b/pyproj/__init__.py index 025976b36..0d7cd5d2b 100644 --- a/pyproj/__init__.py +++ b/pyproj/__init__.py @@ -46,7 +46,12 @@ ] import warnings -from pyproj import _datadir +from pyproj._datadir import ( # noqa: F401 + _pyproj_global_context_initialize, + is_global_context_network_enabled, + set_global_context_network, + set_use_global_context, +) from pyproj._list import ( # noqa: F401 get_authorities, get_codes, @@ -71,6 +76,6 @@ try: - _datadir.pyproj_global_context_initialize() + _pyproj_global_context_initialize() except DataDirError as err: warnings.warn(str(err)) diff --git a/pyproj/_crs.pyx b/pyproj/_crs.pyx index ed959d7a0..968dc7361 100644 --- a/pyproj/_crs.pyx +++ b/pyproj/_crs.pyx @@ -3,7 +3,7 @@ import re import warnings from collections import OrderedDict -from pyproj._datadir cimport pyproj_context_initialize +from pyproj._datadir cimport pyproj_context_create from pyproj.compat import cstrencode, pystrdecode from pyproj.crs.datum import CustomEllipsoid from pyproj.crs.enums import CoordinateOperationType, DatumType @@ -169,8 +169,7 @@ cdef _get_concatenated_operations(PJ_CONTEXT* context, PJ* concatenated_operatio cdef int iii = 0 operations = [] for iii in range(step_count): - sub_context = proj_context_create() - pyproj_context_initialize(sub_context, True, network=network_enabled) + sub_context = pyproj_context_create(network=network_enabled) operation = proj_concatoperation_get_step( sub_context, concatenated_operation, @@ -652,8 +651,7 @@ cdef class CoordinateSystem(_CRSParts): ------- CoordinateSystem """ - cdef PJ_CONTEXT* context = proj_context_create() - pyproj_context_initialize(context, True) + cdef PJ_CONTEXT* context = pyproj_context_create() cdef PJ* coordinate_system_pj = proj_create( context, cstrencode(coordinate_system_string) @@ -861,8 +859,7 @@ cdef class Ellipsoid(_CRSParts): ------- Ellipsoid """ - cdef PJ_CONTEXT* context = proj_context_create() - pyproj_context_initialize(context, True) + cdef PJ_CONTEXT* context = pyproj_context_create() cdef PJ* ellipsoid_pj = proj_create_from_database( context, cstrencode(auth_name), @@ -916,8 +913,7 @@ cdef class Ellipsoid(_CRSParts): ------- Ellipsoid """ - cdef PJ_CONTEXT* context = proj_context_create() - pyproj_context_initialize(context, True) + cdef PJ_CONTEXT* context = pyproj_context_create() cdef PJ* ellipsoid_pj = proj_create( context, cstrencode(ellipsoid_string) @@ -1020,8 +1016,7 @@ cdef class Ellipsoid(_CRSParts): ------- Ellipsoid """ - cdef PJ_CONTEXT* context = proj_context_create() - pyproj_context_initialize(context, True) + cdef PJ_CONTEXT* context = pyproj_context_create() cdef PJ* ellipsoid_pj = _from_name( context, ellipsoid_name, @@ -1142,9 +1137,7 @@ cdef class PrimeMeridian(_CRSParts): ------- PrimeMeridian """ - cdef PJ_CONTEXT* context = proj_context_create() - pyproj_context_initialize(context, True) - + cdef PJ_CONTEXT* context = pyproj_context_create() cdef PJ* prime_meridian_pj = proj_create_from_database( context, cstrencode(auth_name), @@ -1198,8 +1191,7 @@ cdef class PrimeMeridian(_CRSParts): ------- PrimeMeridian """ - cdef PJ_CONTEXT* context = proj_context_create() - pyproj_context_initialize(context, True) + cdef PJ_CONTEXT* context = pyproj_context_create() cdef PJ* prime_meridian_pj = proj_create( context, cstrencode(prime_meridian_string) @@ -1308,8 +1300,7 @@ cdef class PrimeMeridian(_CRSParts): ------- PrimeMeridian """ - cdef PJ_CONTEXT* context = proj_context_create() - pyproj_context_initialize(context, True) + cdef PJ_CONTEXT* context = pyproj_context_create() cdef PJ* prime_meridian_pj = _from_name( context, prime_meridian_name, @@ -1405,8 +1396,7 @@ cdef class Datum(_CRSParts): ------- Datum """ - cdef PJ_CONTEXT* context = proj_context_create() - pyproj_context_initialize(context, True) + cdef PJ_CONTEXT* context = pyproj_context_create() cdef PJ* datum_pj = proj_create_from_database( context, @@ -1460,8 +1450,7 @@ cdef class Datum(_CRSParts): ------- Datum """ - cdef PJ_CONTEXT* context = proj_context_create() - pyproj_context_initialize(context, True) + cdef PJ_CONTEXT* context = pyproj_context_create() cdef PJ* datum_pj = proj_create( context, cstrencode(datum_string) @@ -1532,8 +1521,7 @@ cdef class Datum(_CRSParts): Datum """ pj_datum_type = _PJ_DATUM_TYPE_MAP[datum_type] - cdef PJ_CONTEXT* context = proj_context_create() - pyproj_context_initialize(context, True) + cdef PJ_CONTEXT* context = pyproj_context_create() cdef PJ* datum_pj = _from_name( context, datum_name, @@ -1643,8 +1631,7 @@ cdef class Datum(_CRSParts): """ if self._ellipsoid is not None: return None if self._ellipsoid is False else self._ellipsoid - cdef PJ_CONTEXT* context = proj_context_create() - pyproj_context_initialize(context, True) + cdef PJ_CONTEXT* context = pyproj_context_create() cdef PJ* ellipsoid_pj = proj_get_ellipsoid( context, self.projobj, @@ -1667,8 +1654,7 @@ cdef class Datum(_CRSParts): """ if self._prime_meridian is not None: return None if self._prime_meridian is False else self._prime_meridian - cdef PJ_CONTEXT* context = proj_context_create() - pyproj_context_initialize(context, True) + cdef PJ_CONTEXT* context = pyproj_context_create() cdef PJ* prime_meridian_pj = proj_get_prime_meridian( context, self.projobj, @@ -1971,9 +1957,7 @@ cdef class CoordinateOperation(_CRSParts): ------- CoordinateOperation """ - cdef PJ_CONTEXT* context = proj_context_create() - pyproj_context_initialize(context, True) - + cdef PJ_CONTEXT* context = pyproj_context_create() cdef PJ* coord_operation_pj = proj_create_from_database( context, cstrencode(auth_name), @@ -2026,8 +2010,7 @@ cdef class CoordinateOperation(_CRSParts): ------- CoordinateOperation """ - cdef PJ_CONTEXT* context = proj_context_create() - pyproj_context_initialize(context, True) + cdef PJ_CONTEXT* context = pyproj_context_create() cdef PJ* coord_operation_pj = proj_create( context, cstrencode(coordinate_operation_string) @@ -2147,8 +2130,7 @@ cdef class CoordinateOperation(_CRSParts): pj_coordinate_operation_type = _PJ_COORDINATE_OPERATION_TYPE_MAP[ CoordinateOperationType.create(coordinate_operation_type) ] - cdef PJ_CONTEXT* context = proj_context_create() - pyproj_context_initialize(context, True) + cdef PJ_CONTEXT* context = pyproj_context_create() cdef PJ* coordinate_operation_pj = _from_name( context, coordinate_operation_name, @@ -2341,8 +2323,7 @@ cdef class _CRS(Base): self.type_name = "undefined" def __init__(self, proj_string): - self.context = proj_context_create() - pyproj_context_initialize(self.context, False) + self.context = pyproj_context_create() # initialize projection self.projobj = proj_create( self.context, @@ -2407,8 +2388,7 @@ cdef class _CRS(Base): """ if self._ellipsoid is not None: return None if self._ellipsoid is False else self._ellipsoid - cdef PJ_CONTEXT* context = proj_context_create() - pyproj_context_initialize(context, True) + cdef PJ_CONTEXT* context = pyproj_context_create() cdef PJ* ellipsoid_pj = proj_get_ellipsoid( context, self.projobj @@ -2433,8 +2413,7 @@ cdef class _CRS(Base): """ if self._prime_meridian is not None: return None if self._prime_meridian is True else self._prime_meridian - cdef PJ_CONTEXT* context = proj_context_create() - pyproj_context_initialize(context, True) + cdef PJ_CONTEXT* context = pyproj_context_create() cdef PJ* prime_meridian_pj = proj_get_prime_meridian( context, self.projobj, @@ -2458,8 +2437,7 @@ cdef class _CRS(Base): """ if self._datum is not None: return None if self._datum is False else self._datum - cdef PJ_CONTEXT* context = proj_context_create() - pyproj_context_initialize(context, True) + cdef PJ_CONTEXT* context = pyproj_context_create() cdef PJ* datum_pj = proj_crs_get_datum( context, self.projobj, @@ -2488,8 +2466,7 @@ cdef class _CRS(Base): """ if self._coordinate_system is not None: return None if self._coordinate_system is False else self._coordinate_system - cdef PJ_CONTEXT* context = proj_context_create() - pyproj_context_initialize(context, True) + cdef PJ_CONTEXT* context = pyproj_context_create() cdef PJ* coord_system_pj = proj_crs_get_coordinate_system( context, self.projobj @@ -2521,8 +2498,7 @@ cdef class _CRS(Base): if self._coordinate_operation is False else self._coordinate_operation ) - cdef PJ_CONTEXT* context = proj_context_create() - pyproj_context_initialize(context, True) + cdef PJ_CONTEXT* context = pyproj_context_create() cdef PJ* coord_pj = proj_crs_get_coordoperation( context, self.projobj diff --git a/pyproj/_datadir.pxd b/pyproj/_datadir.pxd index 333cb1fba..1a7332429 100644 --- a/pyproj/_datadir.pxd +++ b/pyproj/_datadir.pxd @@ -1,6 +1,4 @@ include "proj.pxi" -cdef void pyproj_context_initialize( - PJ_CONTEXT* context, - bint free_context_on_error, - network=*) except * + +cdef PJ_CONTEXT* pyproj_context_create(network=*) except * diff --git a/pyproj/_datadir.pyi b/pyproj/_datadir.pyi index adce2aa8a..3ad8ae4c8 100644 --- a/pyproj/_datadir.pyi +++ b/pyproj/_datadir.pyi @@ -1,2 +1,8 @@ -def pyproj_global_context_initialize() -> None: ... +from typing import Optional + +def _pyproj_global_context_initialize() -> None: ... def get_user_data_dir(create: bool = False) -> str: ... +def _global_context_set_data_dir() -> None: ... +def set_use_global_context(active: Optional[bool] = None) -> None: ... +def set_global_context_network(active: Optional[bool] = None) -> None: ... +def is_global_context_network_enabled() -> bool: ... diff --git a/pyproj/_datadir.pyx b/pyproj/_datadir.pyx index b1276f158..39ee3a6d3 100644 --- a/pyproj/_datadir.pyx +++ b/pyproj/_datadir.pyx @@ -1,12 +1,112 @@ import os import warnings +from distutils.util import strtobool from libc.stdlib cimport malloc, free +from typing import Optional from pyproj.compat import cstrencode, pystrdecode -from pyproj.datadir import get_data_dir from pyproj.exceptions import ProjError, DataDirError +# default to False is the safest mode +# as it supports multithreading +_USE_GLOBAL_CONTEXT = strtobool(os.environ.get("PYPROJ_GLOBAL_CONTEXT", "OFF")) + + +def set_use_global_context(active: Optional[bool] = None) -> None: + """ + .. versionadded:: 3.0.0 + + Activates the usage of the global context. Using this + option can enhance the performance of initializing objects + in single-threaded applications. + + .. warning:: The global context is not thread safe. + .. warning:: The global context does not autoclose the database. + + .. note:: You can change the network settings with + :func:`pyproj.set_global_context_network`. + + Parameters + ---------- + active: bool, optional + If True, it activates the use of the global context. If False, + the use of the global context is deactivated. If None, it uses + the environment variable PYPROJ_GLOBAL_CONTEXT and defaults + to False if it is not found. The default is None. + """ + global _USE_GLOBAL_CONTEXT + if active is None: + active = strtobool(os.environ.get("PYPROJ_GLOBAL_CONTEXT", "OFF")) + _USE_GLOBAL_CONTEXT = bool(active) + proj_context_set_autoclose_database(NULL, not _USE_GLOBAL_CONTEXT) + + +def set_global_context_network(active: Optional[bool] = None) -> None: + """ + .. versionadded:: 3.0.0 + + Manages whether PROJ network is enabled on the global context. + + .. note:: You can activate the global context with + :func:`pyproj.set_use_global_context` or with + the PYPROJ_GLOBAL_CONTEXT environment variable. + + Parameters + ---------- + active: bool, optional + Default is None, which uses the system defaults for networking. + If True, it will force the use of network for grids regardless of + any other network setting. If False, it will force disable use of + network for grids regardless of any other network setting. + """ + if active is None: + # in the case of the global context, need to reset network + # setting based on the environment variable every time if None + # because it could have been changed by the user previously + active = strtobool(os.environ.get("PROJ_NETWORK", "OFF")) + pyproj_context_set_enable_network(NULL, bool(active)) + + +def is_global_context_network_enabled() -> bool: + """ + .. versionadded:: 3.0.0 + + .. note:: You can activate the global context with + :func:`pyproj.set_use_global_context` or with + the PYPROJ_GLOBAL_CONTEXT environment variable. + + bool: + If the network is enabled on the global context. + """ + return proj_context_is_network_enabled(NULL) == 1 + + +def get_user_data_dir(bint create: bool = False) -> str: + """ + .. versionadded:: 3.0.0 + + Get the PROJ user writable directory for datumgrid files. + + This is where grids will be downloaded when + `PROJ network `__ capabilities + are enabled. It is also the default download location for the + `projsync `__ command line program. + + Parameters + ---------- + create: bool, optional + If True, it will create the directory if it does not already exist. + Default is False. + + Returns + ------- + str: + The user writable data directory. + """ + return pystrdecode(proj_context_get_user_writable_directory(NULL, create)) + + cdef void pyproj_log_function(void *user_data, int level, const char *error_msg): """ Log function for catching PROJ errors. @@ -19,6 +119,8 @@ cdef void set_context_data_dir(PJ_CONTEXT* context) except *: """ Setup the data directory for the context for pyproj """ + from pyproj.datadir import get_data_dir + data_dir_list = get_data_dir().split(os.pathsep) # the first path will always have the database b_database_path = cstrencode(os.path.join(data_dir_list[0], "proj.db")) @@ -38,45 +140,64 @@ cdef void set_context_data_dir(PJ_CONTEXT* context) except *: free(c_data_dir) +cdef void pyproj_context_set_enable_network( + PJ_CONTEXT* context, + network=None, +) except *: + if network is not None: + enabled = proj_context_set_enable_network(context, bool(network)) + if network and not enabled: + warnings.warn("PROJ network cannot be enabled.") + + cdef void pyproj_context_initialize( PJ_CONTEXT* context, - bint free_context_on_error, network=None, + bint autoclose_database=True, ) except *: """ Setup the context for pyproj """ proj_log_func(context, NULL, pyproj_log_function) proj_context_use_proj4_init_rules(context, 1) - proj_context_set_autoclose_database(context, 1) - if network is not None: - enabled = proj_context_set_enable_network(context, bool(network)) - if network and not enabled: - warnings.warn("PROJ network cannot be enabled.") + if autoclose_database: + proj_context_set_autoclose_database(context, 1) + pyproj_context_set_enable_network(context, network=network) + set_context_data_dir(context) + + +cdef PJ_CONTEXT* pyproj_context_create( + network=None, +): + """ + Create and initialize the context(s) for pyproj. + This also manages whether the global context is used. + """ + global _USE_GLOBAL_CONTEXT + if _USE_GLOBAL_CONTEXT: + return NULL + cdef PJ_CONTEXT* context = proj_context_create() try: - set_context_data_dir(context) + pyproj_context_initialize( + context, + network=network, + ) except DataDirError: - if free_context_on_error and context != NULL: + if context != NULL: proj_context_destroy(context) + context = NULL raise + return context -def pyproj_global_context_initialize(): - proj_log_func(NULL, NULL, pyproj_log_function) - set_context_data_dir(NULL) - +def _pyproj_global_context_initialize(): + global _USE_GLOBAL_CONTEXT + pyproj_context_initialize( + NULL, + network=None, + autoclose_database=not _USE_GLOBAL_CONTEXT + ) -def get_user_data_dir(bint create: bool = False) -> str: - """ - Parameters - ---------- - create: bool, optional - If True, it will create the directory if it does not already exist. - Default is False. - Returns - ------- - str: - The user writable data directory. - """ - return pystrdecode(proj_context_get_user_writable_directory(NULL, create)) +def _global_context_set_data_dir(): + set_context_data_dir(NULL) diff --git a/pyproj/_list.pyx b/pyproj/_list.pyx index 189a7076f..4aa06a5d7 100644 --- a/pyproj/_list.pyx +++ b/pyproj/_list.pyx @@ -6,7 +6,7 @@ import warnings from pyproj.compat import cstrencode, pystrdecode from pyproj.enums import PJType -from pyproj._datadir cimport pyproj_context_initialize +from pyproj._datadir cimport pyproj_context_create def get_proj_operations_map(): @@ -74,8 +74,7 @@ def get_authorities(): List[str]: Authorities in PROJ database. """ - cdef PJ_CONTEXT* context = proj_context_create() - pyproj_context_initialize(context, True) + cdef PJ_CONTEXT* context = pyproj_context_create() cdef PROJ_STRING_LIST proj_auth_list = proj_get_authorities_from_database(context) if proj_auth_list == NULL: proj_context_destroy(context) @@ -143,8 +142,7 @@ def get_codes(auth_name, pj_type, allow_deprecated=False): cdef PJ_TYPE cpj_type = PJ_TYPE_MAP[PJType.create(pj_type)] cdef PROJ_STRING_LIST proj_code_list = NULL try: - context = proj_context_create() - pyproj_context_initialize(context, True) + context = pyproj_context_create() proj_code_list = proj_get_codes_from_database( context, cstrencode(auth_name), @@ -231,8 +229,7 @@ def get_units_map(auth_name=None, category=None, allow_deprecated=False): c_category = category cdef int num_units = 0 - cdef PJ_CONTEXT* context = proj_context_create() - pyproj_context_initialize(context, True) + cdef PJ_CONTEXT* context = pyproj_context_create() cdef PROJ_UNIT_INFO** db_unit_list = proj_get_units_from_database( context, c_auth_name, diff --git a/pyproj/_transformer.pyx b/pyproj/_transformer.pyx index 905702f05..50a69438f 100644 --- a/pyproj/_transformer.pyx +++ b/pyproj/_transformer.pyx @@ -14,7 +14,7 @@ from pyproj._crs cimport ( CoordinateOperation, _get_concatenated_operations, ) -from pyproj._datadir cimport pyproj_context_initialize +from pyproj._datadir cimport pyproj_context_create from pyproj.compat import cstrencode, pystrdecode from pyproj.enums import ProjVersion, TransformDirection from pyproj.exceptions import ProjError @@ -144,8 +144,7 @@ cdef class _TransformerGroup: area of use of the CRS), and by increasing accuracy. Operations with unknown accuracy are sorted last, whatever their area. """ - self.context = proj_context_create() - pyproj_context_initialize(self.context, False, network=network) + self.context = pyproj_context_create(network=network) cdef PJ_OPERATION_FACTORY_CONTEXT* operation_factory_context = NULL cdef PJ_OBJ_LIST * pj_operations = NULL cdef PJ* pj_transform = NULL @@ -195,8 +194,7 @@ cdef class _TransformerGroup: ) num_operations = proj_list_get_count(pj_operations) for iii in range(num_operations): - context = proj_context_create() - pyproj_context_initialize(context, True, network=network) + context = pyproj_context_create(network=network) pj_transform = proj_list_get( context, pj_operations, @@ -358,8 +356,7 @@ cdef class _Transformer(Base): east_lon_degree, north_lat_degree, ) - transformer.context = proj_context_create() - pyproj_context_initialize(transformer.context, False, network=network) + transformer.context = pyproj_context_create(network=network) transformer.projobj = proj_create_crs_to_crs( transformer.context, cstrencode(crs_from.srs), @@ -414,8 +411,7 @@ cdef class _Transformer(Base): Create Transformer from a PROJ pipeline string. """ cdef _Transformer transformer = _Transformer() - transformer.context = proj_context_create() - pyproj_context_initialize(transformer.context, False, network=network) + transformer.context = pyproj_context_create(network=network) # initialize projection transformer.projobj = proj_create( transformer.context, diff --git a/pyproj/datadir.py b/pyproj/datadir.py index 5a23d572d..43317ac7d 100644 --- a/pyproj/datadir.py +++ b/pyproj/datadir.py @@ -5,6 +5,10 @@ import sys from distutils.spawn import find_executable +from pyproj._datadir import ( # noqa: F401 + _global_context_set_data_dir, + get_user_data_dir, +) from pyproj.exceptions import DataDirError _USER_PROJ_DATA = None @@ -28,9 +32,7 @@ def set_data_dir(proj_data_dir: str) -> None: # need to reset the global PROJ context # to prevent core dumping if the data directory # is not found. - from pyproj._datadir import pyproj_global_context_initialize - - pyproj_global_context_initialize() + _global_context_set_data_dir() def append_data_dir(proj_data_dir: str) -> None: @@ -111,30 +113,3 @@ def valid_data_dirs(potential_data_dirs): "with `pyproj.datadir.set_data_dir`." ) return _VALIDATED_PROJ_DATA - - -def get_user_data_dir(create: bool = False) -> str: - """ - .. versionadded:: 7.1.0 - - Get the PROJ user writable directory for datumgrid files. - - This is where grids will be downloaded when - `PROJ network `__ capabilities - are enabled. It is also the default download location for the - `projsync `__ command line program. - - Parameters - ---------- - create: bool, optional - If True, it will create the directory if it does not already exist. - Default is False. - - Returns - ------- - str: - The user writable data directory. - """ - from pyproj import _datadir - - return _datadir.get_user_data_dir(create=create) diff --git a/test/conftest.py b/test/conftest.py index 062649375..b1c1be3e3 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -3,7 +3,7 @@ from pathlib import Path import pyproj -from pyproj.datadir import get_data_dir, get_user_data_dir +from pyproj.datadir import get_data_dir, get_user_data_dir, set_data_dir def unset_data_dir(): @@ -11,6 +11,21 @@ def unset_data_dir(): pyproj.datadir._VALIDATED_PROJ_DATA = None +@contextmanager +def proj_network_env(): + """ + Ensure global context network settings reset + """ + if not pyproj._datadir._USE_GLOBAL_CONTEXT: + yield + else: + network = pyproj.is_global_context_network_enabled() + try: + yield + finally: + pyproj.set_global_context_network(network) + + @contextmanager def proj_env(): """ @@ -22,6 +37,8 @@ def proj_env(): finally: # make sure the data dir is cleared unset_data_dir() + # reset back to the original path + set_data_dir(get_data_dir()) @contextmanager diff --git a/test/test_datadir.py b/test/test_datadir.py index a86be25d5..86ef1642c 100644 --- a/test/test_datadir.py +++ b/test/test_datadir.py @@ -1,10 +1,12 @@ import os +from contextlib import contextmanager import pytest from mock import patch -from pyproj import CRS -from pyproj._datadir import pyproj_global_context_initialize +import pyproj._datadir +from pyproj import CRS, set_use_global_context +from pyproj._datadir import _pyproj_global_context_initialize from pyproj.datadir import ( DataDirError, append_data_dir, @@ -15,6 +17,18 @@ from test.conftest import proj_env +@contextmanager +def proj_context_env(): + """ + Ensure setting for global context is the same at the end. + """ + context = pyproj._datadir._USE_GLOBAL_CONTEXT + try: + yield + finally: + pyproj._datadir._USE_GLOBAL_CONTEXT = context + + def create_projdb(tmpdir): with open(os.path.join(tmpdir, "proj.db"), "w") as pjdb: pjdb.write("DUMMY proj.db") @@ -46,9 +60,9 @@ def test_get_data_dir__missing(): def test_pyproj_global_context_initialize__datadir_missing(): with proj_env(), pytest.raises(DataDirError), patch( - "pyproj._datadir.get_data_dir", side_effect=DataDirError("test") + "pyproj.datadir.get_data_dir", side_effect=DataDirError("test") ): - pyproj_global_context_initialize() + _pyproj_global_context_initialize() def test_get_data_dir__from_user(tmp_path): @@ -168,3 +182,38 @@ def test_creating_multiple_crs_without_file_limit(): def test_get_user_data_dir(): assert get_user_data_dir().endswith("proj") + + +@patch.dict("os.environ", {"PYPROJ_GLOBAL_CONTEXT": "ON"}, clear=True) +def test_set_use_global_context__default_on(): + with proj_context_env(): + set_use_global_context() + assert pyproj._datadir._USE_GLOBAL_CONTEXT is True + + +@patch.dict("os.environ", {"PYPROJ_GLOBAL_CONTEXT": "OFF"}, clear=True) +def test_set_use_global_context__default_off(): + with proj_context_env(): + set_use_global_context() + assert pyproj._datadir._USE_GLOBAL_CONTEXT is False + + +@patch.dict("os.environ", {}, clear=True) +def test_set_use_global_context__default(): + with proj_context_env(): + set_use_global_context() + assert pyproj._datadir._USE_GLOBAL_CONTEXT is False + + +@patch.dict("os.environ", {"PYPROJ_GLOBAL_CONTEXT": "OFF"}, clear=True) +def test_set_use_global_context__on(): + with proj_context_env(): + set_use_global_context(True) + assert pyproj._datadir._USE_GLOBAL_CONTEXT is True + + +@patch.dict("os.environ", {"PYPROJ_GLOBAL_CONTEXT": "ON"}, clear=True) +def test_set_use_global_context__off(): + with proj_context_env(): + set_use_global_context(False) + assert pyproj._datadir._USE_GLOBAL_CONTEXT is False diff --git a/test/test_doctest_wrapper.py b/test/test_doctest_wrapper.py index f62dfe86b..b25b08e3f 100644 --- a/test/test_doctest_wrapper.py +++ b/test/test_doctest_wrapper.py @@ -8,13 +8,16 @@ from mock import patch import pyproj +from test.conftest import proj_network_env @patch.dict("os.environ", {"PROJ_NETWORK": "ON"}, clear=True) def test_doctests(): """run the examples in the docstrings using the doctest module""" - with warnings.catch_warnings(): + with warnings.catch_warnings(), proj_network_env(): + if pyproj._datadir._USE_GLOBAL_CONTEXT: + pyproj.set_global_context_network(active=True) warnings.filterwarnings( "ignore", "You will likely lose important projection information when", diff --git a/test/test_proj.py b/test/test_proj.py index a56ef4c12..eb901989d 100644 --- a/test/test_proj.py +++ b/test/test_proj.py @@ -9,8 +9,10 @@ from mock import patch from numpy.testing import assert_almost_equal +import pyproj from pyproj import Geod, Proj, pj_ellps, pj_list, transform from pyproj.exceptions import CRSError, ProjError +from test.conftest import proj_network_env class BasicTest(unittest.TestCase): @@ -533,19 +535,30 @@ def test_numpy_bool_kwarg_true(): @patch.dict("os.environ", {"PROJ_NETWORK": "ON"}, clear=True) def test_network__disable(): - transformer = Proj(3857, network=False) - assert transformer.is_network_enabled is False + with proj_network_env(): + if pyproj._datadir._USE_GLOBAL_CONTEXT: + pyproj.set_global_context_network(active=False) + transformer = Proj(3857, network=False) + assert transformer.is_network_enabled is False @patch.dict("os.environ", {"PROJ_NETWORK": "OFF"}, clear=True) def test_network__enable(): - transformer = Proj(3857, network=True) - assert transformer.is_network_enabled is True + with proj_network_env(): + if pyproj._datadir._USE_GLOBAL_CONTEXT: + pyproj.set_global_context_network(active=True) + transformer = Proj(3857, network=True) + assert transformer.is_network_enabled is True def test_network__default(): - transformer = Proj(3857) - assert transformer.is_network_enabled == (os.environ.get("PROJ_NETWORK") == "ON") + with proj_network_env(): + if pyproj._datadir._USE_GLOBAL_CONTEXT: + pyproj.set_global_context_network() + transformer = Proj(3857) + assert transformer.is_network_enabled == ( + os.environ.get("PROJ_NETWORK") == "ON" + ) def test_radians(): diff --git a/test/test_transformer.py b/test/test_transformer.py index 1dc55cf49..7decd83a0 100644 --- a/test/test_transformer.py +++ b/test/test_transformer.py @@ -15,7 +15,7 @@ from pyproj.enums import TransformDirection from pyproj.exceptions import ProjError from pyproj.transformer import AreaOfInterest, TransformerGroup -from test.conftest import grids_available, proj_env +from test.conftest import grids_available, proj_env, proj_network_env def test_tranform_wgs84_to_custom(): @@ -791,8 +791,11 @@ def test_pipeline_itransform(pipeline_str): ) @patch.dict("os.environ", {"PROJ_NETWORK": "ON"}, clear=True) def test_network__disable(transformer): - trans = transformer(network=False) - assert trans.is_network_enabled is False + with proj_network_env(): + if pyproj._datadir._USE_GLOBAL_CONTEXT: + pyproj.set_global_context_network(active=False) + trans = transformer(network=False) + assert trans.is_network_enabled is False @pytest.mark.parametrize( @@ -807,8 +810,11 @@ def test_network__disable(transformer): ) @patch.dict("os.environ", {"PROJ_NETWORK": "OFF"}, clear=True) def test_network__enable(transformer): - trans = transformer(network=True) - assert trans.is_network_enabled is True + with proj_network_env(): + if pyproj._datadir._USE_GLOBAL_CONTEXT: + pyproj.set_global_context_network(active=True) + trans = transformer(network=True) + assert trans.is_network_enabled is True @pytest.mark.parametrize( @@ -822,63 +828,75 @@ def test_network__enable(transformer): ], ) def test_network__default(transformer): - trans = transformer() - assert trans.is_network_enabled == (os.environ.get("PROJ_NETWORK") == "ON") + with proj_network_env(): + if pyproj._datadir._USE_GLOBAL_CONTEXT: + pyproj.set_global_context_network() + trans = transformer() + assert trans.is_network_enabled == (os.environ.get("PROJ_NETWORK") == "ON") @patch.dict("os.environ", {"PROJ_NETWORK": "OFF"}, clear=True) def test_transformer_group__network_enabled(): - trans_group = TransformerGroup(4326, 2964, network=True) - assert len(trans_group.unavailable_operations) == 0 - assert len(trans_group.transformers) == 10 - assert trans_group.best_available - for transformer in trans_group.transformers: - assert transformer.is_network_enabled is True - for operation in transformer.operations: - for grid in operation.grids: - assert grid.available + with proj_network_env(): + if pyproj._datadir._USE_GLOBAL_CONTEXT: + pyproj.set_global_context_network(active=True) + trans_group = TransformerGroup(4326, 2964, network=True) + assert len(trans_group.unavailable_operations) == 0 + assert len(trans_group.transformers) == 10 + assert trans_group.best_available + for transformer in trans_group.transformers: + assert transformer.is_network_enabled is True + for operation in transformer.operations: + for grid in operation.grids: + assert grid.available @patch.dict("os.environ", {"PROJ_NETWORK": "ON"}, clear=True) def test_transformer_group__network_disabled(): - trans_group = TransformerGroup(4326, 2964, network=False) - for transformer in trans_group.transformers: - assert transformer.is_network_enabled is False - - if grids_available( - "us_noaa_alaska.tif", "ca_nrc_ntv2_0.tif", check_network=False, check_all=True - ): - assert len(trans_group.unavailable_operations) == 0 - assert len(trans_group.transformers) == 10 - assert ( - trans_group.transformers[0].description - == "Inverse of NAD27 to WGS 84 (85) + Alaska Albers" - ) - assert trans_group.best_available - elif grids_available("us_noaa_alaska.tif", check_network=False): - assert len(trans_group.unavailable_operations) == 1 - assert ( - trans_group.transformers[0].description - == "Inverse of NAD27 to WGS 84 (85) + Alaska Albers" - ) - assert len(trans_group.transformers) == 9 - assert trans_group.best_available - elif grids_available("ca_nrc_ntv2_0.tif", check_network=False): - assert len(trans_group.unavailable_operations) == 1 - assert ( - trans_group.transformers[0].description - == "Inverse of NAD27 to WGS 84 (7) + Alaska Albers" - ) - assert len(trans_group.transformers) == 9 - assert not trans_group.best_available - else: - assert len(trans_group.unavailable_operations) == 2 - assert ( - trans_group.unavailable_operations[0].name - == "Inverse of NAD27 to WGS 84 (85) + Alaska Albers" - ) - assert len(trans_group.transformers) == 8 - assert not trans_group.best_available + with proj_network_env(): + if pyproj._datadir._USE_GLOBAL_CONTEXT: + pyproj.set_global_context_network(active=False) + trans_group = TransformerGroup(4326, 2964, network=False) + for transformer in trans_group.transformers: + assert transformer.is_network_enabled is False + + if grids_available( + "us_noaa_alaska.tif", + "ca_nrc_ntv2_0.tif", + check_network=False, + check_all=True, + ): + assert len(trans_group.unavailable_operations) == 0 + assert len(trans_group.transformers) == 10 + assert ( + trans_group.transformers[0].description + == "Inverse of NAD27 to WGS 84 (85) + Alaska Albers" + ) + assert trans_group.best_available + elif grids_available("us_noaa_alaska.tif", check_network=False): + assert len(trans_group.unavailable_operations) == 1 + assert ( + trans_group.transformers[0].description + == "Inverse of NAD27 to WGS 84 (85) + Alaska Albers" + ) + assert len(trans_group.transformers) == 9 + assert trans_group.best_available + elif grids_available("ca_nrc_ntv2_0.tif", check_network=False): + assert len(trans_group.unavailable_operations) == 1 + assert ( + trans_group.transformers[0].description + == "Inverse of NAD27 to WGS 84 (7) + Alaska Albers" + ) + assert len(trans_group.transformers) == 9 + assert not trans_group.best_available + else: + assert len(trans_group.unavailable_operations) == 2 + assert ( + trans_group.unavailable_operations[0].name + == "Inverse of NAD27 to WGS 84 (85) + Alaska Albers" + ) + assert len(trans_group.transformers) == 8 + assert not trans_group.best_available def test_transform_pipeline_radians(): @@ -937,39 +955,47 @@ def test_transform_honours_input_types(x, y, z): @patch("pyproj.transformer.get_user_data_dir") def test_transformer_group__download_grids(get_user_data_dir_mock, tmp_path, capsys): get_user_data_dir_mock.return_value = str(tmp_path) - trans_group = TransformerGroup(4326, 2964, network=False) - trans_group.download_grids(verbose=True) - captured = capsys.readouterr() - get_user_data_dir_mock.assert_called_with(True) - paths = sorted(Path(path).name for path in glob(str(tmp_path.joinpath("*")))) - if grids_available( - "us_noaa_alaska.tif", "ca_nrc_ntv2_0.tif", check_network=False, check_all=True - ): - assert paths == [] - assert captured.out == "" - elif grids_available("us_noaa_alaska.tif", check_network=False): - assert paths == ["ca_nrc_ntv2_0.tif"] - assert captured.out == "Downloading: https://cdn.proj.org/ca_nrc_ntv2_0.tif\n" - elif grids_available("ca_nrc_ntv2_0.tif", check_network=False): - assert paths == ["us_noaa_alaska.tif"] - assert captured.out == ( - "Downloading: https://cdn.proj.org/us_noaa_alaska.tif\n" - ) - else: - assert paths == ["ca_nrc_ntv2_0.tif", "us_noaa_alaska.tif"] - assert captured.out == ( - "Downloading: https://cdn.proj.org/us_noaa_alaska.tif\n" - "Downloading: https://cdn.proj.org/ca_nrc_ntv2_0.tif\n" - ) - # make sure not downloaded again - with proj_env(), patch( - "pyproj.transformer._download_resource_file" - ) as download_mock: - append_data_dir(str(tmp_path)) + with proj_network_env(): + if pyproj._datadir._USE_GLOBAL_CONTEXT: + pyproj.set_global_context_network(active=False) trans_group = TransformerGroup(4326, 2964, network=False) - trans_group.download_grids() + trans_group.download_grids(verbose=True) + captured = capsys.readouterr() get_user_data_dir_mock.assert_called_with(True) - download_mock.assert_not_called() + paths = sorted(Path(path).name for path in glob(str(tmp_path.joinpath("*")))) + if grids_available( + "us_noaa_alaska.tif", + "ca_nrc_ntv2_0.tif", + check_network=False, + check_all=True, + ): + assert paths == [] + assert captured.out == "" + elif grids_available("us_noaa_alaska.tif", check_network=False): + assert paths == ["ca_nrc_ntv2_0.tif"] + assert ( + captured.out == "Downloading: https://cdn.proj.org/ca_nrc_ntv2_0.tif\n" + ) + elif grids_available("ca_nrc_ntv2_0.tif", check_network=False): + assert paths == ["us_noaa_alaska.tif"] + assert captured.out == ( + "Downloading: https://cdn.proj.org/us_noaa_alaska.tif\n" + ) + else: + assert paths == ["ca_nrc_ntv2_0.tif", "us_noaa_alaska.tif"] + assert captured.out == ( + "Downloading: https://cdn.proj.org/us_noaa_alaska.tif\n" + "Downloading: https://cdn.proj.org/ca_nrc_ntv2_0.tif\n" + ) + # make sure not downloaded again + with proj_env(), patch( + "pyproj.transformer._download_resource_file" + ) as download_mock: + append_data_dir(str(tmp_path)) + trans_group = TransformerGroup(4326, 2964, network=False) + trans_group.download_grids() + get_user_data_dir_mock.assert_called_with(True) + download_mock.assert_not_called() @patch("pyproj.transformer._download_resource_file") @@ -977,44 +1003,50 @@ def test_transformer_group__download_grids(get_user_data_dir_mock, tmp_path, cap def test_transformer_group__download_grids__directory( get_user_data_dir_mock, download_mock, tmp_path, capsys, ): - trans_group = TransformerGroup(4326, 2964, network=False) - trans_group.download_grids(directory=tmp_path) - get_user_data_dir_mock.assert_not_called() - captured = capsys.readouterr() - assert captured.out == "" - if grids_available( - "us_noaa_alaska.tif", "ca_nrc_ntv2_0.tif", check_network=False, check_all=True - ): - download_mock.assert_not_called() - elif grids_available("us_noaa_alaska.tif", check_network=False): - download_mock.assert_called_with( - file_url="https://cdn.proj.org/ca_nrc_ntv2_0.tif", - short_name="ca_nrc_ntv2_0.tif", - directory=tmp_path, - verbose=False, - ) - elif grids_available("ca_nrc_ntv2_0.tif", check_network=False): - download_mock.assert_called_with( - file_url="https://cdn.proj.org/us_noaa_alaska.tif", - short_name="us_noaa_alaska.tif", - directory=tmp_path, - verbose=False, - ) - else: - download_mock.assert_has_calls( - [ - call( - file_url="https://cdn.proj.org/us_noaa_alaska.tif", - short_name="us_noaa_alaska.tif", - directory=tmp_path, - verbose=False, - ), - call( - file_url="https://cdn.proj.org/ca_nrc_ntv2_0.tif", - short_name="ca_nrc_ntv2_0.tif", - directory=tmp_path, - verbose=False, - ), - ], - any_order=True, - ) + with proj_network_env(): + if pyproj._datadir._USE_GLOBAL_CONTEXT: + pyproj.set_global_context_network(active=False) + trans_group = TransformerGroup(4326, 2964, network=False) + trans_group.download_grids(directory=tmp_path) + get_user_data_dir_mock.assert_not_called() + captured = capsys.readouterr() + assert captured.out == "" + if grids_available( + "us_noaa_alaska.tif", + "ca_nrc_ntv2_0.tif", + check_network=False, + check_all=True, + ): + download_mock.assert_not_called() + elif grids_available("us_noaa_alaska.tif", check_network=False): + download_mock.assert_called_with( + file_url="https://cdn.proj.org/ca_nrc_ntv2_0.tif", + short_name="ca_nrc_ntv2_0.tif", + directory=tmp_path, + verbose=False, + ) + elif grids_available("ca_nrc_ntv2_0.tif", check_network=False): + download_mock.assert_called_with( + file_url="https://cdn.proj.org/us_noaa_alaska.tif", + short_name="us_noaa_alaska.tif", + directory=tmp_path, + verbose=False, + ) + else: + download_mock.assert_has_calls( + [ + call( + file_url="https://cdn.proj.org/us_noaa_alaska.tif", + short_name="us_noaa_alaska.tif", + directory=tmp_path, + verbose=False, + ), + call( + file_url="https://cdn.proj.org/ca_nrc_ntv2_0.tif", + short_name="ca_nrc_ntv2_0.tif", + directory=tmp_path, + verbose=False, + ), + ], + any_order=True, + )