fix: do not trust env while being able to use proxy (#33)

And change COPERNICUS_MARINE_SERVICE_{} to COPERNICUSMARINE_SERVICE_{} for USERNAME and PASSWORD

.github/workflows/tests.yml

@@ -24,7 +24,7 @@ jobs:
 
       - name: Run tests
         env:
-          COPERNICUS_MARINE_SERVICE_USERNAME: ${{ secrets.COPERNICUS_MARINE_SERVICE_USERNAME }}
-          COPERNICUS_MARINE_SERVICE_PASSWORD: ${{ secrets.COPERNICUS_MARINE_SERVICE_PASSWORD }}
+          COPERNICUSMARINE_SERVICE_USERNAME: ${{ secrets.COPERNICUSMARINE_SERVICE_USERNAME }}
+          COPERNICUSMARINE_SERVICE_PASSWORD: ${{ secrets.COPERNICUSMARINE_SERVICE_PASSWORD }}
         run: make run-tests
         shell: micromamba-shell {0}

.github/workflows/tox-tests.yml

@@ -24,7 +24,7 @@ jobs:
 
       - name: Run tests
         env:
-          COPERNICUS_MARINE_SERVICE_USERNAME: ${{ secrets.COPERNICUS_MARINE_SERVICE_USERNAME }}
-          COPERNICUS_MARINE_SERVICE_PASSWORD: ${{ secrets.COPERNICUS_MARINE_SERVICE_PASSWORD }}
+          COPERNICUSMARINE_SERVICE_USERNAME: ${{ secrets.COPERNICUSMARINE_SERVICE_USERNAME }}
+          COPERNICUSMARINE_SERVICE_PASSWORD: ${{ secrets.COPERNICUSMARINE_SERVICE_PASSWORD }}
         run: make run-tests-dependencie-versions
         shell: micromamba-shell {0}

CONTRIBUTION.md

@@ -34,8 +34,8 @@ conda activate {name_of_the_test_environment}
 ```
 Export credentials to local variables (if you don't use `moi`, simply put your own credentials):
 ```
-export COPERNICUS_MARINE_SERVICE_USERNAME=$(moi read-secret --name COPERNICUS_MARINE_SERVICE_USERNAME)
-export COPERNICUS_MARINE_SERVICE_PASSWORD=$(moi read-secret --name COPERNICUS_MARINE_SERVICE_PASSWORD)
+export COPERNICUSMARINE_SERVICE_USERNAME=$(moi read-secret --name COPERNICUSMARINE_SERVICE_USERNAME)
+export COPERNICUSMARINE_SERVICE_PASSWORD=$(moi read-secret --name COPERNICUSMARINE_SERVICE_PASSWORD)
 ```
 Finally run the tests:
 ```
@@ -59,8 +59,8 @@ pip install --editable .
 ```
 Export credentials to local variables (if you don't use `moi`, simply put your own credentials):
 ```
-export COPERNICUS_MARINE_SERVICE_USERNAME=$(moi read-secret --name COPERNICUS_MARINE_SERVICE_USERNAME)
-export COPERNICUS_MARINE_SERVICE_PASSWORD=$(moi read-secret --name COPERNICUS_MARINE_SERVICE_PASSWORD)
+export COPERNICUSMARINE_SERVICE_USERNAME=$(moi read-secret --name COPERNICUSMARINE_SERVICE_USERNAME)
+export COPERNICUSMARINE_SERVICE_PASSWORD=$(moi read-secret --name COPERNICUSMARINE_SERVICE_PASSWORD)
 ```
 Finally run the tests:
 ```

README.md

@@ -66,11 +66,26 @@ Cachier library is used for caching part of the requests (as result of `describe
 - on **UNIX** platforms: `export COPERNICUSMARINE_CACHE_DIRECTORY=<PATH>`
 - on **Windows** platforms: `set COPERNICUSMARINE_CACHE_DIRECTORY=<PATH>`
 
+### Network configuration
+
 #### Disable SSL
+
 A global SSL context is used when making HTTP calls using the `copernicusmarine` Toolbox. For some reason, it can lead to unexpected behavior depending on your network configuration. You can set the `COPERNICUSMARINE_DISABLE_SSL_CONTEXT` environment variable to any value to globally disable the usage of SSL in the toolbox:
+
 - on **UNIX** platforms: `export COPERNICUSMARINE_DISABLE_SSL_CONTEXT=True`
 - on **Windows** platforms: `set COPERNICUSMARINE_DISABLE_SSL_CONTEXT=True`
 
+#### Trust Env for python libraries
+
+To do HTTP calls, the Copernicus Marine Toolbox uses two python libraries: requests and aiohttp. By default, those libraries will have `trust_env` values set to `True`. If you want to deactivate this, you can set `COPERNICUSMARINE_TRUST_ENV=False` (default `True`). This can be useful for example if you don't want those libraries to read your `.netrc` file as it has been reported that having a `.netrc` with a line: "default login anonymous password user@site" is incompatible with S3 connection required by the toolbox.
+
+#### Proxy
+
+To use proxies, as describe in the [aiohttp documentation](https://docs.aiohttp.org/en/stable/client_advanced.html#proxy-support) you can use two options:
+
+- set the `HTTPS_PROXY` variable. For eg: `HTTPS_PROXY="http://user:[email protected]"`. It should work even with `COPERNICUSMARINE_TRUST_ENV=False`.
+- use a `.netrc` file but be aware that having a line: "default login anonymous password user@site" is incompatible with S3 connection required by the toolbox. Also note that if you have `COPERNICUSMARINE_TRUST_ENV=True` (the default value) then if `NETRC` environment variable is set with a specified location, the `.netrc` file will be read from the specified location there rather than from `~/.netrc`.
+
 ## Command Line Interface (CLI)
 
 ### The `--help` option

copernicusmarine/catalogue_parser/catalogue_parser.py

@@ -1,6 +1,5 @@
 import asyncio
 import logging
-import os
 import re
 from abc import ABC, abstractmethod
 from dataclasses import dataclass
@@ -12,16 +11,20 @@
 
 import nest_asyncio
 import pystac
-from aiohttp import ContentTypeError
+from aiohttp import ContentTypeError, ServerDisconnectedError
 from cachier.core import cachier
 from tqdm import tqdm
 
 from copernicusmarine.aioretry import RetryInfo, RetryPolicyStrategy, retry
 from copernicusmarine.command_line_interface.exception_handler import (
     log_exception_debug,
 )
+from copernicusmarine.core_functions.environment_variables import (
+    COPERNICUSMARINE_MAX_CONCURRENT_REQUESTS,
+)
 from copernicusmarine.core_functions.sessions import (
     get_configured_aiohttp_session,
+    get_https_proxy,
 )
 from copernicusmarine.core_functions.utils import (
     CACHE_BASE_DIRECTORY,
@@ -64,9 +67,7 @@ class _ServiceShortName(str, Enum):
     MARINE_DATA_STORE_STAC_BASE_URL_STAGING + "/catalog.stac.json"
 )
 
-MAX_CONCURRENT_REQUESTS = int(
-    os.getenv("COPERNICUSMARINE_MAX_CONCURRENT_REQUESTS", "15")
-)
+MAX_CONCURRENT_REQUESTS = int(COPERNICUSMARINE_MAX_CONCURRENT_REQUESTS)
 
 
 @dataclass(frozen=True)
@@ -362,6 +363,7 @@ class CatalogParserConnection:
     def __init__(self, proxy: Optional[str] = None) -> None:
         self.proxy = proxy
         self.session = get_configured_aiohttp_session()
+        self.proxy = get_https_proxy()
         self.__max_retries = 5
         self.__sleep_time = 1
 
@@ -371,6 +373,7 @@ async def get_json_file(self, url: str) -> dict[str, Any]:
         async with self.session.get(
             url,
             params=construct_query_params_for_marine_data_store_monitoring(),
+            proxy=self.proxy,
         ) as response:
             return await response.json()
 
@@ -384,6 +387,7 @@ def _retry_policy(self, info: RetryInfo) -> RetryPolicyStrategy:
                 TimeoutError,
                 ConnectionResetError,
                 ContentTypeError,
+                ServerDisconnectedError,
             ),
         ):
             logger.error(

copernicusmarine/command_line_interface/group_get.py

@@ -74,15 +74,15 @@ def cli_group_get() -> None:
     type=str,
     default=None,
     help="If not set, search for environment variable"
-    + " COPERNICUS_MARINE_SERVICE_USERNAME"
+    + " COPERNICUSMARINE_SERVICE_USERNAME"
     + ", or else look for configuration files, or else ask for user input.",
 )
 @click.option(
     "--password",
     type=str,
     default=None,
     help="If not set, search for environment variable"
-    + " COPERNICUS_MARINE_SERVICE_PASSWORD"
+    + " COPERNICUSMARINE_SERVICE_PASSWORD"
     + ", or else look for configuration files, or else ask for user input.",
 )
 @click.option(

copernicusmarine/command_line_interface/group_login.py

@@ -30,7 +30,7 @@ def cli_group_login() -> None:
     Examples:
 
     \b
-    COPERNICUS_MARINE_SERVICE_USERNAME=<USERNAME> COPERNICUS_MARINE_SERVICE_PASSWORD=<PASSWORD> copernicusmarine login
+    COPERNICUSMARINE_SERVICE_USERNAME=<USERNAME> COPERNICUSMARINE_SERVICE_PASSWORD=<PASSWORD> copernicusmarine login
 
     \b
     copernicusmarine login --username <USERNAME> --password <PASSWORD>
@@ -45,14 +45,14 @@ def cli_group_login() -> None:
     "--username",
     hide_input=False,
     help="If not set, search for environment variable"
-    + " COPERNICUS_MARINE_SERVICE_USERNAME"
+    + " COPERNICUSMARINE_SERVICE_USERNAME"
     + ", or else ask for user input.",
 )
 @click.option(
     "--password",
     hide_input=True,
     help="If not set, search for environment variable"
-    + " COPERNICUS_MARINE_SERVICE_PASSWORD"
+    + " COPERNICUSMARINE_SERVICE_PASSWORD"
     + ", or else ask for user input.",
 )
 @click.option(

copernicusmarine/command_line_interface/group_subset.py

@@ -93,15 +93,15 @@ def cli_group_subset() -> None:
     type=str,
     default=None,
     help="If not set, search for environment variable"
-    + " COPERNICUS_MARINE_SERVICE_USERNAME"
+    + " COPERNICUSMARINE_SERVICE_USERNAME"
     + ", or else look for configuration files, or else ask for user input.",
 )
 @click.option(
     "--password",
     type=str,
     default=None,
     help="If not set, search for environment variable"
-    + " COPERNICUS_MARINE_SERVICE_PASSWORD"
+    + " COPERNICUSMARINE_SERVICE_PASSWORD"
     + ", or else look for configuration files, or else ask for user input.",
 )
 @click.option(

copernicusmarine/core_functions/credentials_utils.py

@@ -1,7 +1,6 @@
 import base64
 import configparser
 import logging
-import os
 import pathlib
 from datetime import timedelta
 from netrc import netrc
@@ -13,6 +12,10 @@
 import requests
 from cachier.core import cachier
 
+from copernicusmarine.core_functions.environment_variables import (
+    COPERNICUSMARINE_SERVICE_PASSWORD,
+    COPERNICUSMARINE_SERVICE_USERNAME,
+)
 from copernicusmarine.core_functions.sessions import (
     get_configured_request_session,
 )
@@ -92,11 +95,10 @@ def _retrieve_credential_from_environment_variable(
 ) -> Optional[str]:
     if credential_type == "username":
         logger.debug("username loaded from environment variable")
-        return os.getenv("COPERNICUS_MARINE_SERVICE_USERNAME")
+        return COPERNICUSMARINE_SERVICE_USERNAME
     if credential_type == "password":
         logger.debug("password loaded from environment variable")
-        return os.getenv("COPERNICUS_MARINE_SERVICE_PASSWORD")
-    return None
+        return COPERNICUSMARINE_SERVICE_PASSWORD
 
 
 def _retrieve_credential_from_custom_configuration_files(
@@ -231,7 +233,9 @@ def _check_credentials_with_cas(username: str, password: str) -> bool:
         f"https://cmems-cas.cls.fr/cas/login?service={service}"
     )
     conn_session = get_configured_request_session()
-    login_session = conn_session.get(cmems_cas_login_url)
+    login_session = conn_session.get(
+        cmems_cas_login_url, proxies=conn_session.proxies
+    )
     login_from_html = lxml.html.fromstring(login_session.text)
     hidden_elements_from_html = login_from_html.xpath(
         '//form//input[@type="hidden"]'
@@ -243,7 +247,9 @@ def _check_credentials_with_cas(username: str, password: str) -> bool:
     playload["username"] = username
     playload["password"] = password
     logger.debug(f"POSTing credentials to {cmems_cas_login_url}...")
-    login_response = conn_session.post(cmems_cas_login_url, data=playload)
+    login_response = conn_session.post(
+        cmems_cas_login_url, data=playload, proxies=conn_session.proxies
+    )
     login_success = 'class="success"' in login_response.text
     logger.debug("User credentials checked")
     return login_success

copernicusmarine/core_functions/environment_variables.py

@@ -0,0 +1,37 @@
+import logging
+import os
+
+logger = logging.getLogger("copernicus_marine_root_logger")
+
+COPERNICUSMARINE_SERVICE_USERNAME = os.getenv(
+    "COPERNICUSMARINE_SERVICE_USERNAME"
+) or os.getenv("COPERNICUS_MARINE_SERVICE_USERNAME")
+if os.getenv("COPERNICUS_MARINE_SERVICE_USERNAME"):
+    logger.warning(
+        "COPERNICUS_MARINE_SERVICE_USERNAME is deprecated. "
+        "Please use COPERNICUSMARINE_SERVICE_USERNAME instead."
+    )
+COPERNICUSMARINE_SERVICE_PASSWORD = os.getenv(
+    "COPERNICUSMARINE_SERVICE_PASSWORD"
+) or os.getenv("COPERNICUS_MARINE_SERVICE_PASSWORD")
+if os.getenv("COPERNICUS_MARINE_SERVICE_PASSWORD"):
+    logger.warning(
+        "COPERNICUS_MARINE_SERVICE_PASSWORD is deprecated. "
+        "Please use COPERNICUSMARINE_SERVICE_PASSWORD instead."
+    )
+
+COPERNICUSMARINE_CACHE_DIRECTORY = os.getenv(
+    "COPERNICUSMARINE_CACHE_DIRECTORY", ""
+)
+
+COPERNICUSMARINE_MAX_CONCURRENT_REQUESTS = os.getenv(
+    "COPERNICUSMARINE_MAX_CONCURRENT_REQUESTS", "15"
+)
+
+COPERNICUSMARINE_DISABLE_SSL_CONTEXT = os.getenv(
+    "COPERNICUSMARINE_DISABLE_SSL_CONTEXT"
+)
+COPERNICUSMARINE_TRUST_ENV = os.getenv("COPERNICUSMARINE_TRUST_ENV", "True")
+
+PROXY_HTTPS = os.getenv("HTTPS_PROXY", "")
+PROXY_HTTP = os.getenv("HTTP_PROXY", "")

copernicusmarine/core_functions/sessions.py

@@ -1,4 +1,3 @@
-import os
 import ssl
 from typing import Optional
 
@@ -9,16 +8,27 @@
 import xarray
 
 from copernicusmarine.core_functions.custom_zarr_store import CustomS3Store
+from copernicusmarine.core_functions.environment_variables import (
+    COPERNICUSMARINE_DISABLE_SSL_CONTEXT,
+    COPERNICUSMARINE_TRUST_ENV,
+    PROXY_HTTP,
+    PROXY_HTTPS,
+)
 from copernicusmarine.core_functions.utils import (
     construct_query_params_for_marine_data_store_monitoring,
     parse_access_dataset_url,
 )
 
-TRUST_ENV = True
+TRUST_ENV = COPERNICUSMARINE_TRUST_ENV == "True"
+PROXIES = {}
+if PROXY_HTTP:
+    PROXIES["http"] = PROXY_HTTP
+if PROXY_HTTPS:
+    PROXIES["https"] = PROXY_HTTPS
 
 
 def _get_ssl_context() -> Optional[ssl.SSLContext]:
-    if os.getenv("COPERNICUSMARINE_DISABLE_SSL_CONTEXT") is not None:
+    if COPERNICUSMARINE_DISABLE_SSL_CONTEXT is not None:
         return None
     return ssl.create_default_context(cafile=certifi.where())
 
@@ -29,10 +39,15 @@ def get_configured_aiohttp_session() -> aiohttp.ClientSession:
     return aiohttp.ClientSession(connector=connector, trust_env=TRUST_ENV)
 
 
+def get_https_proxy() -> Optional[str]:
+    return PROXIES.get("https")
+
+
 def get_configured_request_session() -> requests.Session:
     session = requests.Session()
     session.trust_env = TRUST_ENV
     session.verify = certifi.where()
+    session.proxies = PROXIES
     return session
 
 
@@ -55,7 +70,7 @@ def open_zarr(
                 "params": construct_query_params_for_marine_data_store_monitoring(
                     username=copernicus_marine_username
                 ),
-                "client_kwargs": {"trust_env": TRUST_ENV},
+                "client_kwargs": {"trust_env": TRUST_ENV, "proxies": PROXIES},
                 "ssl": _get_ssl_context(),
             }
         }

copernicusmarine/core_functions/utils.py

@@ -22,6 +22,10 @@
 import xarray
 from requests import PreparedRequest
 
+from copernicusmarine.core_functions.environment_variables import (
+    COPERNICUSMARINE_CACHE_DIRECTORY,
+)
+
 logger = logging.getLogger("copernicus_marine_root_logger")
 
 OVERWRITE_SHORT_OPTION = "--overwrite"
@@ -33,9 +37,7 @@
 
 FORCE_DOWNLOAD_CLI_PROMPT_MESSAGE = "Do you want to proceed with download?"
 
-USER_DEFINED_CACHE_DIRECTORY = os.getenv(
-    "COPERNICUSMARINE_CACHE_DIRECTORY", ""
-)
+USER_DEFINED_CACHE_DIRECTORY = COPERNICUSMARINE_CACHE_DIRECTORY
 DEFAULT_CLIENT_BASE_DIRECTORY = (
     pathlib.Path(USER_DEFINED_CACHE_DIRECTORY)
     if USER_DEFINED_CACHE_DIRECTORY

copernicusmarine/core_functions/versions_verifier.py

@@ -70,5 +70,6 @@ def _get_client_required_versions(
         mds_versions: dict[str, str] = session.get(
             url_mds_versions,
             params=construct_query_params_for_marine_data_store_monitoring(),
+            proxies=session.proxies,
         ).json()["clientVersions"]
         return mds_versions