test: cache retrieved cutouts (#392)

* test: cache retrieved cutouts

* cache even more

.github/workflows/test.yaml

@@ -43,7 +43,27 @@ jobs:
         - windows-latest
     env:
       MPLBACKEND: Agg  # https://github.com/orgs/community/discussions/26434
+
     steps:
+    - name: Setup cache and secrets (Linux & MacOS)
+      if: runner.os != 'Windows'
+      run: |
+        echo "CACHE_PATH=$HOME/.atlite_cache" >> $GITHUB_ENV
+        echo "today=$(date +'%Y-%m-%d')"  >> $GITHUB_ENV
+        echo -ne "url: ${{ vars.CDSAPI_URL }}\nkey: ${{  secrets.CDSAPI_TOKEN  }}\n" > ~/.cdsapirc
+      shell: bash
+
+    - name: Setup cache and secrets (Windows)
+      if: runner.os == 'Windows'
+      run: |
+        echo CACHE_PATH=%USERPROFILE%\.atlite_cache >> %GITHUB_ENV%
+        echo url: ${{ vars.CDSAPI_URL }} > %USERPROFILE%\.cdsapirc
+        echo key: ${{ secrets.CDSAPI_TOKEN }} >> %USERPROFILE%\.cdsapirc
+        for /f "tokens=2 delims==" %%a in ('"wmic os get localdatetime /value"') do set "today=%%a"
+        set mydate=%today:~0,4%-%today:~4,2%-%today:~6,2%
+        echo today=%mydate% >> %GITHUB_ENV%
+      shell: cmd
+
     - uses: actions/checkout@v4
       with:
         fetch-depth: 0 # Needed for setuptools_scm
@@ -53,16 +73,13 @@ jobs:
       with:
         python-version: ${{ matrix.python-version }}
 
-    - name: Install macos dependencies
-      if: matrix.os == 'macos-latest'
-      run: |
-        brew install hdf5
-
-    - name: Setup CDS API
-      run: |
-        echo -ne "url: https://cds-beta.climate.copernicus.eu/api \nkey: ${{secrets.CDSAPI_TOKEN_BETA}}\n" > ~/.cdsapirc
-        python -m pip install --upgrade pip
-        pip install -e .[dev]
+    - name: Cache retrieved cutouts
+      uses: actions/cache@v4
+      with:
+        path: ${{ env.CACHE_PATH }}
+        key: retrieved-cutouts-${{ env.today }}
+        enableCrossOsArchive: true
+      id: cache-env
 
     - name: Download package
       uses: actions/download-artifact@v4
@@ -74,17 +91,12 @@ jobs:
       run: |
         python -m pip install uv
         uv pip install --compile --system "$(ls dist/*.whl)[dev]"
-        # Use --compile to get pip's behavior. Otherwise the pandapower installation 
-        # will be broken on python<3.12
-        # See https://github.com/astral-sh/uv/issues/1928#issuecomment-1968857514
-
     - name: Test with pytest
       run: |
-        coverage run -m pytest
+        coverage run -m pytest . --cache-path=${{ env.CACHE_PATH }} --verbose
         coverage xml
-        cat coverage.xml
 
     - name: Upload code coverage report
       uses: codecov/codecov-action@v4
       with:
-        token: ${{ secrets.CODECOV_TOKEN }}
+        token: ${{ secrets.CODECOV_TOKEN }}

test/conftest.py

@@ -0,0 +1,210 @@
+import os
+from pathlib import Path
+import pytest
+from atlite import Cutout
+import os
+from datetime import date
+
+import pytest
+import urllib3
+from dateutil.relativedelta import relativedelta
+
+
+TIME = "2013-01-01"
+BOUNDS = (-4, 56, 1.5, 62)
+SARAH_DIR = os.getenv("SARAH_DIR", "/home/vres/climate-data/sarah_v2")
+GEBCO_PATH = os.getenv("GEBCO_PATH", "/home/vres/climate-data/GEBCO_2014_2D.nc")
+
+
+def pytest_addoption(parser):
+    parser.addoption(
+        "--cache-path",
+        action="store",
+        default=None,
+        help="Specify path for atlite cache files to not use temporary directory",
+    )
+
+
+@pytest.fixture(scope="session", autouse=True)
+def cutouts_path(tmp_path_factory, pytestconfig):
+    custom_path = pytestconfig.getoption("--cache-path")
+    if custom_path:
+        path = Path(custom_path)
+        path.mkdir(parents=True, exist_ok=True)
+        return path
+    else:
+        return tmp_path_factory.mktemp("atlite_cutouts")
+
+
+@pytest.fixture(scope="session")
+def cutout_era5(cutouts_path):
+    tmp_path = cutouts_path / "cutout_era5.nc"
+    cutout = Cutout(path=tmp_path, module="era5", bounds=BOUNDS, time=TIME)
+    cutout.prepare()
+    return cutout
+
+
+@pytest.fixture(scope="session")
+def cutout_era5_mon(cutouts_path):
+    tmp_path = cutouts_path / "cutout_era5_mon.nc"
+    cutout = Cutout(path=tmp_path, module="era5", bounds=BOUNDS, time=TIME)
+    cutout.prepare(monthly_requests=True, concurrent_requests=False)
+
+    return cutout
+
+
+@pytest.fixture(scope="session")
+def cutout_era5_mon_concurrent(cutouts_path):
+    tmp_path = cutouts_path / "cutout_era5_mon_concurrent.nc"
+    cutout = Cutout(path=tmp_path, module="era5", bounds=BOUNDS, time=TIME)
+    cutout.prepare(monthly_requests=True, concurrent_requests=True)
+
+    return cutout
+
+
+@pytest.fixture(scope="session")
+def cutout_era5_3h_sampling(cutouts_path):
+    tmp_path = cutouts_path / "cutout_era5_3h_sampling.nc"
+    time = [
+        f"{TIME} 00:00",
+        f"{TIME} 03:00",
+        f"{TIME} 06:00",
+        f"{TIME} 09:00",
+        f"{TIME} 12:00",
+        f"{TIME} 15:00",
+        f"{TIME} 18:00",
+        f"{TIME} 21:00",
+    ]
+    cutout = Cutout(path=tmp_path, module="era5", bounds=BOUNDS, time=time)
+    cutout.prepare()
+    return cutout
+
+
+@pytest.fixture(scope="session")
+def cutout_era5_2days_crossing_months(cutouts_path):
+    tmp_path = cutouts_path / "cutout_era5_2days_crossing_months.nc"
+    time = slice("2013-02-28", "2013-03-01")
+    cutout = Cutout(path=tmp_path, module="era5", bounds=BOUNDS, time=time)
+    cutout.prepare()
+    return cutout
+
+
+@pytest.fixture(scope="session")
+def cutout_era5_coarse(cutouts_path):
+    tmp_path = cutouts_path / "cutout_era5_coarse.nc"
+    cutout = Cutout(
+        path=tmp_path, module="era5", bounds=BOUNDS, time=TIME, dx=0.5, dy=0.7
+    )
+    cutout.prepare()
+    return cutout
+
+
+@pytest.fixture(scope="session")
+def cutout_era5_weird_resolution(cutouts_path):
+    tmp_path = cutouts_path / "cutout_era5_weird_resolution.nc"
+    cutout = Cutout(
+        path=tmp_path,
+        module="era5",
+        bounds=BOUNDS,
+        time=TIME,
+        dx=0.132,
+        dy=0.32,
+    )
+    cutout.prepare()
+    return cutout
+
+
+@pytest.fixture(scope="session")
+def cutout_era5_reduced(cutouts_path):
+    tmp_path = cutouts_path / "cutout_era5_reduced.nc"
+    cutout = Cutout(path=tmp_path, module="era5", bounds=BOUNDS, time=TIME)
+    return cutout
+
+
+@pytest.fixture(scope="session")
+def cutout_era5_overwrite(cutouts_path, cutout_era5_reduced):
+    tmp_path = cutouts_path / "cutout_era5_overwrite.nc"
+    cutout = Cutout(path=tmp_path, module="era5", bounds=BOUNDS, time=TIME)
+    # cutout.data = cutout.data.drop_vars("influx_direct")
+    # cutout.prepare("influx", overwrite=True)
+    # TODO Needs to be fixed
+    return cutout
+
+
+@pytest.fixture(scope="session")
+def cutout_era5t(cutouts_path):
+    tmp_path = cutouts_path / "cutout_era5t.nc"
+
+    today = date.today()
+    first_day_this_month = today.replace(day=1)
+    first_day_prev_month = first_day_this_month - relativedelta(months=1)
+    last_day_second_prev_month = first_day_prev_month - relativedelta(days=1)
+
+    cutout = Cutout(
+        path=tmp_path,
+        module="era5",
+        bounds=BOUNDS,
+        time=slice(last_day_second_prev_month, first_day_prev_month),
+    )
+    cutout.prepare()
+    return cutout
+
+
+@pytest.fixture(scope="session")
+def cutout_sarah(cutouts_path):
+    tmp_path = cutouts_path / "cut_out_sarah.nc"
+    cutout = Cutout(
+        path=tmp_path,
+        module=["sarah", "era5"],
+        bounds=BOUNDS,
+        time=TIME,
+        sarah_dir=SARAH_DIR,
+    )
+    cutout.prepare()
+    return cutout
+
+
+@pytest.fixture(scope="session")
+def cutout_sarah_fine(cutouts_path):
+    tmp_path = cutouts_path / "cutout_sarah_fine.nc"
+    cutout = Cutout(
+        path=tmp_path,
+        module="sarah",
+        bounds=BOUNDS,
+        time=TIME,
+        dx=0.05,
+        dy=0.05,
+        sarah_dir=SARAH_DIR,
+    )
+    cutout.prepare()
+    return cutout
+
+
+@pytest.fixture(scope="session")
+def cutout_sarah_weird_resolution(cutouts_path):
+    tmp_path = cutouts_path / "cutout_sarah_weird_resolution.nc"
+    cutout = Cutout(
+        path=tmp_path,
+        module="sarah",
+        bounds=BOUNDS,
+        time=TIME,
+        dx=0.132,
+        dy=0.32,
+        sarah_dir=SARAH_DIR,
+    )
+    cutout.prepare()
+    return cutout
+
+
+@pytest.fixture(scope="session")
+def cutout_gebco(cutouts_path):
+    tmp_path = cutouts_path / "cutout_gebco.nc"
+    cutout = Cutout(
+        path=tmp_path,
+        module="gebco",
+        bounds=BOUNDS,
+        time=TIME,
+        gebco_path=GEBCO_PATH,
+    )
+    cutout.prepare()
+    return cutout

test/test_gis.py

@@ -43,11 +43,10 @@
 raster_clip = 0.25  # this rastio is excluded (True) in the raster
 
 
-@pytest.fixture
-def ref():
-    return Cutout(
-        path="creation_ref", module="era5", bounds=(X0, Y0, X1, Y1), time=TIME
-    )
+@pytest.fixture(scope="session")
+def ref(cutouts_path):
+    tmp_path = cutouts_path / "creation_ref.nc"
+    return Cutout(path=tmp_path, module="era5", bounds=(X0, Y0, X1, Y1), time=TIME)
 
 
 @pytest.fixture(scope="session")