diff --git a/tests/integration_tests/test_parameter_example.py b/tests/integration_tests/test_parameter_example.py new file mode 100644 index 00000000000..47b0a4e8254 --- /dev/null +++ b/tests/integration_tests/test_parameter_example.py @@ -0,0 +1,306 @@ +from datetime import datetime +from enum import Enum, auto +from pathlib import Path +from typing import Literal + +import cwrap +import hypothesis.strategies as st +import numpy as np +import xtgeo +from hypothesis import given, note, settings +from hypothesis.extra.numpy import arrays +from pytest import MonkeyPatch +from resdata import ResDataType +from resdata.grid import GridGenerator +from resdata.resfile import ResdataKW + +from ert.cli import ENSEMBLE_EXPERIMENT_MODE +from ert.field_utils import FieldFileFormat, read_field, save_field +from tests.unit_tests.config.egrid_generator import egrids +from tests.unit_tests.config.summary_generator import summaries + +from .run_cli import run_cli + +config_contents = """ + NUM_REALIZATIONS {num_realizations} + FIELD PARAM_A PARAMETER PARAM_A.grdecl INIT_FILES:PARAM_A%d.grdecl + FIELD AFI PARAMETER AFI.grdecl INIT_FILES:AFI.grdecl FORWARD_INIT:True + FIELD A_ROFF PARAMETER A_ROFF.roff INIT_FILES:A_ROFF%d.roff + FIELD PARAM_M5 PARAMETER PARAM_M5.grdecl INIT_FILES:PARAM_M5%d.grdecl MIN:0.5 + FIELD PARAM_M8 PARAMETER PARAM_M8.grdecl INIT_FILES:PARAM_M8%d.grdecl MAX:0.8 + FIELD TR58 PARAMETER TR58.grdecl INIT_FILES:TR58%d.grdecl MIN:0.5 MAX:0.8 + FIELD TRANS1 PARAMETER TRANS1.roff INIT_FILES:TRANS1%d.grdecl OUTPUT_TRANSFORM:LN + FIELD TRANS2 PARAMETER TRANS2.roff INIT_FILES:TRANS2%d.grdecl INIT_TRANSFORM:LN + + SURFACE TOP OUTPUT_FILE:TOP.irap INIT_FILES:TOP%d.irap BASE_SURFACE:BASETOP.irap + SURFACE BOTTOM OUTPUT_FILE:BOT.irap INIT_FILES:BOT.irap BASE_SURFACE:BASEBOT.irap FORWARD_INIT:True + + + ECLBASE ECLBASE + SUMMARY FOPR + GRID {grid_name}.{grid_format} + + + FORWARD_MODEL COPY_FILE(="../../../AFI.grdecl",="AFI.grdecl") + FORWARD_MODEL COPY_FILE(="../../../BOT.irap",="BOT.irap") + FORWARD_MODEL COPY_FILE(="../../../ECLBASE.UNSMRY",="ECLBASE.UNSMRY") + FORWARD_MODEL COPY_FILE(="../../../ECLBASE.SMSPEC",="ECLBASE.SMSPEC") +""" + +template = """ +MY_KEYWORD +SECOND_KEYWORD +""" + +prior = """ +MY_KEYWORD NORMAL 0 1 +SECOND_KEYWORD NORMAL 0 1 +""" + + +class IoLibrary(Enum): + ERT = auto() + XTGEO = auto() + RESDATA = auto() + + +class IoProvider: + def __init__(self, data: st.DataObject): + self.data = data + self.field_values = {} + self.surface_values = {} + + coordinates = st.integers(min_value=1, max_value=10) + self.dims = data.draw(st.tuples(coordinates, coordinates, coordinates)) + self.size = self.dims[0] * self.dims[1] * self.dims[2] + self.actnum = data.draw( + st.lists( + elements=st.integers(min_value=0, max_value=3), + min_size=self.size, + max_size=self.size, + ) + ) + + def _choose_lib(self): + return self.data.draw(st.sampled_from(IoLibrary)) + + def create_grid(self, grid_name: str, grid_format: Literal["grid", "egrid"]): + lib = self._choose_lib() + grid_file = grid_name + "." + grid_format + if grid_format == "grid": + grid = GridGenerator.create_rectangular( + self.dims, (1, 1, 1), actnum=self.actnum + ) + grid.save_GRID(grid_file) + elif lib == IoLibrary.RESDATA: + grid = GridGenerator.create_rectangular( + self.dims, (1, 1, 1), actnum=self.actnum + ) + grid.save_EGRID(grid_file) + elif lib == IoLibrary.XTGEO: + grid = xtgeo.create_box_grid(dimension=self.dims) + grid.to_file(grid_file, str(grid_format)) + elif lib == IoLibrary.RESDATA: + grid = GridGenerator.create_rectangular( + self.dims, (1, 1, 1), actnum=self.actnum + ) + if grid_format == "egrid": + grid.save_EGRID(grid_file) + elif grid_format == "grid": + grid.save_GRID(grid_file) + else: + raise ValueError() + elif lib == IoLibrary.ERT: + egrid = self.data.draw(egrids) + self.dims = egrid.shape + self.size = self.dims[0] * self.dims[1] * self.dims[2] + self.actnum = egrid.global_grid.actnum + egrid.to_file(grid_file) + else: + raise ValueError() + + def _random_values(self, shape): + return self.data.draw( + arrays( + elements=st.floats(min_value=1.0, max_value=10.0, width=32), + dtype=np.float32, + shape=shape, + ) + ) + + def create_field( + self, + name: str, + file_name: str, + fformat: FieldFileFormat, + ) -> None: + lib = self._choose_lib() + values = self._random_values(self.dims) + self.field_values[file_name] = values + + if lib == IoLibrary.XTGEO: + prop = xtgeo.GridProperty( + ncol=self.dims[0], + nrow=self.dims[1], + nlay=self.dims[2], + name=name, + values=values, + ) + prop.to_file( + file_name, fformat="roff" if fformat == "roff_binary" else fformat + ) + elif lib == IoLibrary.RESDATA: + if fformat == FieldFileFormat.GRDECL: + kw = ResdataKW(name, self.size, ResDataType.RD_FLOAT) + data = values.ravel() + for i in range(self.size): + kw[i] = data[i] + with cwrap.open(file_name, mode="w") as f: + kw.write_grdecl(f) + else: + # resdata cannot write roff + save_field(np.ma.masked_array(values), name, file_name, fformat) + + elif lib == IoLibrary.ERT: + save_field(np.ma.masked_array(values), name, file_name, fformat) + else: + raise ValueError() + + def create_surface(self, file_name: str) -> None: + values = self._random_values((2, 5)) + self.surface_values[file_name] = values + xtgeo.RegularSurface( + ncol=values.shape[0], + nrow=values.shape[1], + xinc=1.0, + yinc=1.0, + values=values, + ).to_file(file_name, fformat="irap_ascii") + + +@settings(max_examples=10) +@given( + io_source=st.builds(IoProvider, data=st.data()), + grid_format=st.sampled_from(["grid", "egrid"]), + summary=summaries( + start_date=st.just(datetime(1996, 1, 1)), + time_deltas=st.just([1.0, 2.0]), + summary_keys=st.just(["FOPR"]), + ), +) +def test_parameter_example(io_source, grid_format, summary, tmp_path_factory): + tmp_path = tmp_path_factory.mktemp("parameter_example") + note(f"Running in directory {dir}") + with MonkeyPatch.context() as patch: + patch.chdir(tmp_path) + NUM_REALIZATIONS = 10 + GRID_NAME = "GRID" + Path("config.ert").write_text( + config_contents.format( + grid_name=GRID_NAME, + grid_format=grid_format, + num_realizations=NUM_REALIZATIONS, + ) + ) + Path("prior.txt").write_text(prior) + Path("template.txt").write_text(template) + io_source.create_grid(GRID_NAME, grid_format) + + # create non-forward-init files that will have to exist before + # first iteration for all iterations + grdecl_fields = ["PARAM_A", "PARAM_M5", "PARAM_M8", "TR58"] + for i in range(NUM_REALIZATIONS): + for field in grdecl_fields: + io_source.create_field( + field, f"{field}{i}.grdecl", FieldFileFormat.GRDECL + ) + io_source.create_surface(f"TOP{i}.irap") + io_source.create_field( + "A_ROFF", f"A_ROFF{i}.roff", FieldFileFormat.ROFF_BINARY + ) + io_source.create_field( + "TRANS1", f"TRANS1{i}.grdecl", FieldFileFormat.GRDECL + ) + io_source.create_field( + "TRANS2", f"TRANS2{i}.grdecl", FieldFileFormat.GRDECL + ) + + # Creates forward init files that will be copied in by the + # COPY_FILE forward model. This fails if ert has already created + # the file. + io_source.create_field("AFI", "AFI.grdecl", FieldFileFormat.GRDECL) + io_source.create_surface("BOT.irap") + + # A COPY_FILE forward model also copies in the + # summary files expected to be created for the + # SUMMARY keyword + smspec, unsmry = summary + smspec.to_file("ECLBASE.SMSPEC") + unsmry.to_file("ECLBASE.UNSMRY") + + io_source.create_surface("BASETOP.irap") + io_source.create_surface("BASEBOT.irap") + + run_cli(ENSEMBLE_EXPERIMENT_MODE, "config.ert") + + for i in range(NUM_REALIZATIONS): + path = Path(f"simulations/realization-{i}/iter-0") + np.testing.assert_allclose( + read_field( + path / "PARAM_A.grdecl", + "PARAM_A", + shape=io_source.dims, + mask=io_source.actnum != 0, + ), + io_source.field_values[f"PARAM_A{i}.grdecl"], + atol=5e-5, + ) + np.testing.assert_allclose( + read_field( + path / "PARAM_M5.grdecl", + "PARAM_M5", + shape=io_source.dims, + mask=io_source.actnum != 0, + ), + np.clip(io_source.field_values[f"PARAM_M5{i}.grdecl"], 5.0, None), + atol=5e-5, + ) + np.testing.assert_allclose( + read_field( + path / "TR58.grdecl", + "TR58", + shape=io_source.dims, + mask=io_source.actnum != 0, + ), + np.clip(io_source.field_values[f"PARAM_M8{i}.grdecl"], 5.0, 8.0), + atol=5e-5, + ) + np.testing.assert_allclose( + read_field( + path / "TRANS1.roff", + "TRANS1", + shape=io_source.dims, + mask=io_source.actnum != 0, + ), + np.log(io_source.field_values[f"TRANS1{i}.grdecl"]), + atol=5e-5, + ) + np.testing.assert_allclose( + read_field( + path / "TRANS2.roff", + "TRANS2", + shape=io_source.dims, + mask=io_source.actnum != 0, + ), + np.log(io_source.field_values[f"TRANS2{i}.grdecl"]), + atol=5e-5, + ) + + np.testing.assert_allclose( + xtgeo.surface_from_file( + path / "TOP.irap", + "irap_ascii", + ).values, + (io_source.surface_values[f"TOP{i}.irap"]), + atol=5e-5, + ) diff --git a/tests/unit_tests/storage/test_field_parameter.py b/tests/unit_tests/storage/test_field_parameter.py deleted file mode 100644 index 150b9e94217..00000000000 --- a/tests/unit_tests/storage/test_field_parameter.py +++ /dev/null @@ -1,657 +0,0 @@ -import math -from pathlib import Path -from textwrap import dedent - -import cwrap -import numpy as np -import numpy.testing -import pytest -import xtgeo -from resdata import ResDataType -from resdata.geometry import Surface -from resdata.grid import GridGenerator -from resdata.resfile import ResdataKW - -from ert.config import Field, SummaryConfig -from ert.field_utils import Shape -from ert.storage import StorageAccessor, open_storage - -from .create_runpath import create_runpath, load_from_forward_model - - -def write_grid_property(name, grid, filename, file_format, shape, buffer): - arr = np.ndarray(shape=shape, buffer=buffer, dtype=buffer.dtype) - prop = xtgeo.GridProperty( - ncol=shape[0], nrow=shape[1], nlay=shape[2], values=arr, grid=grid, name=name - ) - prop.to_file(filename, fformat=file_format) - return arr - - -@pytest.fixture -def storage(tmp_path): - with open_storage(tmp_path / "storage", mode="w") as storage: - yield storage - - -def test_load_two_parameters_forward_init(storage, tmpdir): - with tmpdir.as_cwd(): - config = dedent( - """ - NUM_REALIZATIONS 1 - FIELD PARAM_A PARAMETER param_a.grdecl INIT_FILES:../../../param_a.grdecl FORWARD_INIT:True - FIELD PARAM_B PARAMETER param_b.GRDECL INIT_FILES:../../../param_b.GRDECL FORWARD_INIT:True - GRID MY_EGRID.EGRID - """ - ) - with open("config.ert", "w", encoding="utf-8") as fh: - fh.writelines(config) - - grid = xtgeo.create_box_grid(dimension=(10, 10, 1)) - grid.to_file("MY_EGRID.EGRID", "egrid") - - param_a = write_grid_property( - "PARAM_A", grid, "param_a.grdecl", "grdecl", (10, 10, 1), np.full((100), 22) - ) - param_b = write_grid_property( - "PARAM_B", grid, "param_b.GRDECL", "grdecl", (10, 10, 1), np.full((100), 77) - ) - - ensemble_config, fs = create_runpath(storage, "config.ert", iteration=0) - assert ensemble_config["PARAM_A"].forward_init - assert ensemble_config["PARAM_B"].forward_init - assert not Path("simulations/realization-0/iter-0/param_a.grdecl").exists() - assert not Path("simulations/realization-0/iter-0/param_b.GRDECL").exists() - - # should not be loaded yet - with pytest.raises( - KeyError, match="No dataset 'PARAM_A' in storage for realization 0" - ): - _ = fs.load_parameters("PARAM_A", [0])["values"] - - with pytest.raises( - KeyError, match="No dataset 'PARAM_B' in storage for realization 0" - ): - _ = fs.load_parameters("PARAM_B", [0])["values"] - - assert load_from_forward_model("config.ert", fs) == 1 - - create_runpath(storage, "config.ert", ensemble=fs, iteration=1) - - prop_a = xtgeo.gridproperty_from_file( - pfile="simulations/realization-0/iter-1/param_a.grdecl", - name="PARAM_A", - grid=grid, - ) - numpy.testing.assert_equal(prop_a.values.data, param_a) - - prop_b = xtgeo.gridproperty_from_file( - pfile="simulations/realization-0/iter-1/param_b.GRDECL", - name="PARAM_B", - grid=grid, - ) - numpy.testing.assert_equal(prop_b.values.data, param_b) - - # should be loaded now - loaded_a = fs.load_parameters("PARAM_A", [0])["values"] - assert (loaded_a.values == 22).all() - - loaded_b = fs.load_parameters("PARAM_B", [0])["values"] - assert (loaded_b.values == 77).all() - - -def test_load_two_parameters_roff(storage, tmpdir): - with tmpdir.as_cwd(): - config = dedent( - """ - NUM_REALIZATIONS 1 - FIELD PARAM_A PARAMETER param_a.roff INIT_FILES:param_a_%d.roff - FIELD PARAM_B PARAMETER param_b.roff INIT_FILES:param_b_%d.roff - GRID MY_EGRID.EGRID - """ - ) - with open("config.ert", "w", encoding="utf-8") as fh: - fh.writelines(config) - - grid = xtgeo.create_box_grid(dimension=(10, 10, 1)) - grid.to_file("MY_EGRID.EGRID", "egrid") - - param_a = write_grid_property( - "PARAM_A", grid, "param_a_0.roff", "roff", (10, 10, 1), np.full((100), 22) - ) - param_b = write_grid_property( - "PARAM_B", grid, "param_b_0.roff", "roff", (10, 10, 1), np.full((100), 77) - ) - - ensemble_config, fs = create_runpath(storage, "config.ert") - assert not ensemble_config["PARAM_A"].forward_init - assert not ensemble_config["PARAM_B"].forward_init - - loaded_a = fs.load_parameters("PARAM_A", [0])["values"] - assert (loaded_a.values == 22).all() - - loaded_b = fs.load_parameters("PARAM_B", [0])["values"] - assert (loaded_b.values == 77).all() - - prop_a = xtgeo.gridproperty_from_file( - pfile="simulations/realization-0/iter-0/param_a.roff", - name="PARAM_A", - ) - numpy.testing.assert_equal(prop_a.values.data, param_a) - - prop_b = xtgeo.gridproperty_from_file( - pfile="simulations/realization-0/iter-0/param_b.roff", - name="PARAM_B", - ) - numpy.testing.assert_equal(prop_b.values.data, param_b) - - -def test_load_two_parameters(storage, tmpdir): - with tmpdir.as_cwd(): - config = dedent( - """ - NUM_REALIZATIONS 1 - FIELD PARAM_A PARAMETER param_a.grdecl INIT_FILES:param_a_%d.grdecl - FIELD PARAM_B PARAMETER param_b.grdecl INIT_FILES:param_b_%d.grdecl - GRID MY_EGRID.EGRID - """ - ) - with open("config.ert", "w", encoding="utf-8") as fh: - fh.writelines(config) - - grid = xtgeo.create_box_grid(dimension=(10, 10, 1)) - grid.to_file("MY_EGRID.EGRID", "egrid") - - param_a = write_grid_property( - "PARAM_A", - grid, - "param_a_0.grdecl", - "grdecl", - (10, 10, 1), - np.full((100), 22), - ) - param_b = write_grid_property( - "PARAM_B", - grid, - "param_b_0.grdecl", - "grdecl", - (10, 10, 1), - np.full((100), 77), - ) - - ensemble_config, fs = create_runpath(storage, "config.ert") - assert not ensemble_config["PARAM_A"].forward_init - assert not ensemble_config["PARAM_B"].forward_init - - loaded_a = fs.load_parameters("PARAM_A", [0])["values"] - assert (loaded_a.values == 22).all() - - loaded_b = fs.load_parameters("PARAM_B", [0])["values"] - assert (loaded_b.values == 77).all() - - prop_a = xtgeo.gridproperty_from_file( - pfile="simulations/realization-0/iter-0/param_a.grdecl", - name="PARAM_A", - grid=grid, - ) - numpy.testing.assert_equal(prop_a.values.data, param_a) - - prop_b = xtgeo.gridproperty_from_file( - pfile="simulations/realization-0/iter-0/param_b.grdecl", - name="PARAM_B", - grid=grid, - ) - numpy.testing.assert_equal(prop_b.values.data, param_b) - - -@pytest.mark.parametrize( - "min_, max_, field_config", - [ - ( - 0.5, - None, - "FIELD MY_PARAM PARAMETER my_param.grdecl INIT_FILES:my_param_%d.grdecl MIN:0.5", - ), - ( - None, - 0.8, - "FIELD MY_PARAM PARAMETER my_param.grdecl INIT_FILES:my_param_%d.grdecl MAX:0.8", - ), - ( - 0.5, - 0.8, - "FIELD MY_PARAM PARAMETER my_param.grdecl INIT_FILES:my_param_%d.grdecl MIN:0.5 MAX:0.8", - ), - ], -) -def test_min_max(storage, tmpdir, min_: int, max_: int, field_config: str): - with tmpdir.as_cwd(): - config = dedent( - """ - NUM_REALIZATIONS 1 - GRID MY_EGRID.EGRID - """ - ) - config += field_config - with open("config.ert", "w", encoding="utf-8") as fh: - fh.writelines(config) - - grid = xtgeo.create_box_grid(dimension=(10, 10, 1)) - grid.to_file("MY_EGRID.EGRID", "egrid") - - buffer = np.random.random_sample(100) - buffer[56] = 0.001 - buffer[34] = 1.001 - write_grid_property( - "MY_PARAM", grid, "my_param_0.grdecl", "grdecl", (10, 10, 1), buffer - ) - - create_runpath(storage, "config.ert", [True]) - - my_prop = xtgeo.gridproperty_from_file( - pfile="simulations/realization-0/iter-0/my_param.grdecl", - name="MY_PARAM", - grid=grid, - ) - if min_ and max_: - vfunc = np.vectorize( - lambda x: ((x + 0.0001) >= min_) and ((x - 0.0001) <= max_) - ) - assert vfunc(my_prop.values.data).all() - elif min_: - vfunc = np.vectorize(lambda x: (x + 0.0001) >= min_) - assert vfunc(my_prop.values.data).all() - elif max_: - vfunc = np.vectorize(lambda x: (x - 0.0001) <= max_) - assert vfunc(my_prop.values.data).all() - - -def test_transformation(storage, tmpdir): - with tmpdir.as_cwd(): - config = dedent( - """ - NUM_REALIZATIONS 2 - FIELD PARAM_A PARAMETER param_a.grdecl INIT_FILES:param_a_%d.grdecl INIT_TRANSFORM:LN OUTPUT_TRANSFORM:EXP - GRID MY_EGRID.EGRID - """ - ) - with open("config.ert", "w", encoding="utf-8") as fh: - fh.writelines(config) - - grid = xtgeo.create_box_grid(dimension=(10, 10, 1)) - grid.to_file("MY_EGRID.EGRID", "egrid") - - param_a_1 = write_grid_property( - "PARAM_A", - grid, - "param_a_0.grdecl", - "grdecl", - (10, 10, 1), - np.full((100), math.exp(2.5), dtype=float), - ) - param_a_2 = write_grid_property( - "PARAM_A", - grid, - "param_a_1.grdecl", - "grdecl", - (10, 10, 1), - np.full((100), math.exp(1.5), dtype=float), - ) - - _, fs = create_runpath(storage, "config.ert", [True, True]) - - # stored internally as 2.5, 1.5 - loaded_a = fs.load_parameters("PARAM_A", [0, 1])["values"] - assert np.isclose(loaded_a.values[0], 2.5).all() - assert np.isclose(loaded_a.values[1], 1.5).all() - - prop_a_1 = xtgeo.gridproperty_from_file( - pfile="simulations/realization-0/iter-0/param_a.grdecl", - name="PARAM_A", - grid=grid, - ) - numpy.testing.assert_almost_equal(prop_a_1.values.data, param_a_1, decimal=5) - - prop_a_2 = xtgeo.gridproperty_from_file( - pfile="simulations/realization-1/iter-0/param_a.grdecl", - name="PARAM_A", - grid=grid, - ) - numpy.testing.assert_almost_equal(prop_a_2.values.data, param_a_2, decimal=5) - - -@pytest.mark.parametrize( - "config_str, expect_forward_init", - [ - ( - "FIELD MY_PARAM PARAMETER my_param.grdecl " - "INIT_FILES:../../../my_param_0.grdecl FORWARD_INIT:True", - True, - ), - ( - "FIELD MY_PARAM PARAMETER my_param.grdecl INIT_FILES:my_param_%d.grdecl", - False, - ), - ], -) -def test_forward_init(storage, tmpdir, config_str, expect_forward_init): - with tmpdir.as_cwd(): - config = dedent( - """ - NUM_REALIZATIONS 1 - GRID MY_EGRID.EGRID - """ - ) - config += config_str - with open("config.ert", mode="w", encoding="utf-8") as fh: - fh.writelines(config) - - grid = xtgeo.create_box_grid(dimension=(4, 4, 1)) - grid.to_file("MY_EGRID.EGRID", "egrid") - - expect_param = write_grid_property( - "MY_PARAM", - grid, - "my_param_0.grdecl", - "grdecl", - (4, 4, 1), - np.arange(start=0, stop=4 * 4), - ) - - ensemble_config, fs = create_runpath(storage, "config.ert") - assert ensemble_config["MY_PARAM"].forward_init is expect_forward_init - - # Assert that the data has been written to runpath - if expect_forward_init: - # FORWARD_INIT: True means that ERT waits until the end of the - # forward model to internalise the data - assert not Path("simulations/realization-0/iter-0/my_param.grdecl").exists() - - with pytest.raises( - KeyError, match="No dataset 'MY_PARAM' in storage for realization 0" - ): - fs.load_parameters("MY_PARAM", [0])["values"] - - # We try to load the parameters from the forward model, this would fail if - # forward init was not set correctly - assert load_from_forward_model("config.ert", fs) == 1 - - # Once data has been internalised, ERT will generate the - # parameter files - create_runpath(storage, "config.ert", ensemble=fs, iteration=1) - expected_iter = 1 if expect_forward_init else 0 - prop = xtgeo.gridproperty_from_file( - pfile=f"simulations/realization-0/iter-{expected_iter}/my_param.grdecl", - name="MY_PARAM", - grid=grid, - ) - numpy.testing.assert_equal(prop.values.data, expect_param) - - if expect_forward_init: - arr = fs.load_parameters("MY_PARAM", [0])["values"] - assert len(arr.values.ravel()) == 16 - - -@pytest.mark.parametrize( - "actnum", - [ - [True] * 24, - [True] * 12 + [False] * 12, - [False] * 12 + [True] * 12, - ], -) -def test_inactive_grdecl_ecl(tmpdir, storage, actnum): - fformat = "grdecl" - with tmpdir.as_cwd(): - config = dedent( - f""" - NUM_REALIZATIONS 1 - GRID MY_GRID.GRID - FIELD MY_PARAM PARAMETER my_param.{fformat} INIT_FILES:my_param_%d.{fformat} - """ - ) - missing_value = np.nan - expected_result = [ - float(i) if mask else missing_value for (i, mask) in enumerate(actnum) - ] - - grid = GridGenerator.create_rectangular((4, 3, 2), (1, 1, 1), actnum=actnum) - grid.save_GRID("MY_GRID.GRID") - - expect_param = ResdataKW( - "MY_PARAM", grid.get_global_size(), ResDataType.RD_FLOAT - ) - for i in range(grid.get_global_size()): - expect_param[i] = i - - with cwrap.open(f"my_param_0.{fformat}", mode="w") as f: - grid.write_grdecl(expect_param, f, default_value=missing_value) - - with open("config.ert", mode="w", encoding="utf-8") as fh: - fh.writelines(config) - create_runpath(storage, "config.ert") - - # # Assert that the data has been written to runpath - with cwrap.open( - f"simulations/realization-0/iter-0/my_param.{fformat}", "rb" - ) as f: - actual_param = ResdataKW.read_grdecl(f, "MY_PARAM") - - read_result = list(actual_param.numpy_view()) - numpy.testing.assert_array_equal(read_result, expected_result) - - -@pytest.mark.parametrize( - "actnum", - [ - [True] * 24, - [True] * 12 + [False] * 12, - [False] * 12 + [True] * 12, - ], -) -def test_inactive_grdecl_xtgeo(tmpdir, storage, actnum): - fformat = "grdecl" - with tmpdir.as_cwd(): - config = dedent( - f""" - NUM_REALIZATIONS 1 - GRID MY_GRID.EGRID - FIELD MY_PARAM PARAMETER my_param.{fformat} INIT_FILES:my_param_%d.{fformat} - """ - ) - missing_value = np.nan - expected_result = [ - float(i) if mask else missing_value for (i, mask) in enumerate(actnum) - ] - - grid = xtgeo.create_box_grid(dimension=(2, 3, 4)) - mask = grid.get_actnum() - mask.values = [int(mask) for mask in actnum] - grid.set_actnum(mask) - - grid.to_file("MY_GRID.EGRID", "egrid") - - prop = xtgeo.GridProperty( - ncol=grid.ncol, - nrow=grid.nrow, - nlay=grid.nlay, - name="MY_PARAM", - grid=grid, - values=np.arange(24), - ) - prop.to_file(f"my_param_0.{fformat}", fformat=fformat) - - with open("config.ert", mode="w", encoding="utf-8") as fh: - fh.writelines(config) - create_runpath(storage, "config.ert") - - read_prop = xtgeo.grid_property.gridproperty_from_file( - f"simulations/realization-0/iter-0/my_param.{fformat}", - fformat=fformat, - grid=grid, - name="MY_PARAM", - ) - read_result = list(read_prop.get_npvalues1d(fill_value=missing_value)) - numpy.testing.assert_array_equal(read_result, expected_result) - - -@pytest.mark.parametrize( - "actnum", - [ - [True] * 24, - [True] * 12 + [False] * 12, - [False] * 12 + [True] * 12, - ], -) -def test_inactive_roff_xtgeo(tmpdir, storage, actnum): - fformat = "roff" - with tmpdir.as_cwd(): - config = dedent( - f""" - NUM_REALIZATIONS 1 - GRID MY_GRID.EGRID - FIELD MY_PARAM PARAMETER my_param.{fformat} INIT_FILES:my_param_%d.{fformat} - """ - ) - missing_value = np.nan - expected_result = [ - float(i) if mask else missing_value for (i, mask) in enumerate(actnum) - ] - - grid = xtgeo.create_box_grid(dimension=(4, 2, 3)) - mask = grid.get_actnum() - mask.values = [int(mask) for mask in actnum] - grid.set_actnum(mask) - - grid.to_file("MY_GRID.EGRID", "egrid") - - prop = xtgeo.GridProperty( - ncol=grid.ncol, - nrow=grid.nrow, - nlay=grid.nlay, - name="MY_PARAM", - values=np.arange(24), - ) - prop.to_file(f"my_param_0.{fformat}", fformat=fformat) - - with open("config.ert", mode="w", encoding="utf-8") as fh: - fh.writelines(config) - create_runpath(storage, "config.ert") - - read_prop = xtgeo.grid_property.gridproperty_from_file( - f"simulations/realization-0/iter-0/my_param.{fformat}", fformat=fformat - ) - read_result = list(read_prop.get_npvalues1d(fill_value=missing_value)) - numpy.testing.assert_array_equal(read_result, expected_result) - - -def test_config_node_meta_information(storage, tmpdir): - """ - Populate nodes GEN_DATA, GEN_KW, FIELD, SURFACE & SUMMARY in configuration - Verify that properties stored for these nodes are correct - """ - - with tmpdir.as_cwd(): - config = dedent( - """ - NUM_REALIZATIONS 1 - GRID MY_EGRID.EGRID - SURFACE TOP OUTPUT_FILE:surf.irap INIT_FILES:Surfaces/surf%d.irap BASE_SURFACE:Surfaces/surf0.irap - SURFACE BOTTOM OUTPUT_FILE:surf.wrap INIT_FILES:surf.wrap BASE_SURFACE:Surfaces/surf0.wrap FORWARD_INIT:True - - GEN_DATA ABC RESULT_FILE:SimulatedABC_%d.txt INPUT_FORMAT:ASCII REPORT_STEPS:0 - GEN_DATA DEF RESULT_FILE:SimulatedDEF_%d.txt INPUT_FORMAT:ASCII REPORT_STEPS:0 - - GEN_KW KW_NAME template.txt kw.txt prior.txt INIT_FILES:custom_param%d.txt - - ECLBASE eclipse/model/MY_VERY_OWN_OIL_FIELD- - SUMMARY WOPR:MY_WELL - SUMMARY WOPR:MY_BASIN - - FIELD MY_PARAM PARAMETER my_param.grdecl INIT_FILES:my_param_%d.grdecl MIN:0.5 MAX:0.8 - FIELD MY_PARAM2 PARAMETER my_param.grdecl INIT_FILES:my_param_%d.grdecl MIN:0.5 MAX:0.8 FORWARD_INIT:True - """ - ) - - expect_surface = Surface( - nx=1, ny=3, xinc=1, yinc=1, xstart=1, ystart=1, angle=0 - ) - expect_surface.write("Surfaces/surf0.irap") - expect_surface_top = Surface( - nx=1, ny=3, xinc=1, yinc=1, xstart=1, ystart=1, angle=0 - ) - expect_surface_top.write("Surfaces/surf0.wrap") - - with open("template.txt", mode="w", encoding="utf-8") as fh: - fh.writelines("MY_KEYWORD ") - with open("prior.txt", mode="w", encoding="utf-8") as fh: - fh.writelines("MY_KEYWORD NORMAL 0 1") - with open("custom_param0.txt", mode="w", encoding="utf-8") as fh: - fh.writelines("MY_KEYWORD 1.31") - - with open("config.ert", "w", encoding="utf-8") as fh: - fh.writelines(config) - - grid = xtgeo.create_box_grid(dimension=(10, 10, 1)) - grid.to_file("MY_EGRID.EGRID", "egrid") - - buffer = np.random.random_sample(100) - buffer[56] = 0.001 - buffer[34] = 1.001 - write_grid_property( - "MY_PARAM", grid, "my_param_0.grdecl", "grdecl", (10, 10, 1), buffer - ) - - ensemble_config, _ = create_runpath(storage, "config.ert", [True]) - - # invalid object - with pytest.raises(KeyError, match="The key:X is not in"): - _ = ensemble_config["X"] - - # surface - assert ensemble_config["TOP"].forward_init is False - assert str(ensemble_config["TOP"].forward_init_file) == "Surfaces/surf%d.irap" - assert str(ensemble_config["TOP"].output_file) == "surf.irap" - - assert ensemble_config["BOTTOM"].forward_init is True - - # gen_data - assert ensemble_config["ABC"].input_file == "SimulatedABC_%d.txt" - - # gen_kw - assert ensemble_config["KW_NAME"].forward_init is False - assert ensemble_config["KW_NAME"].forward_init_file == str( - Path().cwd() / "custom_param%d.txt" - ) - assert ensemble_config["KW_NAME"].output_file == "kw.txt" - - # summary - assert isinstance(ensemble_config["summary"], SummaryConfig) - - # field - assert ensemble_config["MY_PARAM2"].forward_init is True - - assert ensemble_config["MY_PARAM"].forward_init is False - assert ensemble_config["MY_PARAM"].forward_init_file == "my_param_%d.grdecl" - assert ensemble_config["MY_PARAM"].output_file == Path("my_param.grdecl") - - -def test_field_parameter_size(tmpdir, storage: StorageAccessor): - with tmpdir.as_cwd(): - shape = Shape(8, 10, 2) - grid = xtgeo.create_box_grid(dimension=(shape.nx, shape.ny, shape.nz)) - grid.to_file("MY_EGRID.EGRID", "egrid") - - config = Field.from_config_list( - "MY_EGRID.EGRID", - shape, - [ - "PARAM", - "PARAMETER", - "param.GRDECL", - "INIT_FILES:param_%d.GRDECL", - "FORWARD_INIT:False", - ], - ) - experiment = storage.create_experiment(parameters=[config]) - assert len(config) == len(experiment.parameter_configuration["PARAM"]) == 160 diff --git a/tests/unit_tests/storage/test_local_ensemble.py b/tests/unit_tests/storage/test_local_ensemble.py deleted file mode 100644 index 31270915b12..00000000000 --- a/tests/unit_tests/storage/test_local_ensemble.py +++ /dev/null @@ -1,157 +0,0 @@ -import numpy as np -import pytest -import xarray as xr -import xtgeo -from resdata.grid import GridGenerator - -from ert.config import GenKwConfig -from ert.config.field import Field -from ert.field_utils import FieldFileFormat -from ert.storage import open_storage - - -def test_that_egrid_files_are_saved_and_loaded_correctly(tmp_path): - with open_storage(tmp_path, mode="w") as storage: - grid = xtgeo.create_box_grid(dimension=(4, 5, 1)) - mask = grid.get_actnum() - mask_values = [True] * 3 + [False] * 16 + [True] - mask.values = mask_values - grid_file = str(tmp_path / "grid.EGRID") - grid.to_file(grid_file, fformat="egrid") - param_group = "MY_PARAM" - - field_config = Field( - name=param_group, - forward_init=True, - nx=grid.nrow, - ny=grid.ncol, - nz=grid.nlay, - file_format=FieldFileFormat.GRDECL, - output_transformation=None, - input_transformation=None, - truncation_min=None, - truncation_max=None, - forward_init_file="", - output_file="", - grid_file=grid_file, - ) - - experiment = storage.create_experiment(parameters=[field_config]) - ensemble = storage.create_ensemble(experiment, name="foo", ensemble_size=2) - ensemble_dir = tmp_path / "ensembles" / str(ensemble.id) - assert ensemble_dir.exists() - - data = np.full_like(mask_values, np.nan, dtype=np.float32) - np.place(data, mask_values, np.array([1.2, 1.1, 4.3, 3.1], dtype=np.float32)) - da = xr.DataArray( - data.reshape((4, 5, 1)), - name="values", - dims=["x", "y", "z"], # type: ignore - ) - ds = da.to_dataset() - ensemble.save_parameters("MY_PARAM", 1, ds) - assert (ensemble_dir / "realization-1" / "MY_PARAM.nc").exists() - loaded_data = ensemble.load_parameters("MY_PARAM", 1)["values"] - np.testing.assert_array_equal(loaded_data.values, data.reshape((4, 5, 1))) - - -def test_that_grid_files_are_saved_and_loaded_correctly(tmp_path): - with open_storage(tmp_path / "storage", mode="w") as storage: - mask = [True] * 3 + [False] * 16 + [True] - grid = GridGenerator.create_rectangular((4, 5, 1), (1, 1, 1), actnum=mask) - grid_file = str(storage.path / "grid.GRID") - grid.save_GRID(grid_file) - param_group = "MY_PARAM" - field_config = Field( - name=param_group, - forward_init=True, - nx=grid.nx, - ny=grid.ny, - nz=grid.nz, - file_format=FieldFileFormat.GRDECL, - output_transformation=None, - input_transformation=None, - truncation_min=None, - truncation_max=None, - forward_init_file="", - output_file="", - grid_file=grid_file, - ) - experiment = storage.create_experiment(parameters=[field_config]) - ensemble = storage.create_ensemble(experiment, name="foo", ensemble_size=2) - ensemble_dir = tmp_path / "storage" / "ensembles" / str(ensemble.id) - assert ensemble_dir.exists() - - data = np.full_like(mask, np.nan, dtype=np.float32) - np.place(data, mask, np.array([1.2, 1.1, 4.3, 3.1], dtype=np.float32)) - da = xr.DataArray( - data.reshape((grid.nx, grid.ny, grid.nz)), - name="values", - dims=["x", "y", "z"], # type: ignore - ) - ds = da.to_dataset() - ensemble.save_parameters(param_group, 1, ds) - - saved_file = ensemble_dir / "realization-1" / f"{param_group}.nc" - assert saved_file.exists() - - loaded_data = ensemble.load_parameters(param_group, 1)["values"] - np.testing.assert_array_equal( - loaded_data.values, data.reshape((grid.nx, grid.ny, grid.nz)) - ) - - -def test_that_load_responses_throws_exception(tmp_path): - with open_storage(tmp_path, mode="w") as storage: - experiment = storage.create_experiment() - ensemble = storage.create_ensemble(experiment, name="foo", ensemble_size=1) - - with pytest.raises( - expected_exception=ValueError, match="I_DONT_EXIST is not a response" - ): - ensemble.load_responses("I_DONT_EXIST", (1,)) - - -def test_that_load_parameters_throws_exception(tmp_path): - with open_storage(tmp_path, mode="w") as storage: - experiment = storage.create_experiment() - ensemble = storage.create_ensemble(experiment, name="foo", ensemble_size=1) - - with pytest.raises(expected_exception=KeyError): - ensemble.load_parameters("I_DONT_EXIST", 1) - - -def test_that_loading_parameter_via_response_api_fails(tmp_path): - uniform_parameter = GenKwConfig( - name="PARAMETER", - forward_init=False, - template_file="", - transfer_function_definitions=[ - "KEY1 UNIFORM 0 1", - ], - output_file="kw.txt", - ) - with open_storage(tmp_path, mode="w") as storage: - experiment = storage.create_experiment( - parameters=[uniform_parameter], - ) - prior = storage.create_ensemble( - experiment, - ensemble_size=1, - iteration=0, - name="prior", - ) - - prior.save_parameters( - "PARAMETER", - 0, - xr.Dataset( - { - "values": ("names", [1.0]), - "transformed_values": ("names", [1.0]), - "names": ["KEY_1"], - } - ), - ) - with pytest.raises(ValueError, match="PARAMETER is not a response"): - prior.load_responses("PARAMETER", (0,)) diff --git a/tests/unit_tests/storage/test_local_storage.py b/tests/unit_tests/storage/test_local_storage.py index 50de709854c..5d3722d88d3 100644 --- a/tests/unit_tests/storage/test_local_storage.py +++ b/tests/unit_tests/storage/test_local_storage.py @@ -11,6 +11,7 @@ import numpy as np import pytest import xarray as xr +from hypothesis import assume from hypothesis.extra.numpy import arrays from hypothesis.stateful import Bundle, RuleBasedStateMachine, initialize, rule @@ -40,6 +41,62 @@ def _cases(storage): return sorted(x.name for x in storage.ensembles) +def test_that_loading_parameter_via_response_api_fails(tmp_path): + uniform_parameter = GenKwConfig( + name="PARAMETER", + forward_init=False, + template_file="", + transfer_function_definitions=[ + "KEY1 UNIFORM 0 1", + ], + output_file="kw.txt", + ) + with open_storage(tmp_path, mode="w") as storage: + experiment = storage.create_experiment( + parameters=[uniform_parameter], + ) + prior = storage.create_ensemble( + experiment, + ensemble_size=1, + iteration=0, + name="prior", + ) + + prior.save_parameters( + "PARAMETER", + 0, + xr.Dataset( + { + "values": ("names", [1.0]), + "transformed_values": ("names", [1.0]), + "names": ["KEY_1"], + } + ), + ) + with pytest.raises(ValueError, match="PARAMETER is not a response"): + prior.load_responses("PARAMETER", (0,)) + + +def test_that_load_responses_throws_exception(tmp_path): + with open_storage(tmp_path, mode="w") as storage: + experiment = storage.create_experiment() + ensemble = storage.create_ensemble(experiment, name="foo", ensemble_size=1) + + with pytest.raises( + expected_exception=ValueError, match="I_DONT_EXIST is not a response" + ): + ensemble.load_responses("I_DONT_EXIST", (1,)) + + +def test_that_load_parameters_throws_exception(tmp_path): + with open_storage(tmp_path, mode="w") as storage: + experiment = storage.create_experiment() + ensemble = storage.create_ensemble(experiment, name="foo", ensemble_size=1) + + with pytest.raises(expected_exception=KeyError): + ensemble.load_parameters("I_DONT_EXIST", 1) + + def test_open_empty_reader(tmp_path): with open_storage(tmp_path / "empty", mode="r") as storage: assert _cases(storage) == [] @@ -313,6 +370,17 @@ def get_field(self, ensemble_id: UUID): field_data["values"], ) + @rule(ensemble_id=ensemble_ids, parameter=words) + def load_unknown_parameter(self, ensemble_id: UUID, parameter: str): + ensemble = self.storage.get_ensemble(ensemble_id) + experiment_id = ensemble.experiment_id + parameter_names = [p.name for p in self.experiments[experiment_id].parameters] + assume(parameter not in parameter_names) + with pytest.raises( + KeyError, match=f"No dataset '{parameter}' in storage for realization 0" + ): + _ = ensemble.load_parameters(parameter, 0) + @rule( target=ensemble_ids, experiment=experiment_ids,