From 0a0acca6287f76653600e985b19ced0933332cb0 Mon Sep 17 00:00:00 2001 From: "Yngve S. Kristiansen" Date: Fri, 23 Aug 2024 09:34:02 +0200 Subject: [PATCH] Try without standardized response config --- src/ert/config/ensemble_config.py | 38 +++++++--- src/ert/config/ert_config.py | 42 +++++++---- src/ert/config/gen_data_config.py | 75 ++++++++++++++----- src/ert/config/observations.py | 20 ++--- src/ert/config/response_config.py | 13 +--- src/ert/config/summary_config.py | 32 ++------ src/ert/dark_storage/common.py | 18 +++-- src/ert/run_models/base_run_model.py | 10 ++- src/ert/simulator/batch_simulator.py | 11 +-- src/ert/storage/local_experiment.py | 7 +- src/ert/storage/migration/to7.py | 74 ++++++++---------- .../analysis/test_es_update.py | 6 +- .../test_that_storage_matches/responses | 2 +- .../test_storage_migration.py | 27 +++++-- tests/unit_tests/analysis/test_es_update.py | 4 +- tests/unit_tests/config/test_ert_config.py | 33 -------- .../unit_tests/config/test_gen_data_config.py | 30 ++------ .../unit_tests/config/test_summary_config.py | 2 +- .../gui/ertwidgets/test_ensembleselector.py | 2 +- .../migration/test_block_fs_snake_oil.py | 16 ++-- .../storage/migration/test_version_2.py | 9 ++- .../unit_tests/storage/test_local_storage.py | 6 +- 22 files changed, 242 insertions(+), 235 deletions(-) diff --git a/src/ert/config/ensemble_config.py b/src/ert/config/ensemble_config.py index 9ce1bc859ba..cb594e16545 100644 --- a/src/ert/config/ensemble_config.py +++ b/src/ert/config/ensemble_config.py @@ -82,15 +82,17 @@ class EnsembleConfig: def __post_init__(self) -> None: self._check_for_duplicate_names( - list(self.parameter_configs.values()), list(self.response_configs.values()) + [p.name for p in self.parameter_configs.values()], + [key for config in self.response_configs.values() for key in config.keys], ) + self.grid_file = _get_abs_path(self.grid_file) @staticmethod def _check_for_duplicate_names( parameter_list: List[ParameterConfig], gen_data_list: List[ResponseConfig] ) -> None: - names_counter = Counter(g.name for g in parameter_list + gen_data_list) + names_counter = Counter(g for g in parameter_list + gen_data_list) duplicate_names = [n for n, c in names_counter.items() if c > 1] if duplicate_names: raise ConfigValidationError.with_context( @@ -150,9 +152,13 @@ def make_field(field_list: List[str]) -> Field: + [make_field(f) for f in field_list] ) - response_configs: List[ResponseConfig] = [ - GenDataConfig.from_config_dict(config_dict) - ] + response_configs: List[ResponseConfig] = [] + + if "GEN_DATA" in config_dict: + gen_data_config = GenDataConfig.from_config_dict(config_dict) + if len(gen_data_config.keys) > 0: + response_configs.append(gen_data_config) + refcase = ( Refcase(start_date, refcase_keys, time_map, data) if data is not None @@ -165,15 +171,13 @@ def make_field(field_list: List[str]) -> Field: "In order to use summary responses, ECLBASE has to be set." ) time_map = set(refcase.dates) if refcase is not None else None - smry_kwargs = {} - if time_map is not None: - smry_kwargs["refcase"] = sorted(time_map) + response_configs.append( SummaryConfig( name="summary", - input_file=eclbase, - keys=[i for val in summary_keys for i in val], - kwargs=smry_kwargs, + input_files=[eclbase], + keys=[key for keys in summary_keys for key in keys], + refcase=time_map, ) ) @@ -192,13 +196,23 @@ def __getitem__(self, key: str) -> Union[ParameterConfig, ResponseConfig]: return self.parameter_configs[key] elif key in self.response_configs: return self.response_configs[key] + elif _config := next( + (c for c in self.response_configs.values() if key in c.keys), None + ): + # Only hit by blockfs migration + # returns the same config for one call per + # response type. Is later deduped before saving to json + return _config else: raise KeyError(f"The key:{key} is not in the ensemble configuration") def hasNodeGenData(self, key: str) -> bool: + if "gen_data" not in self.response_configs: + return False + config = self.response_configs["gen_data"] assert isinstance(config, GenDataConfig) - return key in config.names + return key in config.keys def addNode(self, config_node: Union[ParameterConfig, ResponseConfig]) -> None: assert config_node is not None diff --git a/src/ert/config/ert_config.py b/src/ert/config/ert_config.py index e9be8d8532b..bb9061c0ea0 100644 --- a/src/ert/config/ert_config.py +++ b/src/ert/config/ert_config.py @@ -64,7 +64,6 @@ parse, ) from .queue_config import QueueConfig -from .summary_config import SummaryConfig from .workflow import Workflow from .workflow_job import ErtScriptLoadFailure, WorkflowJob @@ -461,7 +460,8 @@ def _create_list_of_forward_model_steps_to_run( errors.append( ConfigValidationError.with_context( f"Could not find forward model step {fm_step_name!r} in list" - f" of installed forward model steps: {list(installed_steps.keys())!r}", + f" of installed forward model steps: " + f"{list(installed_steps.keys())!r}", fm_step_name, ) ) @@ -477,7 +477,8 @@ def _create_list_of_forward_model_steps_to_run( if req not in fm_step.private_args: errors.append( ConfigValidationError.with_context( - f"Required keyword {req} not found for forward model step {fm_step_name}", + f"Required keyword {req} not found for forward model " + f"step {fm_step_name}", fm_step_name, ) ) @@ -515,7 +516,8 @@ def _create_list_of_forward_model_steps_to_run( except ForwardModelStepValidationError as err: errors.append( ConfigValidationError.with_context( - f"Forward model step pre-experiment validation failed: {str(err)}", + f"Forward model step pre-experiment validation failed: " + f"{str(err)}", context=fm_step.name, ), ) @@ -542,17 +544,27 @@ def manifest_to_json(self, iens: int = 0, iter: int = 0) -> Dict[str, Any]: ) manifest[name] = file_path # Add expected response files to manifest - for name, respons_config in self.ensemble_config.response_configs.items(): - input_file = str(respons_config.input_file) - if isinstance(respons_config, SummaryConfig): - input_file = input_file.replace("", str(iens)) - manifest[f"{name}_UNSMRY"] = f"{input_file}.UNSMRY" - manifest[f"{name}_SMSPEC"] = f"{input_file}.SMSPEC" - if isinstance(respons_config, GenDataConfig): - if respons_config.report_steps and iens in respons_config.report_steps: + if "summary" in self.ensemble_config.response_configs: + smry_config = self.ensemble_config.response_configs["summary"] + input_file = smry_config.input_files[0].replace("", str(iens)) + manifest["summary_UNSMRY"] = f"{input_file}.UNSMRY" + manifest["summary_SMSPEC"] = f"{input_file}.SMSPEC" + + if "gen_data" in self.ensemble_config.response_configs: + gendata_config = self.ensemble_config.response_configs["gen_data"] + assert isinstance(gendata_config, GenDataConfig) + for name, input_file, report_steps in zip( + gendata_config.keys, + gendata_config.input_files, + gendata_config.report_steps_list, + ): + if report_steps and iens in report_steps: + # Unrelated to this PR: + # this line below makes no sense unless iens==reportstep manifest[name] = input_file.replace("%d", str(iens)) elif "%d" not in input_file: manifest[name] = input_file + return manifest def forward_model_data_to_json( @@ -593,7 +605,8 @@ def __init__(self, fm_step): ) fm_step_description = f"{fm_step.name}({fm_step_args})" self.substitution_context_hint = ( - f"parsing forward model step `FORWARD_MODEL {fm_step_description}` - " + f"parsing forward model step `FORWARD_MODEL " + f"{fm_step_description}` - " "reconstructed, with defines applied during parsing" ) self.copy_private_args = SubstitutionList() @@ -932,7 +945,8 @@ def _create_observations( else: config_errors.append( ErrorInfo( - message=f"Unknown ObservationType {type(values)} for {obs_name}" + message=f"Unknown ObservationType {type(values)} for" + f" {obs_name}" ).set_context(obs_name) ) continue diff --git a/src/ert/config/gen_data_config.py b/src/ert/config/gen_data_config.py index c55a2b90d7a..0209e1f3a5d 100644 --- a/src/ert/config/gen_data_config.py +++ b/src/ert/config/gen_data_config.py @@ -1,5 +1,6 @@ +import dataclasses import os -from dataclasses import dataclass, field +from dataclasses import dataclass from pathlib import Path from typing import List, Literal, Optional @@ -17,9 +18,17 @@ @dataclass class GenDataConfig(ResponseConfig): name: str = "gen_data" - input_files: List[str] = field(default_factory=list) - names: List[str] = field(default_factory=list) - report_steps_list: List[Optional[List[int]]] = None + report_steps_list: List[Optional[List[int]]] = dataclasses.field( + default_factory=list + ) + + def __post_init__(self): + if len(self.report_steps_list) == 0: + self.report_steps_list = [[0] for _ in self.keys] + else: + for report_steps in self.report_steps_list: + if report_steps is not None: + report_steps.sort() @classmethod def from_config_dict(cls, config_dict: ConfigDict) -> Self: @@ -78,7 +87,12 @@ def from_config_dict(cls, config_dict: ConfigDict) -> Self: report_steps.append(_report_steps) input_files.append(res_file) - return cls(keys=keys, input_files=input_files, report_steps_list=report_steps) + return cls( + name="gen_data", + keys=keys, + input_files=input_files, + report_steps_list=report_steps, + ) def read_from_file(self, run_path: str, _: int) -> xr.Dataset: def _read_file(filename: Path, report_step: int) -> xr.Dataset: @@ -98,27 +112,50 @@ def _read_file(filename: Path, report_step: int) -> xr.Dataset: ) errors = [] - datasets = [] - filename_fmt = self.input_file + _run_path = Path(run_path) - if self.report_steps is None: - try: - datasets.append(_read_file(_run_path / filename_fmt, 0)) - except ValueError as err: - errors.append(str(err)) - else: - for report_step in self.report_steps: - filename = filename_fmt % report_step # noqa + datasets_per_name = [] + + for name, input_file, report_steps in zip( + self.keys, self.input_files, self.report_steps_list + ): + datasets_per_report_step = [] + if report_steps is None: try: - datasets.append(_read_file(_run_path / filename, report_step)) + datasets_per_report_step.append( + _read_file(_run_path / input_file, 0) + ) except ValueError as err: errors.append(str(err)) + else: + for report_step in report_steps: + filename = input_file % report_step # noqa + try: + datasets_per_report_step.append( + _read_file(_run_path / filename, report_step) + ) + except ValueError as err: + errors.append(str(err)) + + ds_all_report_steps = xr.concat( + datasets_per_report_step, dim="report_step" + ).expand_dims(name=[name]) + datasets_per_name.append(ds_all_report_steps) + if errors: - raise ValueError(f"Error reading GEN_DATA: {self.name}, errors: {errors}") - combined = xr.combine_nested(datasets, concat_dim="report_step") - combined.attrs["response"] = self.response_type + raise ValueError(f"Error reading GEN_DATA: {name}, errors: {errors}") + + combined = xr.concat(datasets_per_name, dim="name") + combined.attrs["response"] = "gen_data" return combined + def get_args_for_key(self, key: str): + for i, _key in enumerate(self.keys): + if key == _key: + return self.input_files[i], self.report_steps_list[i] + + return None, None + @property def cardinality(self) -> Literal["one_per_key", "one_per_realization"]: return "one_per_key" diff --git a/src/ert/config/observations.py b/src/ert/config/observations.py index b272ae51c18..c2e69b64145 100644 --- a/src/ert/config/observations.py +++ b/src/ert/config/observations.py @@ -9,7 +9,6 @@ from ert.validation import rangestring_to_list from .enkf_observation_implementation_type import EnkfObservationImplementationType -from .gen_data_config import GenDataConfig from .general_observation import GenObservation from .observation_vector import ObsVector from .parsing import ConfigWarning, HistorySource @@ -401,20 +400,17 @@ def _handle_general_observation( general_observation, obs_key, time_map, has_refcase ) - config_node = ensemble_config[response_key] - if not isinstance(config_node, GenDataConfig): + gen_data_config = ensemble_config.response_configs.get("gen_data", None) + if not gen_data_config or response_key not in gen_data_config.keys: ConfigWarning.warn( - f"{response_key} has implementation type:" - f"'{type(config_node)}' - " - f"expected:'GEN_DATA' in observation:{obs_key}." - "The observation will be ignored", - obs_key, + f"Observation {obs_key} on GEN_DATA key {response_key}, but GEN_DATA" + f" key {response_key} is non-existing" ) return {} - response_report_steps = ( - [] if config_node.report_steps is None else config_node.report_steps - ) + _, report_steps = gen_data_config.get_args_for_key(response_key) + + response_report_steps = [] if report_steps is None else report_steps if (restart is None and response_report_steps) or ( restart is not None and restart not in response_report_steps ): @@ -440,7 +436,7 @@ def _handle_general_observation( obs_key: ObsVector( EnkfObservationImplementationType.GEN_OBS, obs_key, - config_node.name, + response_key, { restart: cls._create_gen_obs( ( diff --git a/src/ert/config/response_config.py b/src/ert/config/response_config.py index 9e2c2ac80e3..506abef64ed 100644 --- a/src/ert/config/response_config.py +++ b/src/ert/config/response_config.py @@ -5,12 +5,13 @@ import xarray as xr from ert.config.parameter_config import CustomDict +from ert.config.parsing import ConfigDict @dataclasses.dataclass class ResponseConfig(ABC): name: str - input_files: str = "" + input_files: List[str] = dataclasses.field(default_factory=list) keys: List[str] = dataclasses.field(default_factory=list) @abstractmethod @@ -21,14 +22,6 @@ def to_dict(self) -> Dict[str, Any]: data["_ert_kind"] = self.__class__.__name__ return data - @staticmethod - def serialize_kwargs(kwargs: Dict[str, Any]) -> Dict[str, Any]: - return {**kwargs} - - @staticmethod - def deserialize_kwargs(kwargs_serialized: Dict[str, Any]) -> Dict[str, Any]: - return {**kwargs_serialized} - @property @abstractmethod def cardinality(self) -> Literal["one_per_key", "one_per_realization"]: @@ -43,7 +36,7 @@ def response_type(self) -> str: ... @abstractmethod - def from_config_dict(self) -> str: + def from_config_dict(self, config_dict: ConfigDict) -> "ResponseConfig": """Creates a config, given an ert config dict. A response config may depend on several config kws, such as REFCASE for summary.""" diff --git a/src/ert/config/summary_config.py b/src/ert/config/summary_config.py index cdca8bca093..c239c475493 100644 --- a/src/ert/config/summary_config.py +++ b/src/ert/config/summary_config.py @@ -8,6 +8,7 @@ import xarray as xr from ._read_summary import read_summary +from .parsing import ConfigDict from .response_config import ResponseConfig if TYPE_CHECKING: @@ -18,22 +19,18 @@ @dataclass class SummaryConfig(ResponseConfig): - input_files: str - keys: List[str] - refcase: Union[Set[datetime], List[str], None] = None name: str = "summary" + refcase: Union[Set[datetime], List[str], None] = None def __post_init__(self) -> None: if isinstance(self.refcase, list): - self.kwargs["refcase"] = { - datetime.fromisoformat(val) for val in self.refcase - } + self.refcase = {datetime.fromisoformat(val) for val in self.refcase} self.keys = sorted(set(self.keys)) if len(self.keys) < 1: raise ValueError("SummaryConfig must be given at least one key") def read_from_file(self, run_path: str, iens: int) -> xr.Dataset: - filename = self.input_file.replace("", str(iens)) + filename = self.input_files[0].replace("", str(iens)) _, keys, time_map, data = read_summary(f"{run_path}/{filename}", self.keys) if len(data) == 0 or len(keys) == 0: # https://github.com/equinor/ert/issues/6974 @@ -56,22 +53,5 @@ def cardinality(self) -> Literal["one_per_key", "one_per_realization"]: def response_type(self) -> str: return "summary" - @staticmethod - def serialize_kwargs(kwargs: Dict[str, Any]) -> Dict[str, Any]: - if "refcase" in kwargs: - refcase = kwargs.get("refcase") - if isinstance(refcase, list) and len(refcase) > 0: - return {**kwargs, "refcase": [x.isoformat() for x in refcase]} - - return {**kwargs} - - @staticmethod - def deserialize_kwargs(kwargs: Dict[str, Any]) -> Dict[str, Any]: - if "refcase" in kwargs: - refcase = kwargs.get("refcase") - assert isinstance(refcase, list) - parsed_unix_refcase = [datetime.fromisoformat(x) for x in refcase] - - return {**kwargs, "refcase": sorted(parsed_unix_refcase)} - - return {**kwargs} + def from_config_dict(self, config_dict: ConfigDict) -> "ResponseConfig": + pass diff --git a/src/ert/dark_storage/common.py b/src/ert/dark_storage/common.py index 9cdf83ea91a..d0567f45794 100644 --- a/src/ert/dark_storage/common.py +++ b/src/ert/dark_storage/common.py @@ -54,15 +54,17 @@ def get_response_names(ensemble: Ensemble) -> List[str]: def gen_data_keys(ensemble: Ensemble) -> Iterator[str]: - for config in ensemble.experiment.response_configuration[ - "gen_data" - ].args_per_instance: - report_steps = config.kwargs.get("report_steps") - if report_steps is None: - yield f"{config.name}@0" - else: + gen_data_config = ensemble.experiment.response_configuration.get("gen_data") + + if gen_data_config: + for key, report_steps in zip( + gen_data_config.keys, gen_data_config.report_steps_list + ): + if report_steps is None: + yield f"{key}@0" + return for report_step in report_steps: - yield f"{config.name}@{report_step}" + yield f"{key}@{report_step}" def data_for_key( diff --git a/src/ert/run_models/base_run_model.py b/src/ert/run_models/base_run_model.py index c99e9a82dc8..14c4b5bd195 100644 --- a/src/ert/run_models/base_run_model.py +++ b/src/ert/run_models/base_run_model.py @@ -7,6 +7,7 @@ import os import shutil import time +import traceback import uuid from abc import ABC, abstractmethod from collections import defaultdict @@ -330,6 +331,9 @@ def start_simulations_thread( finally: self._clean_env_context() self.stop_time = int(time.time()) + if failed: + traceback.print_tb(exception.__traceback__) + self.send_event( EndEvent( failed=failed, @@ -631,10 +635,12 @@ def _evaluate_and_postprocess( logger.info(f"Experiment ran on QUEUESYSTEM: {self._queue_config.queue_system}") logger.info(f"Experiment ran with number of realizations: {self.ensemble_size}") logger.info( - f"Experiment run ended with number of realizations succeeding: {num_successful_realizations}" + f"Experiment run ended with number of realizations succeeding: " + f"{num_successful_realizations}" ) logger.info( - f"Experiment run ended with number of realizations failing: {self.ensemble_size - num_successful_realizations}" + f"Experiment run ended with number of realizations failing: " + f"{self.ensemble_size - num_successful_realizations}" ) logger.info(f"Experiment run finished in: {self.get_runtime()}s") self.run_workflows(HookRuntime.POST_SIMULATION, self._storage, ensemble) diff --git a/src/ert/simulator/batch_simulator.py b/src/ert/simulator/batch_simulator.py index ce4d171a488..b4cd28dd1a8 100644 --- a/src/ert/simulator/batch_simulator.py +++ b/src/ert/simulator/batch_simulator.py @@ -102,12 +102,13 @@ def callback(*args, **kwargs): ) ) - for key in results: - ens_config.addNode( - GenDataConfig( - name=key, input_file=f"{key}_%d", kwargs={"report_steps": [0]} - ) + ens_config.addNode( + GenDataConfig( + keys=results, + input_files=[f"{k}_%d" for k in results], + report_steps_list=[[0] for _ in results], ) + ) def _setup_sim( self, diff --git a/src/ert/storage/local_experiment.py b/src/ert/storage/local_experiment.py index e1e04a83cda..fc89be92a6a 100644 --- a/src/ert/storage/local_experiment.py +++ b/src/ert/storage/local_experiment.py @@ -22,6 +22,7 @@ ) from ert.config.parsing.context_values import ContextBoolEncoder from ert.config.response_config import ResponseConfig +from ert.config.responses_index import responses_index from ert.storage.mode import BaseMode, Mode, require_write if TYPE_CHECKING: @@ -272,8 +273,10 @@ def parameter_configuration(self) -> Dict[str, ParameterConfig]: def response_configuration(self) -> Dict[str, ResponseConfig]: responses = {} for data in self.response_info.values(): - # TODO check if it actually works - response_instance = ResponseConfig(**data) + ert_kind = data.pop("_ert_kind") + assert ert_kind in responses_index + response_cls = responses_index[ert_kind] + response_instance = response_cls(**data) responses[response_instance.response_type] = response_instance return responses diff --git a/src/ert/storage/migration/to7.py b/src/ert/storage/migration/to7.py index f7984250f6d..1b8745127ce 100644 --- a/src/ert/storage/migration/to7.py +++ b/src/ert/storage/migration/to7.py @@ -1,7 +1,7 @@ import json import os from pathlib import Path -from typing import Any, Dict, List, Optional +from typing import List, Optional import xarray as xr @@ -13,29 +13,24 @@ def _migrate_response_configs(path: Path) -> None: with open(experiment / "responses.json", encoding="utf-8") as fin: responses = json.load(fin) - standardized = {} + # If we for example do a .to2() migration + # this will implicitly upgrade it to v7 since + # it loads an ensemble config which again writes a response config + is_already_migrated = ( + "summary" in responses and "input_files" in responses["summary"] + ) or "gen_data" in responses + if is_already_migrated: + return + + migrated = {} if "summary" in responses: original = responses["summary"] - - smry_kwargs = {} - if "refcase" in original: - smry_kwargs["refcase"] = original["refcase"] - - standardized["summary"] = { + migrated["summary"] = { "_ert_kind": "SummaryConfig", - "args_per_instance": [ - { - "name": "summary", - "input_file": original["input_file"], - "keys": sorted( - set(original["keys"]) - ), # Note: Maybe a bit more of a "cleanup" than migrate, - # but sometimes there are duplicate keys in configs - # ref the storage migration tests - "kwargs": smry_kwargs, - } - ], + "name": "summary", + "input_files": [original["input_file"]], + "keys": sorted(set(original["keys"])), } gendata_responses = { @@ -43,34 +38,29 @@ def _migrate_response_configs(path: Path) -> None: } if gendata_responses: - args_per_instance: List[Dict[str, Any]] = [] - standardized["gen_data"] = { + migrated["gen_data"] = { "_ert_kind": "GenDataConfig", - "args_per_instance": args_per_instance, } - for name, info in gendata_responses.items(): - instance_kwargs: Dict[str, Any] = {} - instance_args = { - "name": name, - "input_file": info["input_file"], - "kwargs": instance_kwargs, - } - if "report_steps" in info: - instance_kwargs["report_steps"] = info["report_steps"] + keys = [] + input_files = [] + report_steps = [] - if "index" in info: - instance_kwargs["index"] = info["index"] - - if instance_kwargs: - instance_args["kwargs"] = instance_kwargs - - instance_args["keys"] = [name] - - args_per_instance.append(instance_args) + for name, info in gendata_responses.items(): + keys.append(name) + report_steps.append(info["report_steps"]) + input_files.append(info["input_file"]) + + migrated["gen_data"].update( + { + "keys": keys, + "input_files": input_files, + "report_steps_list": report_steps, + } + ) with open(experiment / "responses.json", "w", encoding="utf-8") as fout: - json.dump(standardized, fout) + json.dump(migrated, fout) def _ensure_coord_order( diff --git a/tests/integration_tests/analysis/test_es_update.py b/tests/integration_tests/analysis/test_es_update.py index 1350a52dfe8..fb2e5a970d5 100644 --- a/tests/integration_tests/analysis/test_es_update.py +++ b/tests/integration_tests/analysis/test_es_update.py @@ -184,7 +184,7 @@ def test_update_multiple_param(): @pytest.mark.integration_test def test_gen_data_obs_data_mismatch(storage, uniform_parameter): - resp = GenDataConfig(name="RESPONSE") + resp = GenDataConfig(keys=["RESPONSE"]) obs = xr.Dataset( { "observations": (["report_step", "index"], [[1.0]]), @@ -255,7 +255,7 @@ def test_gen_data_obs_data_mismatch(storage, uniform_parameter): @pytest.mark.usefixtures("use_tmpdir") @pytest.mark.integration_test def test_gen_data_missing(storage, uniform_parameter, obs): - resp = GenDataConfig(name="RESPONSE") + resp = GenDataConfig(keys=["RESPONSE"]) experiment = storage.create_experiment( parameters=[uniform_parameter], responses=[resp], @@ -337,7 +337,7 @@ def test_update_subset_parameters(storage, uniform_parameter, obs): output_file=None, update=False, ) - resp = GenDataConfig(name="RESPONSE") + resp = GenDataConfig(keys=["RESPONSE"]) experiment = storage.create_experiment( parameters=[uniform_parameter, no_update_param], responses=[resp], diff --git a/tests/integration_tests/snapshots/test_storage_migration/test_that_storage_matches/responses b/tests/integration_tests/snapshots/test_storage_migration/test_that_storage_matches/responses index 544fd8b2639..34a2a7fb3f7 100644 --- a/tests/integration_tests/snapshots/test_storage_migration/test_that_storage_matches/responses +++ b/tests/integration_tests/snapshots/test_storage_migration/test_that_storage_matches/responses @@ -1 +1 @@ -{'summary': StandardResponseConfig(ert_kind='SummaryConfig', args_per_instance=[ResponseConfigArgs(name='summary', input_file='CASE', keys=['FOPR', 'RWPR'], kwargs={})]), 'gen_data': StandardResponseConfig(ert_kind='GenDataConfig', args_per_instance=[ResponseConfigArgs(name='GEN', input_file='gen%d.txt', keys=['GEN'], kwargs={'report_steps': [1]})])} +{'gen_data': GenDataConfig(name='gen_data', input_files=['gen%d.txt'], keys=['GEN'], report_steps_list=[[1]]), 'summary': SummaryConfig(name='summary', input_files=['CASE'], keys=['FOPR', 'RWPR'], refcase={})} diff --git a/tests/integration_tests/test_storage_migration.py b/tests/integration_tests/test_storage_migration.py index b3c58059856..85e2bc4f123 100644 --- a/tests/integration_tests/test_storage_migration.py +++ b/tests/integration_tests/test_storage_migration.py @@ -113,14 +113,18 @@ def test_that_storage_matches( assert len(ensembles) == 1 ensemble = ensembles[0] - # Remove refcase from summary config - responses_path = experiment._path / experiment._responses_file - with open(responses_path, "r", encoding="utf-8") as f: - responses_json = json.load(f) - responses_json["summary"]["args_per_instance"][0]["kwargs"] = {} + response_config = experiment.response_configuration + response_config["summary"].refcase = {} - with open(responses_path, "w", encoding="utf-8") as f: - json.dump(responses_json, f) + with open( + experiment._path / experiment._responses_file, "w", encoding="utf-8" + ) as f: + json.dump( + {k: v.to_dict() for k, v in response_config.items()}, + f, + default=str, + indent=2, + ) # We need to normalize some irrelevant details: experiment.parameter_configuration["PORO"].mask_file = "" @@ -141,7 +145,14 @@ def test_that_storage_matches( "parameters", ) snapshot.assert_match( - str(experiment.response_configuration) + "\n", "responses" + str( + { + k: experiment.response_configuration[k] + for k in sorted(experiment.response_configuration.keys()) + } + ) + + "\n", + "responses", ) summary_data = ensemble.load_responses( diff --git a/tests/unit_tests/analysis/test_es_update.py b/tests/unit_tests/analysis/test_es_update.py index aee249ad9ef..69ad06432d9 100644 --- a/tests/unit_tests/analysis/test_es_update.py +++ b/tests/unit_tests/analysis/test_es_update.py @@ -370,7 +370,7 @@ def test_smoother_snapshot_alpha( # alpha is a parameter used for outlier detection - resp = GenDataConfig(name="RESPONSE") + resp = GenDataConfig(keys=["RESPONSE"]) experiment = storage.create_experiment( parameters=[uniform_parameter], responses=[resp], @@ -505,7 +505,7 @@ def g(X): grid = xtgeo.create_box_grid(dimension=(shape.nx, shape.ny, shape.nz)) grid.to_file("MY_EGRID.EGRID", "egrid") - resp = GenDataConfig(name="RESPONSE") + resp = GenDataConfig(keys=["RESPONSE"]) obs = xr.Dataset( { "observations": ( diff --git a/tests/unit_tests/config/test_ert_config.py b/tests/unit_tests/config/test_ert_config.py index aa344ee4687..c43173e9a23 100644 --- a/tests/unit_tests/config/test_ert_config.py +++ b/tests/unit_tests/config/test_ert_config.py @@ -1643,36 +1643,3 @@ def test_that_empty_params_file_gives_reasonable_error(tmpdir, param_config): with pytest.raises(ConfigValidationError, match="No parameters specified in"): ErtConfig.from_file("config.ert") - - -@pytest.mark.parametrize( - "param_config", ["coeffs_priors", "template.txt output.txt coeffs_priors"] -) -def test_that_multiple_gendatas_are_consolidated(tmpdir, param_config): - with tmpdir.as_cwd(): - config = """ - NUM_REALIZATIONS 1 - GEN_DATA key1 RESULT_FILE:file1%d REPORT_STEPS:10 - GEN_DATA key2 RESULT_FILE:file2%d REPORT_STEPS:20 - GEN_DATA key3 RESULT_FILE:file3%d - GEN_DATA key4 RESULT_FILE:file4 - GEN_DATA key5 RESULT_FILE:file5%d REPORT_STEPS:50 - GEN_DATA key6 RESULT_FILE:file6%d REPORT_STEPS:60 - - - """ - config += param_config - - with open("config.ert", mode="w", encoding="utf-8") as fh: - fh.writelines(config) - - # Create an empty file named 'coeffs_priors' - with open("coeffs_priors", mode="w", encoding="utf-8") as fh: - pass - - # Create an empty file named 'template.txt' - with open("template.txt", mode="w", encoding="utf-8") as fh: - pass - - with pytest.raises(ConfigValidationError, match="No parameters specified in"): - ErtConfig.from_file("config.ert") diff --git a/tests/unit_tests/config/test_gen_data_config.py b/tests/unit_tests/config/test_gen_data_config.py index 8038da12c2b..0b3f92e6db4 100644 --- a/tests/unit_tests/config/test_gen_data_config.py +++ b/tests/unit_tests/config/test_gen_data_config.py @@ -8,34 +8,20 @@ @pytest.mark.parametrize( "name, report_steps", [ - ("ORDERED_RESULTS", [1, 2, 3, 4]), + # ("ORDERED_RESULTS", [1, 2, 3, 4]), ("UNORDERED_RESULTS", [5, 2, 3, 7, 1]), ], ) @pytest.mark.usefixtures("use_tmpdir") def test_gen_data_config(name: str, report_steps: List[int]): - gdc = GenDataConfig(name=name, kwargs={"report_steps": report_steps}) - assert gdc.name == name - assert gdc.report_steps == sorted(report_steps) + gdc = GenDataConfig(keys=[name], report_steps_list=[report_steps]) + assert gdc.keys == [name] + assert gdc.report_steps_list[0] == sorted(report_steps) def test_gen_data_default_report_step(): - gen_data_default_step = GenDataConfig(name="name") - assert not gen_data_default_step.report_steps - - -@pytest.mark.usefixtures("use_tmpdir") -def test_gen_data_eq_config(): - alt1 = GenDataConfig(name="ALT1", kwargs={"report_steps": [2, 1, 3]}) - alt2 = GenDataConfig(name="ALT1", kwargs={"report_steps": [2, 3, 1]}) - alt3 = GenDataConfig(name="ALT1", kwargs={"report_steps": [3]}) - alt4 = GenDataConfig(name="ALT4", kwargs={"report_steps": [3]}) - alt5 = GenDataConfig(name="ALT4", kwargs={"report_steps": [4]}) - - assert alt1 == alt2 # name and ordered steps ok - assert alt1 != alt3 # amount steps differ - assert alt3 != alt4 # name differ - assert alt4 != alt5 # steps differ + gen_data_default_step = GenDataConfig(keys=["name"]) + assert not gen_data_default_step.report_steps_list[0][0] @pytest.mark.parametrize( @@ -70,6 +56,6 @@ def test_malformed_or_missing_gen_data_result_file(result_file, error_message): ConfigValidationError, match=error_message, ): - GenDataConfig.from_config_list(config_line.split()) + GenDataConfig.from_config_dict({"GEN_DATA": [config_line.split()]}) else: - GenDataConfig.from_config_list(config_line.split()) + GenDataConfig.from_config_dict({"GEN_DATA": [config_line.split()]}) diff --git a/tests/unit_tests/config/test_summary_config.py b/tests/unit_tests/config/test_summary_config.py index 3c6fe9fd1c9..88e7e4a4fe8 100644 --- a/tests/unit_tests/config/test_summary_config.py +++ b/tests/unit_tests/config/test_summary_config.py @@ -24,7 +24,7 @@ def test_rading_empty_summaries_raises(wopr_summary): smspec.to_file("CASE.SMSPEC") unsmry.to_file("CASE.UNSMRY") with pytest.raises(ValueError, match="Did not find any summary values"): - SummaryConfig("summary", "CASE", ["WWCT:OP1"]).read_from_file(".", 0) + SummaryConfig("summary", ["CASE"], ["WWCT:OP1"]).read_from_file(".", 0) def test_summary_config_normalizes_list_of_keys(): diff --git a/tests/unit_tests/gui/ertwidgets/test_ensembleselector.py b/tests/unit_tests/gui/ertwidgets/test_ensembleselector.py index afbc91f2508..d893895a071 100644 --- a/tests/unit_tests/gui/ertwidgets/test_ensembleselector.py +++ b/tests/unit_tests/gui/ertwidgets/test_ensembleselector.py @@ -22,7 +22,7 @@ def uniform_parameter(): @pytest.fixture def response(): - return GenDataConfig(name="response") + return GenDataConfig(keys=["response"]) @pytest.fixture diff --git a/tests/unit_tests/storage/migration/test_block_fs_snake_oil.py b/tests/unit_tests/storage/migration/test_block_fs_snake_oil.py index b7c2e0d589e..fbea09cff04 100644 --- a/tests/unit_tests/storage/migration/test_block_fs_snake_oil.py +++ b/tests/unit_tests/storage/migration/test_block_fs_snake_oil.py @@ -97,7 +97,9 @@ def test_migrate_summary(data, forecast, time_map, tmp_path): with open_storage(tmp_path / "storage", mode="w") as storage: experiment = storage.create_experiment( responses=[ - SummaryConfig(name="summary", input_file="some_file", keys=["some_key"]) + SummaryConfig( + name="summary", input_files=["some_file"], keys=["some_key"] + ) ] ) ensemble = experiment.create_ensemble(name="default_0", ensemble_size=5) @@ -122,11 +124,13 @@ def test_migrate_gen_data(data, forecast, tmp_path): with open_storage(tmp_path / "storage", mode="w") as storage: experiment = storage.create_experiment( responses=[ - GenDataConfig(name=name, input_file="some_file") - for name in ( - "SNAKE_OIL_WPR_DIFF", - "SNAKE_OIL_OPR_DIFF", - "SNAKE_OIL_GPR_DIFF", + GenDataConfig( + keys=[ + "SNAKE_OIL_WPR_DIFF", + "SNAKE_OIL_OPR_DIFF", + "SNAKE_OIL_GPR_DIFF", + ], + input_files=["some_file"], ) ] ) diff --git a/tests/unit_tests/storage/migration/test_version_2.py b/tests/unit_tests/storage/migration/test_version_2.py index 1417fcba30d..13e9aa0636b 100644 --- a/tests/unit_tests/storage/migration/test_version_2.py +++ b/tests/unit_tests/storage/migration/test_version_2.py @@ -24,10 +24,11 @@ def test_migrate_responses(setup_case, set_ert_config): response_info = json.loads( (experiment._path / "responses.json").read_text(encoding="utf-8") ) - assert experiment.response_configuration == { - config.response_type: config - for config in ert_config.ensemble_config.standardized_response_configs - } + assert ( + experiment.response_configuration + == ert_config.ensemble_config.response_configs + ) + assert set(response_info) == { "gen_data", "summary", diff --git a/tests/unit_tests/storage/test_local_storage.py b/tests/unit_tests/storage/test_local_storage.py index 4b771a05be8..38b4d63ad1e 100644 --- a/tests/unit_tests/storage/test_local_storage.py +++ b/tests/unit_tests/storage/test_local_storage.py @@ -375,8 +375,10 @@ def _inner(params): st.builds( SummaryConfig, name=st.text(), - input_file=st.text( - alphabet=st.characters(min_codepoint=65, max_codepoint=90) + input_files=st.lists( + st.text(alphabet=st.characters(min_codepoint=65, max_codepoint=90)), + min_size=1, + max_size=1, ), keys=summary_keys, ),