Try without standardized response config

src/ert/config/ensemble_config.py

@@ -82,15 +82,17 @@ class EnsembleConfig:
 
     def __post_init__(self) -> None:
         self._check_for_duplicate_names(
-            list(self.parameter_configs.values()), list(self.response_configs.values())
+            [p.name for p in self.parameter_configs.values()],
+            [key for config in self.response_configs.values() for key in config.keys],
         )
+
         self.grid_file = _get_abs_path(self.grid_file)
 
     @staticmethod
     def _check_for_duplicate_names(
         parameter_list: List[ParameterConfig], gen_data_list: List[ResponseConfig]
     ) -> None:
-        names_counter = Counter(g.name for g in parameter_list + gen_data_list)
+        names_counter = Counter(g for g in parameter_list + gen_data_list)
         duplicate_names = [n for n, c in names_counter.items() if c > 1]
         if duplicate_names:
             raise ConfigValidationError.with_context(
@@ -150,9 +152,13 @@ def make_field(field_list: List[str]) -> Field:
             + [make_field(f) for f in field_list]
         )
 
-        response_configs: List[ResponseConfig] = [
-            GenDataConfig.from_config_dict(config_dict)
-        ]
+        response_configs: List[ResponseConfig] = []
+
+        if "GEN_DATA" in config_dict:
+            gen_data_config = GenDataConfig.from_config_dict(config_dict)
+            if len(gen_data_config.keys) > 0:
+                response_configs.append(gen_data_config)
+
         refcase = (
             Refcase(start_date, refcase_keys, time_map, data)
             if data is not None
@@ -165,15 +171,13 @@ def make_field(field_list: List[str]) -> Field:
                     "In order to use summary responses, ECLBASE has to be set."
                 )
             time_map = set(refcase.dates) if refcase is not None else None
-            smry_kwargs = {}
-            if time_map is not None:
-                smry_kwargs["refcase"] = sorted(time_map)
+
             response_configs.append(
                 SummaryConfig(
                     name="summary",
-                    input_file=eclbase,
-                    keys=[i for val in summary_keys for i in val],
-                    kwargs=smry_kwargs,
+                    input_files=[eclbase],
+                    keys=[key for keys in summary_keys for key in keys],
+                    refcase=time_map,
                 )
             )
 
@@ -192,13 +196,23 @@ def __getitem__(self, key: str) -> Union[ParameterConfig, ResponseConfig]:
             return self.parameter_configs[key]
         elif key in self.response_configs:
             return self.response_configs[key]
+        elif _config := next(
+            (c for c in self.response_configs.values() if key in c.keys), None
+        ):
+            # Only hit by blockfs migration
+            # returns the same config for one call per
+            # response type. Is later deduped before saving to json
+            return _config
         else:
             raise KeyError(f"The key:{key} is not in the ensemble configuration")
 
     def hasNodeGenData(self, key: str) -> bool:
+        if "gen_data" not in self.response_configs:
+            return False
+
         config = self.response_configs["gen_data"]
         assert isinstance(config, GenDataConfig)
-        return key in config.names
+        return key in config.keys
 
     def addNode(self, config_node: Union[ParameterConfig, ResponseConfig]) -> None:
         assert config_node is not None

src/ert/config/ert_config.py

@@ -64,7 +64,6 @@
     parse,
 )
 from .queue_config import QueueConfig
-from .summary_config import SummaryConfig
 from .workflow import Workflow
 from .workflow_job import ErtScriptLoadFailure, WorkflowJob
 
@@ -461,7 +460,8 @@ def _create_list_of_forward_model_steps_to_run(
                 errors.append(
                     ConfigValidationError.with_context(
                         f"Could not find forward model step {fm_step_name!r} in list"
-                        f" of installed forward model steps: {list(installed_steps.keys())!r}",
+                        f" of installed forward model steps: "
+                        f"{list(installed_steps.keys())!r}",
                         fm_step_name,
                     )
                 )
@@ -477,7 +477,8 @@ def _create_list_of_forward_model_steps_to_run(
                     if req not in fm_step.private_args:
                         errors.append(
                             ConfigValidationError.with_context(
-                                f"Required keyword {req} not found for forward model step {fm_step_name}",
+                                f"Required keyword {req} not found for forward model "
+                                f"step {fm_step_name}",
                                 fm_step_name,
                             )
                         )
@@ -515,7 +516,8 @@ def _create_list_of_forward_model_steps_to_run(
                 except ForwardModelStepValidationError as err:
                     errors.append(
                         ConfigValidationError.with_context(
-                            f"Forward model step pre-experiment validation failed: {str(err)}",
+                            f"Forward model step pre-experiment validation failed: "
+                            f"{str(err)}",
                             context=fm_step.name,
                         ),
                     )
@@ -542,17 +544,27 @@ def manifest_to_json(self, iens: int = 0, iter: int = 0) -> Dict[str, Any]:
                     )
                     manifest[name] = file_path
         # Add expected response files to manifest
-        for name, respons_config in self.ensemble_config.response_configs.items():
-            input_file = str(respons_config.input_file)
-            if isinstance(respons_config, SummaryConfig):
-                input_file = input_file.replace("<IENS>", str(iens))
-                manifest[f"{name}_UNSMRY"] = f"{input_file}.UNSMRY"
-                manifest[f"{name}_SMSPEC"] = f"{input_file}.SMSPEC"
-            if isinstance(respons_config, GenDataConfig):
-                if respons_config.report_steps and iens in respons_config.report_steps:
+        if "summary" in self.ensemble_config.response_configs:
+            smry_config = self.ensemble_config.response_configs["summary"]
+            input_file = smry_config.input_files[0].replace("<IENS>", str(iens))
+            manifest["summary_UNSMRY"] = f"{input_file}.UNSMRY"
+            manifest["summary_SMSPEC"] = f"{input_file}.SMSPEC"
+
+        if "gen_data" in self.ensemble_config.response_configs:
+            gendata_config = self.ensemble_config.response_configs["gen_data"]
+            assert isinstance(gendata_config, GenDataConfig)
+            for name, input_file, report_steps in zip(
+                gendata_config.keys,
+                gendata_config.input_files,
+                gendata_config.report_steps_list,
+            ):
+                if report_steps and iens in report_steps:
+                    # Unrelated to this PR:
+                    # this line below makes no sense unless iens==reportstep
                     manifest[name] = input_file.replace("%d", str(iens))
                 elif "%d" not in input_file:
                     manifest[name] = input_file
+
         return manifest
 
     def forward_model_data_to_json(
@@ -593,7 +605,8 @@ def __init__(self, fm_step):
                 )
                 fm_step_description = f"{fm_step.name}({fm_step_args})"
                 self.substitution_context_hint = (
-                    f"parsing forward model step `FORWARD_MODEL {fm_step_description}` - "
+                    f"parsing forward model step `FORWARD_MODEL "
+                    f"{fm_step_description}` - "
                     "reconstructed, with defines applied during parsing"
                 )
                 self.copy_private_args = SubstitutionList()
@@ -932,7 +945,8 @@ def _create_observations(
                 else:
                     config_errors.append(
                         ErrorInfo(
-                            message=f"Unknown ObservationType {type(values)} for {obs_name}"
+                            message=f"Unknown ObservationType {type(values)} for"
+                            f" {obs_name}"
                         ).set_context(obs_name)
                     )
                     continue

src/ert/config/gen_data_config.py

@@ -1,5 +1,6 @@
+import dataclasses
 import os
-from dataclasses import dataclass, field
+from dataclasses import dataclass
 from pathlib import Path
 from typing import List, Literal, Optional
 
@@ -17,9 +18,17 @@
 @dataclass
 class GenDataConfig(ResponseConfig):
     name: str = "gen_data"
-    input_files: List[str] = field(default_factory=list)
-    names: List[str] = field(default_factory=list)
-    report_steps_list: List[Optional[List[int]]] = None
+    report_steps_list: List[Optional[List[int]]] = dataclasses.field(
+        default_factory=list
+    )
+
+    def __post_init__(self):
+        if len(self.report_steps_list) == 0:
+            self.report_steps_list = [[0] for _ in self.keys]
+        else:
+            for report_steps in self.report_steps_list:
+                if report_steps is not None:
+                    report_steps.sort()
 
     @classmethod
     def from_config_dict(cls, config_dict: ConfigDict) -> Self:
@@ -78,7 +87,12 @@ def from_config_dict(cls, config_dict: ConfigDict) -> Self:
             report_steps.append(_report_steps)
             input_files.append(res_file)
 
-        return cls(keys=keys, input_files=input_files, report_steps_list=report_steps)
+        return cls(
+            name="gen_data",
+            keys=keys,
+            input_files=input_files,
+            report_steps_list=report_steps,
+        )
 
     def read_from_file(self, run_path: str, _: int) -> xr.Dataset:
         def _read_file(filename: Path, report_step: int) -> xr.Dataset:
@@ -98,27 +112,50 @@ def _read_file(filename: Path, report_step: int) -> xr.Dataset:
             )
 
         errors = []
-        datasets = []
-        filename_fmt = self.input_file
+
         _run_path = Path(run_path)
-        if self.report_steps is None:
-            try:
-                datasets.append(_read_file(_run_path / filename_fmt, 0))
-            except ValueError as err:
-                errors.append(str(err))
-        else:
-            for report_step in self.report_steps:
-                filename = filename_fmt % report_step  # noqa
+        datasets_per_name = []
+
+        for name, input_file, report_steps in zip(
+            self.keys, self.input_files, self.report_steps_list
+        ):
+            datasets_per_report_step = []
+            if report_steps is None:
                 try:
-                    datasets.append(_read_file(_run_path / filename, report_step))
+                    datasets_per_report_step.append(
+                        _read_file(_run_path / input_file, 0)
+                    )
                 except ValueError as err:
                     errors.append(str(err))
+            else:
+                for report_step in report_steps:
+                    filename = input_file % report_step  # noqa
+                    try:
+                        datasets_per_report_step.append(
+                            _read_file(_run_path / filename, report_step)
+                        )
+                    except ValueError as err:
+                        errors.append(str(err))
+
+            ds_all_report_steps = xr.concat(
+                datasets_per_report_step, dim="report_step"
+            ).expand_dims(name=[name])
+            datasets_per_name.append(ds_all_report_steps)
+
         if errors:
-            raise ValueError(f"Error reading GEN_DATA: {self.name}, errors: {errors}")
-        combined = xr.combine_nested(datasets, concat_dim="report_step")
-        combined.attrs["response"] = self.response_type
+            raise ValueError(f"Error reading GEN_DATA: {name}, errors: {errors}")
+
+        combined = xr.concat(datasets_per_name, dim="name")
+        combined.attrs["response"] = "gen_data"
         return combined
 
+    def get_args_for_key(self, key: str):
+        for i, _key in enumerate(self.keys):
+            if key == _key:
+                return self.input_files[i], self.report_steps_list[i]
+
+        return None, None
+
     @property
     def cardinality(self) -> Literal["one_per_key", "one_per_realization"]:
         return "one_per_key"

src/ert/config/observations.py

@@ -9,7 +9,6 @@
 from ert.validation import rangestring_to_list
 
 from .enkf_observation_implementation_type import EnkfObservationImplementationType
-from .gen_data_config import GenDataConfig
 from .general_observation import GenObservation
 from .observation_vector import ObsVector
 from .parsing import ConfigWarning, HistorySource
@@ -401,20 +400,17 @@ def _handle_general_observation(
                 general_observation, obs_key, time_map, has_refcase
             )
 
-        config_node = ensemble_config[response_key]
-        if not isinstance(config_node, GenDataConfig):
+        gen_data_config = ensemble_config.response_configs.get("gen_data", None)
+        if not gen_data_config or response_key not in gen_data_config.keys:
             ConfigWarning.warn(
-                f"{response_key} has implementation type:"
-                f"'{type(config_node)}' - "
-                f"expected:'GEN_DATA' in observation:{obs_key}."
-                "The observation will be ignored",
-                obs_key,
+                f"Observation {obs_key} on GEN_DATA key {response_key}, but GEN_DATA"
+                f" key {response_key} is non-existing"
             )
             return {}
 
-        response_report_steps = (
-            [] if config_node.report_steps is None else config_node.report_steps
-        )
+        _, report_steps = gen_data_config.get_args_for_key(response_key)
+
+        response_report_steps = [] if report_steps is None else report_steps
         if (restart is None and response_report_steps) or (
             restart is not None and restart not in response_report_steps
         ):
@@ -440,7 +436,7 @@ def _handle_general_observation(
                 obs_key: ObsVector(
                     EnkfObservationImplementationType.GEN_OBS,
                     obs_key,
-                    config_node.name,
+                    response_key,
                     {
                         restart: cls._create_gen_obs(
                             (

src/ert/config/response_config.py

@@ -5,12 +5,13 @@
 import xarray as xr
 
 from ert.config.parameter_config import CustomDict
+from ert.config.parsing import ConfigDict
 
 
 @dataclasses.dataclass
 class ResponseConfig(ABC):
     name: str
-    input_files: str = ""
+    input_files: List[str] = dataclasses.field(default_factory=list)
     keys: List[str] = dataclasses.field(default_factory=list)
 
     @abstractmethod
@@ -21,14 +22,6 @@ def to_dict(self) -> Dict[str, Any]:
         data["_ert_kind"] = self.__class__.__name__
         return data
 
-    @staticmethod
-    def serialize_kwargs(kwargs: Dict[str, Any]) -> Dict[str, Any]:
-        return {**kwargs}
-
-    @staticmethod
-    def deserialize_kwargs(kwargs_serialized: Dict[str, Any]) -> Dict[str, Any]:
-        return {**kwargs_serialized}
-
     @property
     @abstractmethod
     def cardinality(self) -> Literal["one_per_key", "one_per_realization"]:
@@ -43,7 +36,7 @@ def response_type(self) -> str:
         ...
 
     @abstractmethod
-    def from_config_dict(self) -> str:
+    def from_config_dict(self, config_dict: ConfigDict) -> "ResponseConfig":
         """Creates a config, given an ert config dict.
         A response config may depend on several config kws, such as REFCASE
         for summary."""