From 8983ac551d073caf629a117235f3d6b513d4ef3e Mon Sep 17 00:00:00 2001 From: Andreas Eknes Lie <114403625+andreas-el@users.noreply.github.com> Date: Tue, 13 Aug 2024 10:31:17 +0200 Subject: [PATCH] Concatenate observations for selected ensembles * Do not unset observations if found * Concatenate observations from ensembles --- src/ert/gui/tools/plot/plot_api.py | 76 +++++++++++-------- src/ert/gui/tools/plot/plot_window.py | 2 +- .../gui/tools/plot/test_plot_api.py | 4 +- 3 files changed, 47 insertions(+), 35 deletions(-) diff --git a/src/ert/gui/tools/plot/plot_api.py b/src/ert/gui/tools/plot/plot_api.py index 6cc35d1982e..b1ce4ef9774 100644 --- a/src/ert/gui/tools/plot/plot_api.py +++ b/src/ert/gui/tools/plot/plot_api.py @@ -116,10 +116,16 @@ def all_data_type_keys(self) -> List[PlotApiKeyDefinition]: self._check_response(response) for key, value in response.json().items(): assert isinstance(key, str) + + has_observation = value["has_observations"] + k = all_keys.get(key) + if k and k.observations: + has_observation = True + all_keys[key] = PlotApiKeyDefinition( key=key, index_type="VALUE", - observations=value["has_observations"], + observations=has_observation, dimensionality=2, metadata=value["userdata"], log_scale=key.startswith("LOG10_"), @@ -175,43 +181,49 @@ def data_for_key(self, ensemble_name: str, key: str) -> pd.DataFrame: except ValueError: return df - def observations_for_key(self, ensemble_name: str, key: str) -> pd.DataFrame: + def observations_for_key(self, ensemble_names: List[str], key: str) -> pd.DataFrame: """Returns a pandas DataFrame with the datapoints for a given observation key - for a given ensemble. The row index is the realization number, and the column index + for a given ensembles. The row index is the realization number, and the column index is a multi-index with (obs_key, index/date, obs_index), where index/date is used to relate the observation to the data point it relates to, and obs_index is the index for the observation itself""" + all_observations = pd.DataFrame() + for ensemble_name in ensemble_names: + ensemble = self._get_ensemble(ensemble_name) + if not ensemble: + continue - ensemble = self._get_ensemble(ensemble_name) - if not ensemble: - return pd.DataFrame() + with StorageService.session() as client: + response = client.get( + f"/ensembles/{ensemble.id}/records/{key}/observations", + timeout=self._timeout, + ) + self._check_response(response) + if not response.json(): + continue + try: + obs = response.json()[0] + except (KeyError, IndexError, JSONDecodeError) as e: + raise httpx.RequestError( + f"Observation schema might have changed key={key}, ensemble_name={ensemble_name}, e={e}" + ) from e + try: + int(obs["x_axis"][0]) + key_index = [int(v) for v in obs["x_axis"]] + except ValueError: + key_index = [pd.Timestamp(v) for v in obs["x_axis"]] + + data_struct = { + "STD": obs["errors"], + "OBS": obs["values"], + "key_index": key_index, + } + + all_observations = pd.concat( + [all_observations, pd.DataFrame(data_struct)] + ) - with StorageService.session() as client: - response = client.get( - f"/ensembles/{ensemble.id}/records/{key}/observations", - timeout=self._timeout, - ) - self._check_response(response) - if not response.json(): - return pd.DataFrame() - try: - obs = response.json()[0] - except (KeyError, IndexError, JSONDecodeError) as e: - raise httpx.RequestError( - f"Observation schema might have changed key={key}, ensemble_name={ensemble_name}, e={e}" - ) from e - try: - int(obs["x_axis"][0]) - key_index = [int(v) for v in obs["x_axis"]] - except ValueError: - key_index = [pd.Timestamp(v) for v in obs["x_axis"]] - - data_struct = { - "STD": obs["errors"], - "OBS": obs["values"], - "key_index": key_index, - } - return pd.DataFrame(data_struct).T + return all_observations.T def history_data(self, key: str, ensembles: Optional[List[str]]) -> pd.DataFrame: """Returns a pandas DataFrame with the data points for the history for a diff --git a/src/ert/gui/tools/plot/plot_window.py b/src/ert/gui/tools/plot/plot_window.py index d12d3e060b3..46cb6721270 100644 --- a/src/ert/gui/tools/plot/plot_window.py +++ b/src/ert/gui/tools/plot/plot_window.py @@ -203,7 +203,7 @@ def updatePlot(self, layer: Optional[int] = None) -> None: if key_def.observations and selected_ensembles: try: observations = self._api.observations_for_key( - selected_ensembles[0].name, key + [ensembles.name for ensembles in selected_ensembles], key ) except (RequestError, TimeoutError) as e: logger.exception(e) diff --git a/tests/unit_tests/gui/tools/plot/test_plot_api.py b/tests/unit_tests/gui/tools/plot/test_plot_api.py index ebc7f435973..864d07ecfbc 100644 --- a/tests/unit_tests/gui/tools/plot/test_plot_api.py +++ b/tests/unit_tests/gui/tools/plot/test_plot_api.py @@ -79,7 +79,7 @@ def test_can_load_data_and_observations(api): for key, value in keys.items(): observations = value["observations"] if observations: - obs_data = api.observations_for_key(ensemble_name, key) + obs_data = api.observations_for_key([ensemble_name], key) assert not obs_data.empty data = api.data_for_key(ensemble_name, key) assert not data.empty @@ -131,7 +131,7 @@ def test_plot_api_request_errors_all_data_type_keys(api, mocker): def test_plot_api_request_errors(api): ensemble_name = "default_0" with pytest.raises(httpx.RequestError): - api.observations_for_key(ensemble_name, "should_not_be_there") + api.observations_for_key([ensemble_name], "should_not_be_there") with pytest.raises(httpx.RequestError): api.data_for_key(ensemble_name, "should_not_be_there")