Add workaround for problems with microseconds

Workaround for storage not handling datetimes
with microseconds due to index overflow in netcdf3.
#6952

src/ert/analysis/_es_update.py

@@ -311,22 +311,28 @@ def _get_obs_and_measure_data(
                 name: list(set(index.get_level_values(name))) for name in index.names
             }
             observation = observation.sel(sub_selection)
-        ds = source_fs.load_responses(group, tuple(iens_active_index))
+        response = source_fs.load_responses(group, tuple(iens_active_index))
+        if "time" in observation.coords:
+            response = response.reindex(
+                time=observation.time, method="nearest", tolerance="1s"  # type: ignore
+            )
         try:
-            filtered_ds = observation.merge(ds, join="left")
+            filtered_response = observation.merge(response, join="left")
         except KeyError as e:
             raise ErtAnalysisError(
                 f"Mismatched index for: "
                 f"Observation: {obs_key} attached to response: {group}"
             ) from e
 
-        observation_keys.append([obs_key] * len(filtered_ds.observations.data.ravel()))
-        observation_values.append(filtered_ds["observations"].data.ravel())
-        observation_errors.append(filtered_ds["std"].data.ravel())
+        observation_keys.append(
+            [obs_key] * len(filtered_response.observations.data.ravel())
+        )
+        observation_values.append(filtered_response["observations"].data.ravel())
+        observation_errors.append(filtered_response["std"].data.ravel())
         measured_data.append(
-            filtered_ds["values"]
+            filtered_response["values"]
             .transpose(..., "realization")
-            .values.reshape((-1, len(filtered_ds.realization)))
+            .values.reshape((-1, len(filtered_response.realization)))
         )
     source_fs.load_responses.cache_clear()
     return (

src/ert/config/_read_summary.py

@@ -316,7 +316,8 @@ def _read_spec(
                         hour=hour,
                         minute=minute,
                         second=microsecond // 10**6,
-                        microsecond=microsecond % 10**6,
+                        # Due to https://github.com/equinor/ert/issues/6952
+                        # microsecond=self.micro_seconds % 10**6,
                     )
                 except Exception as err:
                     raise ValueError(
@@ -404,6 +405,19 @@ def optional_get(arr: Optional[npt.NDArray[Any]], idx: int) -> Any:
     )
 
 
+def _round_to_seconds(dt: datetime) -> datetime:
+    """
+    >>> _round_to_seconds(datetime(2000, 1, 1, 1, 0, 1, 1))
+    datetime.datetime(2000, 1, 1, 1, 0, 1)
+    >>> _round_to_seconds(datetime(2000, 1, 1, 1, 0, 1, 500001))
+    datetime.datetime(2000, 1, 1, 1, 0, 2)
+    >>> _round_to_seconds(datetime(2000, 1, 1, 1, 0, 1, 499999))
+    datetime.datetime(2000, 1, 1, 1, 0, 1)
+    """
+    extra_sec = round(dt.microsecond / 10**6)
+    return dt.replace(microsecond=0) + timedelta(seconds=extra_sec)
+
+
 def _read_summary(
     summary: str,
     start_date: datetime,
@@ -427,7 +441,14 @@ def read_params() -> None:
         if last_params is not None:
             vals = _check_vals("PARAMS", summary, last_params.read_array())
             values.append(vals[indices])
-            dates.append(start_date + unit.make_delta(float(vals[date_index])))
+
+            dates.append(
+                _round_to_seconds(
+                    start_date + unit.make_delta(float(vals[date_index]))
+                ),
+            )
+            # Due to https://github.com/equinor/ert/issues/6952
+            # dates.append(start_date + unit.make_delta(float(vals[date_index])))
             last_params = None
 
     with open(summary, mode) as fp:

src/ert/config/summary_config.py

@@ -30,19 +30,6 @@ def __post_init__(self) -> None:
     def read_from_file(self, run_path: str, iens: int) -> xr.Dataset:
         filename = self.input_file.replace("<IENS>", str(iens))
         _, keys, time_map, data = read_summary(f"{run_path}/{filename}", self.keys)
-
-        if self.refcase:
-            assert isinstance(self.refcase, set)
-            missing = self.refcase.difference(time_map)
-            if missing:
-                first, last = min(missing), max(missing)
-                logger.warning(
-                    f"Realization: {iens}, load warning: {len(missing)} "
-                    f"inconsistencies in time map, first: Time mismatch for response "
-                    f"time: {first}, last: Time mismatch for response time: "
-                    f"{last} from: {run_path}/{filename}.UNSMRY"
-                )
-
         ds = xr.Dataset(
             {"values": (["name", "time"], data)},
             coords={"time": time_map, "name": keys},

tests/unit_tests/config/summary_generator.py

@@ -205,7 +205,8 @@ def to_datetime(self) -> datetime:
             hour=self.hour,
             minute=self.minutes,
             second=self.micro_seconds // 10**6,
-            microsecond=self.micro_seconds % 10**6,
+            # Due to https://github.com/equinor/ert/issues/6952
+            # microsecond=self.micro_seconds % 10**6,
         )
 
     @classmethod
@@ -216,7 +217,9 @@ def from_datetime(cls, dt: datetime) -> Self:
             day=dt.day,
             hour=dt.hour,
             minutes=dt.minute,
-            micro_seconds=dt.second * 10**6 + dt.microsecond,
+            # Due to https://github.com/equinor/ert/issues/6952
+            micro_seconds=dt.second * 10**6,
+            # micro_seconds=dt.second * 10**6 + dt.microsecond,
         )
 
 

tests/unit_tests/test_load_forward_model.py

@@ -69,48 +69,6 @@ def run_simulator(time_step_count, start_date) -> Summary:
     return summary
 
 
-@pytest.mark.usefixtures("copy_snake_oil_case_storage")
-def test_load_inconsistent_time_map_summary(caplog):
-    """
-    Checking that we dont util_abort, we fail the forward model instead
-    """
-    cwd = os.getcwd()
-
-    # Get rid of GEN_DATA as we are only interested in SUMMARY
-    with fileinput.input("snake_oil.ert", inplace=True) as fin:
-        for line in fin:
-            if line.startswith("GEN_DATA"):
-                continue
-            print(line, end="")
-
-    facade = LibresFacade.from_config_file("snake_oil.ert")
-    realisation_number = 0
-    storage = open_storage(facade.enspath, mode="w")
-    ensemble = storage.get_ensemble_by_name("default_0")
-    assert ensemble.get_realization_mask_with_responses()[
-        realisation_number
-    ]  # Check prior state
-
-    # Create a result that is incompatible with the refcase
-    run_path = Path("storage") / "snake_oil" / "runpath" / "realization-0" / "iter-0"
-    os.chdir(run_path)
-    ecl_sum = run_simulator(1, datetime(2000, 1, 1))
-    ecl_sum.fwrite()
-    os.chdir(cwd)
-
-    realizations = [False] * facade.get_ensemble_size()
-    realizations[realisation_number] = True
-    with caplog.at_level(logging.WARNING):
-        loaded = facade.load_from_forward_model(ensemble, realizations, 0)
-    assert (
-        "Realization: 0, load warning: 200 inconsistencies in time map, first: "
-        "Time mismatch for response time: 2010-01-10 00:00:00, last: Time mismatch "
-        f"for response time: 2015-06-23 00:00:00 from: {run_path.absolute()}"
-        f"/SNAKE_OIL_FIELD.UNSMRY"
-    ) in caplog.messages
-    assert loaded == 1
-
-
 @pytest.mark.usefixtures("copy_snake_oil_case_storage")
 def test_load_forward_model(snake_oil_default_storage):
     """