diff --git a/polyfill/lib/ecmascript.mjs b/polyfill/lib/ecmascript.mjs
index dd84b6a123..2a5a4a253a 100644
--- a/polyfill/lib/ecmascript.mjs
+++ b/polyfill/lib/ecmascript.mjs
@@ -4877,7 +4877,10 @@ export function AddDuration(
     const TemporalInstant = GetIntrinsic('%Temporal.Instant%');
     const calendar = GetSlot(zonedRelativeTo, CALENDAR);
     const startInstant = GetSlot(zonedRelativeTo, INSTANT);
-    const startDateTime = precalculatedPlainDateTime ?? GetPlainDateTimeFor(timeZoneRec, startInstant, calendar);
+    let startDateTime = precalculatedPlainDateTime;
+    if (largestUnit === 'year' || largestUnit === 'month' || largestUnit === 'week' || largestUnit === 'day') {
+      startDateTime ??= GetPlainDateTimeFor(timeZoneRec, startInstant, calendar);
+    }
     const intermediateNs = AddZonedDateTime(
       startInstant,
       timeZoneRec,
diff --git a/spec/duration.html b/spec/duration.html
index 67bff9e7e6..11c33b1a4a 100644
--- a/spec/duration.html
+++ b/spec/duration.html
@@ -1643,7 +1643,8 @@ <h1>
           1. Return ! CreateDurationRecord(_dateDifference_.[[Years]], _dateDifference_.[[Months]], _dateDifference_.[[Weeks]], _result_.[[Days]], _result_.[[Hours]], _result_.[[Minutes]], _result_.[[Seconds]], _result_.[[Milliseconds]], _result_.[[Microseconds]], _result_.[[Nanoseconds]]).
         1. Assert: _zonedRelativeTo_ is not *undefined*.
         1. If _precalculatedPlainDateTime_ is not present, let _precalculatedPlainDateTime_ be *undefined*.
-        1. If _precalculatedPlainDateTime_ is *undefined*, then
+        1. If _largestUnit_ is *"year"*, or _largestUnit_ is *"month"*, or _largestUnit_ is *"week"*, or _largestUnit_ is *"day"*, let _startDateTimeNeeded_ be *true*; else let _startDateTimeNeeded_ be *false*.
+        1. If _precalculatedPlainDateTime_ is *undefined* and _startDateTimeNeeded_ is *true*, then
           1. Let _startDateTime_ be ? GetPlainDateTimeFor(_timeZoneRec_, _zonedRelativeTo_.[[Nanoseconds]], _calendarRec_.[[Receiver]]).
         1. Else,
           1. Let _startDateTime_ be _precalculatedPlainDateTime_.