diff --git a/packages/react-dom/src/client/ReactDOMComponent.js b/packages/react-dom/src/client/ReactDOMComponent.js
index 899eb47f40194..840de06a34310 100644
--- a/packages/react-dom/src/client/ReactDOMComponent.js
+++ b/packages/react-dom/src/client/ReactDOMComponent.js
@@ -1358,14 +1358,15 @@ export function listenToEventResponderEventTypes(
           // existing passive event listener before we add the
           // active event listener.
           const passiveKey = targetEventType + '_passive';
-          const passiveListener = listenerMap.get(passiveKey);
-          if (passiveListener != null) {
+          const passiveItem = listenerMap.get(passiveKey);
+          if (passiveItem !== undefined) {
             removeTrappedEventListener(
               document,
-              targetEventType,
-              passiveListener,
+              (targetEventType: any),
+              passiveItem.listener,
               true,
             );
+            listenerMap.delete(passiveKey);
           }
         }
         const eventListener = addResponderEventSystemEvent(
@@ -1373,7 +1374,10 @@ export function listenToEventResponderEventTypes(
           targetEventType,
           isPassive,
         );
-        listenerMap.set(eventKey, eventListener);
+        listenerMap.set(eventKey, {
+          passive: isPassive,
+          listener: eventListener,
+        });
       }
     }
   }
diff --git a/packages/react-dom/src/events/DOMEventListenerMap.js b/packages/react-dom/src/events/DOMEventListenerMap.js
index d0f0bbd9a34f8..ec4ad61c3e57c 100644
--- a/packages/react-dom/src/events/DOMEventListenerMap.js
+++ b/packages/react-dom/src/events/DOMEventListenerMap.js
@@ -16,13 +16,18 @@ const PossiblyWeakMap = typeof WeakMap === 'function' ? WeakMap : Map;
 const elementListenerMap:
   // $FlowFixMe Work around Flow bug
   | WeakMap
-  | Map<EventTarget, Map<DOMTopLevelEventType | string, null | (any => void)>> = new PossiblyWeakMap();
+  | Map<EventTarget, ElementListenerMap> = new PossiblyWeakMap();
 
 export type ElementListenerMap = Map<
   DOMTopLevelEventType | string,
-  null | (any => void),
+  ElementListenerMapEntry,
 >;
 
+export type ElementListenerMapEntry = {
+  passive: void | boolean,
+  listener: any => void,
+};
+
 export function getListenerMapForElement(
   target: EventTarget,
 ): ElementListenerMap {
diff --git a/packages/react-dom/src/events/DOMLegacyEventPluginSystem.js b/packages/react-dom/src/events/DOMLegacyEventPluginSystem.js
index 8d0b6c4cf2449..2c9791d38406c 100644
--- a/packages/react-dom/src/events/DOMLegacyEventPluginSystem.js
+++ b/packages/react-dom/src/events/DOMLegacyEventPluginSystem.js
@@ -9,6 +9,7 @@
 
 import type {AnyNativeEvent} from 'legacy-events/PluginModuleType';
 import type {DOMTopLevelEventType} from 'legacy-events/TopLevelEventTypes';
+import type {ElementListenerMap} from '../events/DOMEventListenerMap';
 import type {EventSystemFlags} from 'legacy-events/EventSystemFlags';
 import type {Fiber} from 'react-reconciler/src/ReactFiber';
 import type {PluginModule} from 'legacy-events/PluginModuleType';
@@ -323,57 +324,61 @@ export function legacyListenToEvent(
 export function legacyListenToTopLevelEvent(
   topLevelType: DOMTopLevelEventType,
   mountAt: Document | Element,
-  listenerMap: Map<DOMTopLevelEventType | string, null | (any => void)>,
+  listenerMap: ElementListenerMap,
 ): void {
   if (!listenerMap.has(topLevelType)) {
     switch (topLevelType) {
-      case TOP_SCROLL:
-        legacyTrapCapturedEvent(TOP_SCROLL, mountAt);
+      case TOP_SCROLL: {
+        legacyTrapCapturedEvent(TOP_SCROLL, mountAt, listenerMap);
         break;
+      }
       case TOP_FOCUS:
       case TOP_BLUR:
-        legacyTrapCapturedEvent(TOP_FOCUS, mountAt);
-        legacyTrapCapturedEvent(TOP_BLUR, mountAt);
-        // We set the flag for a single dependency later in this function,
-        // but this ensures we mark both as attached rather than just one.
-        listenerMap.set(TOP_BLUR, null);
-        listenerMap.set(TOP_FOCUS, null);
+        legacyTrapCapturedEvent(TOP_FOCUS, mountAt, listenerMap);
+        legacyTrapCapturedEvent(TOP_BLUR, mountAt, listenerMap);
         break;
       case TOP_CANCEL:
-      case TOP_CLOSE:
+      case TOP_CLOSE: {
         if (isEventSupported(getRawEventName(topLevelType))) {
-          legacyTrapCapturedEvent(topLevelType, mountAt);
+          legacyTrapCapturedEvent(topLevelType, mountAt, listenerMap);
         }
         break;
+      }
       case TOP_INVALID:
       case TOP_SUBMIT:
       case TOP_RESET:
         // We listen to them on the target DOM elements.
         // Some of them bubble so we don't want them to fire twice.
         break;
-      default:
+      default: {
         // By default, listen on the top level to all non-media events.
         // Media events don't bubble so adding the listener wouldn't do anything.
         const isMediaEvent = mediaEventTypes.indexOf(topLevelType) !== -1;
         if (!isMediaEvent) {
-          legacyTrapBubbledEvent(topLevelType, mountAt);
+          legacyTrapBubbledEvent(topLevelType, mountAt, listenerMap);
         }
         break;
+      }
     }
-    listenerMap.set(topLevelType, null);
   }
 }
 
 export function legacyTrapBubbledEvent(
   topLevelType: DOMTopLevelEventType,
   element: Document | Element,
+  listenerMap?: ElementListenerMap,
 ): void {
-  addTrappedEventListener(element, topLevelType, false);
+  const listener = addTrappedEventListener(element, topLevelType, false);
+  if (listenerMap) {
+    listenerMap.set(topLevelType, {passive: undefined, listener});
+  }
 }
 
 export function legacyTrapCapturedEvent(
   topLevelType: DOMTopLevelEventType,
   element: Document | Element,
+  listenerMap: ElementListenerMap,
 ): void {
-  addTrappedEventListener(element, topLevelType, true);
+  const listener = addTrappedEventListener(element, topLevelType, true);
+  listenerMap.set(topLevelType, {passive: undefined, listener});
 }
diff --git a/packages/react-dom/src/events/DOMModernPluginEventSystem.js b/packages/react-dom/src/events/DOMModernPluginEventSystem.js
index fc7706e48dad3..16fac264ea790 100644
--- a/packages/react-dom/src/events/DOMModernPluginEventSystem.js
+++ b/packages/react-dom/src/events/DOMModernPluginEventSystem.js
@@ -9,7 +9,10 @@
 
 import type {AnyNativeEvent} from 'legacy-events/PluginModuleType';
 import type {DOMTopLevelEventType} from 'legacy-events/TopLevelEventTypes';
-import type {ElementListenerMap} from '../events/DOMEventListenerMap';
+import type {
+  ElementListenerMap,
+  ElementListenerMapEntry,
+} from '../events/DOMEventListenerMap';
 import type {EventSystemFlags} from 'legacy-events/EventSystemFlags';
 import type {EventPriority} from 'shared/ReactTypes';
 import type {Fiber} from 'react-reconciler/src/ReactFiber';
@@ -24,7 +27,10 @@ import {plugins} from 'legacy-events/EventPluginRegistry';
 
 import {HostRoot, HostPortal} from 'shared/ReactWorkTags';
 
-import {addTrappedEventListener} from './ReactDOMEventListener';
+import {
+  addTrappedEventListener,
+  removeTrappedEventListener,
+} from './ReactDOMEventListener';
 import getEventTarget from './getEventTarget';
 import {getListenerMapForElement} from './DOMEventListenerMap';
 import {
@@ -153,6 +159,25 @@ function dispatchEventsForPlugins(
   }
 }
 
+function shouldUpgradeListener(
+  listenerEntry: void | ElementListenerMapEntry,
+  passive: void | boolean,
+): boolean {
+  if (listenerEntry === undefined) {
+    return false;
+  }
+  // Upgrade from passive to active.
+  if (passive !== false && listenerEntry.passive) {
+    return true;
+  }
+  // Upgrade from default-active (browser default) to active.
+  if (passive === false && listenerEntry.passive === undefined) {
+    return true;
+  }
+  // Otherwise, do not upgrade
+  return false;
+}
+
 export function listenToTopLevelEvent(
   topLevelType: DOMTopLevelEventType,
   targetContainer: EventTarget,
@@ -160,10 +185,19 @@ export function listenToTopLevelEvent(
   passive?: boolean,
   priority?: EventPriority,
 ): void {
-  // TODO: we need to know if the listenerMap previously was passive
-  // and to check if we need to upgrade to active. This will come in
-  // a useEvent follow up PR.
-  if (!listenerMap.has(topLevelType)) {
+  const listenerEntry = listenerMap.get(topLevelType);
+  const shouldUpgrade = shouldUpgradeListener(listenerEntry, passive);
+  if (listenerEntry === undefined || shouldUpgrade) {
+    // If we should upgrade, then we need to remove the existing trapped
+    // event listener for the target container.
+    if (shouldUpgrade) {
+      removeTrappedEventListener(
+        targetContainer,
+        topLevelType,
+        ((listenerEntry: any): ElementListenerMapEntry).listener,
+        ((listenerEntry: any): ElementListenerMapEntry).passive,
+      );
+    }
     const isCapturePhase = capturePhaseEvents.has(topLevelType);
     const listener = addTrappedEventListener(
       targetContainer,
@@ -173,7 +207,7 @@ export function listenToTopLevelEvent(
       passive,
       priority,
     );
-    listenerMap.set(topLevelType, listener);
+    listenerMap.set(topLevelType, {passive, listener});
   }
 }
 
diff --git a/packages/react-dom/src/events/ReactDOMEventListener.js b/packages/react-dom/src/events/ReactDOMEventListener.js
index d1ad26afbc881..673db4be8487d 100644
--- a/packages/react-dom/src/events/ReactDOMEventListener.js
+++ b/packages/react-dom/src/events/ReactDOMEventListener.js
@@ -255,20 +255,21 @@ export function addTrappedEventListener(
 
 export function removeTrappedEventListener(
   targetContainer: EventTarget,
-  topLevelType: string,
+  topLevelType: DOMTopLevelEventType,
   listener: any => void,
-  passive: boolean,
+  passive: void | boolean,
 ) {
   if (listener.remove != null) {
     listener.remove();
   } else {
+    const rawEventName = getRawEventName(topLevelType);
     if (passiveBrowserEventsSupported) {
-      targetContainer.removeEventListener(topLevelType, listener, {
+      targetContainer.removeEventListener(rawEventName, listener, {
         capture: true,
         passive,
       });
     } else {
-      targetContainer.removeEventListener(topLevelType, listener, true);
+      targetContainer.removeEventListener(rawEventName, listener, true);
     }
   }
 }
diff --git a/packages/react-dom/src/events/ReactDOMEventReplaying.js b/packages/react-dom/src/events/ReactDOMEventReplaying.js
index 5c9d1f2a02464..83348eb6404dc 100644
--- a/packages/react-dom/src/events/ReactDOMEventReplaying.js
+++ b/packages/react-dom/src/events/ReactDOMEventReplaying.js
@@ -244,7 +244,7 @@ function trapReplayableEventForDocument(
         topLevelTypeString,
         false,
       );
-      listenerMap.set(activeEventKey, listener);
+      listenerMap.set(activeEventKey, {passive: false, listener});
     }
   }
 }