diff --git a/packages/block-editor/src/components/rich-text/index.js b/packages/block-editor/src/components/rich-text/index.js
index 4c69023cb70e78..3731e709876e4c 100644
--- a/packages/block-editor/src/components/rich-text/index.js
+++ b/packages/block-editor/src/components/rich-text/index.js
@@ -13,6 +13,7 @@ import {
 	useRef,
 	useCallback,
 	forwardRef,
+	useEffect,
 } from '@wordpress/element';
 import { useDispatch, useSelect } from '@wordpress/data';
 import {
@@ -114,6 +115,157 @@ getAllowedFormats.EMPTY_ARRAY = [];
 
 const isShortcode = ( text ) => regexp( '.*' ).test( text );
 
+function Element( {
+	listBoxId,
+	activeId,
+	onKeyDown,
+	value,
+	onChange,
+	editableProps,
+	TagName,
+	hasActiveFormats,
+	removeEditorOnlyFormats,
+	onReplace,
+	onSplit,
+	multiline,
+	disableLineBreaks,
+	splitValue,
+	onSplitAtEnd,
+	onMerge,
+	onRemove,
+	props,
+} ) {
+	const { isCaretWithinFormattedText } = useSelect( blockEditorStore );
+	const {
+		enterFormattedText,
+		exitFormattedText,
+		__unstableMarkAutomaticChange,
+	} = useDispatch( blockEditorStore );
+
+	useEffect( () => {
+		if ( hasActiveFormats ) {
+			if ( ! isCaretWithinFormattedText() ) {
+				enterFormattedText();
+			}
+		} else if ( isCaretWithinFormattedText() ) {
+			exitFormattedText();
+		}
+	}, [ hasActiveFormats ] );
+
+	function _onKeyDown( event ) {
+		const { keyCode } = event;
+
+		if ( event.defaultPrevented ) {
+			return;
+		}
+
+		if ( event.keyCode === ENTER ) {
+			event.preventDefault();
+
+			const _value = removeEditorOnlyFormats( value );
+			const canSplit = onReplace && onSplit;
+
+			if ( onReplace ) {
+				const transforms = getBlockTransforms( 'from' ).filter(
+					( { type } ) => type === 'enter'
+				);
+				const transformation = findTransform( transforms, ( item ) => {
+					return item.regExp.test( _value.text );
+				} );
+
+				if ( transformation ) {
+					onReplace( [
+						transformation.transform( {
+							content: _value.text,
+						} ),
+					] );
+					__unstableMarkAutomaticChange();
+				}
+			}
+
+			if ( multiline ) {
+				if ( event.shiftKey ) {
+					if ( ! disableLineBreaks ) {
+						onChange( insert( _value, '\n' ) );
+					}
+				} else if ( canSplit && isEmptyLine( _value ) ) {
+					splitValue( _value );
+				} else {
+					onChange( insertLineSeparator( _value ) );
+				}
+			} else {
+				const { text, start, end } = _value;
+				const canSplitAtEnd =
+					onSplitAtEnd && start === end && end === text.length;
+
+				if ( event.shiftKey || ( ! canSplit && ! canSplitAtEnd ) ) {
+					if ( ! disableLineBreaks ) {
+						onChange( insert( _value, '\n' ) );
+					}
+				} else if ( ! canSplit && canSplitAtEnd ) {
+					onSplitAtEnd();
+				} else if ( canSplit ) {
+					splitValue( _value );
+				}
+			}
+		} else if ( keyCode === DELETE || keyCode === BACKSPACE ) {
+			const { start, end, text } = value;
+			const isReverse = keyCode === BACKSPACE;
+
+			// Only process delete if the key press occurs at an uncollapsed edge.
+			if (
+				! isCollapsed( value ) ||
+				hasActiveFormats ||
+				( isReverse && start !== 0 ) ||
+				( ! isReverse && end !== text.length )
+			) {
+				return;
+			}
+
+			if ( onMerge ) {
+				onMerge( ! isReverse );
+			}
+
+			// Only handle remove on Backspace. This serves dual-purpose of being
+			// an intentional user interaction distinguishing between Backspace and
+			// Delete to remove the empty field, but also to avoid merge & remove
+			// causing destruction of two fields (merge, then removed merged).
+			if ( onRemove && isEmpty( value ) && isReverse ) {
+				onRemove( ! isReverse );
+			}
+
+			event.preventDefault();
+		}
+	}
+
+	return (
+		<TagName
+			{ ...editableProps }
+			{ ...props }
+			style={
+				props.style
+					? {
+							...props.style,
+							...editableProps.style,
+					  }
+					: editableProps.style
+			}
+			className={ classnames(
+				classes,
+				props.className,
+				editableProps.className
+			) }
+			aria-autocomplete={ listBoxId ? 'list' : undefined }
+			aria-owns={ listBoxId }
+			aria-activedescendant={ activeId }
+			onKeyDown={ ( event ) => {
+				onKeyDown( event );
+				_onKeyDown( event );
+			} }
+		/>
+	);
+}
+
 function RichTextWrapper(
 	{
 		children,
@@ -174,7 +326,6 @@ function RichTextWrapper(
 	const nativeProps = useNativeProps();
 	const selector = ( select ) => {
 		const {
-			isCaretWithinFormattedText,
 			getSelectionStart,
 			getSelectionEnd,
 			getSettings,
@@ -213,7 +364,6 @@ function RichTextWrapper(
 		}
 
 		return {
-			isCaretWithinFormattedText: isCaretWithinFormattedText(),
 			selectionStart: isSelected ? selectionStart.offset : undefined,
 			selectionEnd: isSelected ? selectionEnd.offset : undefined,
 			isSelected,
@@ -227,7 +377,6 @@ function RichTextWrapper(
 	// retreived from the store on merge.
 	// To do: fix this somehow.
 	const {
-		isCaretWithinFormattedText,
 		selectionStart,
 		selectionEnd,
 		isSelected,
@@ -238,8 +387,6 @@ function RichTextWrapper(
 	} = useSelect( selector );
 	const {
 		__unstableMarkLastChangeAsPersistent,
-		enterFormattedText,
-		exitFormattedText,
 		selectionChange,
 		__unstableMarkAutomaticChange,
 	} = useDispatch( blockEditorStore );
@@ -531,9 +678,6 @@ function RichTextWrapper(
 			__unstableIsSelected={ isSelected }
 			__unstableInputRule={ inputRule }
 			__unstableMultilineTag={ multilineTag }
-			__unstableIsCaretWithinFormattedText={ isCaretWithinFormattedText }
-			__unstableOnEnterFormattedText={ enterFormattedText }
-			__unstableOnExitFormattedText={ exitFormattedText }
 			__unstableOnCreateUndoLevel={ __unstableMarkLastChangeAsPersistent }
 			__unstableMarkAutomaticChange={ __unstableMarkAutomaticChange }
 			__unstableDidAutomaticChange={ didAutomaticChange }
@@ -583,153 +727,54 @@ function RichTextWrapper(
 				onFocus,
 				editableProps,
 				editableTagName: TagName,
-				activeFormats,
+				hasActiveFormats,
 				removeEditorOnlyFormats,
-			} ) => {
-				function _onKeyDown( event ) {
-					const { keyCode } = event;
-
-					if ( event.defaultPrevented ) {
-						return;
-					}
-
-					if ( event.keyCode === ENTER ) {
-						event.preventDefault();
-
-						const _value = removeEditorOnlyFormats( value );
-						const canSplit = onReplace && onSplit;
-
-						if ( onReplace ) {
-							const transforms = getBlockTransforms(
-								'from'
-							).filter( ( { type } ) => type === 'enter' );
-							const transformation = findTransform(
-								transforms,
-								( item ) => {
-									return item.regExp.test( _value.text );
-								}
-							);
-
-							if ( transformation ) {
-								onReplace( [
-									transformation.transform( {
-										content: _value.text,
-									} ),
-								] );
-								__unstableMarkAutomaticChange();
-							}
-						}
-
-						if ( multiline ) {
-							if ( event.shiftKey ) {
-								if ( ! disableLineBreaks ) {
-									onChange( insert( _value, '\n' ) );
-								}
-							} else if ( canSplit && isEmptyLine( _value ) ) {
-								splitValue( _value );
-							} else {
-								onChange( insertLineSeparator( _value ) );
-							}
-						} else {
-							const { text, start, end } = _value;
-							const canSplitAtEnd =
-								onSplitAtEnd &&
-								start === end &&
-								end === text.length;
-
-							if (
-								event.shiftKey ||
-								( ! canSplit && ! canSplitAtEnd )
-							) {
-								if ( ! disableLineBreaks ) {
-									onChange( insert( _value, '\n' ) );
-								}
-							} else if ( ! canSplit && canSplitAtEnd ) {
-								onSplitAtEnd();
-							} else if ( canSplit ) {
-								splitValue( _value );
-							}
-						}
-					} else if ( keyCode === DELETE || keyCode === BACKSPACE ) {
-						const { start, end, text } = value;
-						const isReverse = keyCode === BACKSPACE;
-
-						// Only process delete if the key press occurs at an uncollapsed edge.
-						if (
-							! isCollapsed( value ) ||
-							activeFormats.length ||
-							( isReverse && start !== 0 ) ||
-							( ! isReverse && end !== text.length )
-						) {
-							return;
-						}
-
-						if ( onMerge ) {
-							onMerge( ! isReverse );
-						}
-
-						// Only handle remove on Backspace. This serves dual-purpose of being
-						// an intentional user interaction distinguishing between Backspace and
-						// Delete to remove the empty field, but also to avoid merge & remove
-						// causing destruction of two fields (merge, then removed merged).
-						if ( onRemove && isEmpty( value ) && isReverse ) {
-							onRemove( ! isReverse );
-						}
-
-						event.preventDefault();
-					}
-				}
-
-				return (
-					<>
-						{ children && children( { value, onChange, onFocus } ) }
-						{ nestedIsSelected && hasFormats && (
-							<FormatToolbarContainer
-								inline={ inlineToolbar }
-								anchorRef={ fallbackRef.current }
+			} ) => (
+				<>
+					{ children && children( { value, onChange, onFocus } ) }
+					{ nestedIsSelected && hasFormats && (
+						<FormatToolbarContainer
+							inline={ inlineToolbar }
+							anchorRef={ fallbackRef.current }
+						/>
+					) }
+					{ nestedIsSelected && <RemoveBrowserShortcuts /> }
+					<Autocomplete
+						onReplace={ onReplace }
+						completers={ autocompleters }
+						record={ value }
+						onChange={ onChange }
+						isSelected={ nestedIsSelected }
+						contentRef={ fallbackRef }
+					>
+						{ ( { listBoxId, activeId, onKeyDown } ) => (
+							<Element
+								{ ...{
+									listBoxId,
+									activeId,
+									onKeyDown,
+									value,
+									onChange,
+									editableProps,
+									TagName,
+									hasActiveFormats,
+									removeEditorOnlyFormats,
+									onReplace,
+									onSplit,
+									__unstableMarkAutomaticChange,
+									multiline,
+									disableLineBreaks,
+									splitValue,
+									onSplitAtEnd,
+									onMerge,
+									onRemove,
+									props,
+								} }
 							/>
 						) }
-						{ nestedIsSelected && <RemoveBrowserShortcuts /> }
-						<Autocomplete
-							onReplace={ onReplace }
-							completers={ autocompleters }
-							record={ value }
-							onChange={ onChange }
-							isSelected={ nestedIsSelected }
-							contentRef={ fallbackRef }
-						>
-							{ ( { listBoxId, activeId, onKeyDown } ) => (
-								<TagName
-									{ ...editableProps }
-									{ ...props }
-									style={
-										props.style
-											? {
-													...props.style,
-													...editableProps.style,
-											  }
-											: editableProps.style
-									}
-									className={ classnames(
-										classes,
-										props.className,
-										editableProps.className
-									) }
-									aria-autocomplete={
-										listBoxId ? 'list' : undefined
-									}
-									aria-owns={ listBoxId }
-									aria-activedescendant={ activeId }
-									onKeyDown={ ( event ) => {
-										onKeyDown( event );
-										_onKeyDown( event );
-									} }
-								/>
-							) }
-						</Autocomplete>
-					</>
-				);
-			} }
+					</Autocomplete>
+				</>
+			) }
 		</RichText>
 	);
 
diff --git a/packages/e2e-tests/specs/editor/various/rich-text.test.js b/packages/e2e-tests/specs/editor/various/rich-text.test.js
index 282d42a8238dfe..c728b937e3e4da 100644
--- a/packages/e2e-tests/specs/editor/various/rich-text.test.js
+++ b/packages/e2e-tests/specs/editor/various/rich-text.test.js
@@ -427,4 +427,26 @@ describe( 'RichText', () => {
 		// Expect '1🍓'.
 		expect( await getEditedPostContent() ).toMatchSnapshot();
 	} );
+
+	it( 'should show/hide toolbar when entering/exiting format', async () => {
+		const blockToolbarSelector = '.block-editor-block-toolbar';
+		await clickBlockAppender();
+		await page.keyboard.type( '1' );
+		expect( await page.$( blockToolbarSelector ) ).toBe( null );
+		await pressKeyWithModifier( 'primary', 'b' );
+		expect( await page.$( blockToolbarSelector ) ).not.toBe( null );
+		await page.keyboard.type( '2' );
+		expect( await page.$( blockToolbarSelector ) ).not.toBe( null );
+		await pressKeyWithModifier( 'primary', 'b' );
+		expect( await page.$( blockToolbarSelector ) ).toBe( null );
+		await page.keyboard.type( '3' );
+		await page.keyboard.press( 'ArrowLeft' );
+		expect( await page.$( blockToolbarSelector ) ).toBe( null );
+		await page.keyboard.press( 'ArrowLeft' );
+		expect( await page.$( blockToolbarSelector ) ).not.toBe( null );
+		await page.keyboard.press( 'ArrowLeft' );
+		expect( await page.$( blockToolbarSelector ) ).not.toBe( null );
+		await page.keyboard.press( 'ArrowLeft' );
+		expect( await page.$( blockToolbarSelector ) ).toBe( null );
+	} );
 } );
diff --git a/packages/rich-text/src/component/index.js b/packages/rich-text/src/component/index.js
index 80df45f96de54c..e4f17e5206572a 100644
--- a/packages/rich-text/src/component/index.js
+++ b/packages/rich-text/src/component/index.js
@@ -67,9 +67,6 @@ function RichText(
 		__unstableMarkAutomaticChange: markAutomaticChange,
 		__unstableAllowPrefixTransformations: allowPrefixTransformations,
 		__unstableUndo: undo,
-		__unstableIsCaretWithinFormattedText: isCaretWithinFormattedText,
-		__unstableOnEnterFormattedText: onEnterFormattedText,
-		__unstableOnExitFormattedText: onExitFormattedText,
 		__unstableOnCreateUndoLevel: onCreateUndoLevel,
 		__unstableIsSelected: isSelected,
 	},
@@ -368,9 +365,6 @@ function RichText(
 				markAutomaticChange,
 				isSelected,
 				disabled,
-				isCaretWithinFormattedText,
-				onEnterFormattedText,
-				onExitFormattedText,
 				onSelectionChange,
 				setActiveFormats,
 			} ),
@@ -401,7 +395,7 @@ function RichText(
 					onFocus: focus,
 					editableProps,
 					editableTagName: TagName,
-					activeFormats,
+					hasActiveFormats: activeFormats.length,
 					removeEditorOnlyFormats,
 				} ) }
 			{ ! children && <TagName { ...editableProps } /> }
diff --git a/packages/rich-text/src/component/use-input-and-selection.js b/packages/rich-text/src/component/use-input-and-selection.js
index 25f71b41cd9d50..125fe0fbbd0ed2 100644
--- a/packages/rich-text/src/component/use-input-and-selection.js
+++ b/packages/rich-text/src/component/use-input-and-selection.js
@@ -178,9 +178,6 @@ export function useInputAndSelection( props ) {
 				createRecord,
 				isSelected,
 				disabled,
-				isCaretWithinFormattedText,
-				onEnterFormattedText,
-				onExitFormattedText,
 				onSelectionChange,
 				setActiveFormats,
 			} = propsRef.current;
@@ -239,15 +236,6 @@ export function useInputAndSelection( props ) {
 			// Update the value with the new active formats.
 			newValue.activeFormats = newActiveFormats;
 
-			if ( ! isCaretWithinFormattedText && newActiveFormats.length ) {
-				onEnterFormattedText();
-			} else if (
-				isCaretWithinFormattedText &&
-				! newActiveFormats.length
-			) {
-				onExitFormattedText();
-			}
-
 			// It is important that the internal value is updated first,
 			// otherwise the value will be wrong on render!
 			record.current = newValue;
@@ -303,15 +291,6 @@ export function useInputAndSelection( props ) {
 				setActiveFormats( EMPTY_ACTIVE_FORMATS );
 			} else {
 				onSelectionChange( record.current.start, record.current.end );
-				setActiveFormats(
-					getActiveFormats(
-						{
-							...record.current,
-							activeFormats: undefined,
-						},
-						EMPTY_ACTIVE_FORMATS
-					)
-				);
 			}
 
 			// Update selection as soon as possible, which is at the next animation