diff --git a/packages/components/src/palette-edit/index.js b/packages/components/src/palette-edit/index.js
index 065302acc10865..ec38edcc72a586 100644
--- a/packages/components/src/palette-edit/index.js
+++ b/packages/components/src/palette-edit/index.js
@@ -316,7 +316,7 @@ export default function PaletteEdit( {
 			<PaletteHStackHeader>
 				<PaletteHeading>{ paletteLabel }</PaletteHeading>
 				<PaletteActionsContainer>
-					{ isEditing && (
+					{ hasElements && isEditing && (
 						<DoneButton
 							isSmall
 							onClick={ () => {