diff --git a/blocks/conditional-panel-color/index.js b/blocks/conditional-panel-color/index.js
new file mode 100644
index 00000000000000..a9828547387840
--- /dev/null
+++ b/blocks/conditional-panel-color/index.js
@@ -0,0 +1,24 @@
+/**
+ * External dependencies
+ */
+import { isEmpty } from 'lodash';
+
+/**
+ * WordPress dependencies
+ */
+import { ifCondition, PanelColor, withContext } from '@wordpress/components';
+import { compose } from '@wordpress/element';
+
+export default compose(
+	withContext( 'editor' )(
+		( settings, props ) => ( {
+			colors: props.colors || settings.colors,
+			disableCustomColors: props.disableCustomColors !== undefined ?
+				props.disableCustomColors :
+				settings.disableCustomColors,
+		} )
+	),
+	ifCondition(
+		( { colors, disableCustomColors } ) => ! ( isEmpty( colors ) && disableCustomColors )
+	)
+)( PanelColor );
diff --git a/blocks/library/button/index.js b/blocks/library/button/index.js
index fdcf21f6f38442..6abe851538b710 100644
--- a/blocks/library/button/index.js
+++ b/blocks/library/button/index.js
@@ -3,7 +3,7 @@
  */
 import { __ } from '@wordpress/i18n';
 import { Component } from '@wordpress/element';
-import { Dashicon, IconButton, PanelColor, ToggleControl, withFallbackStyles } from '@wordpress/components';
+import { Dashicon, IconButton, ToggleControl, withFallbackStyles } from '@wordpress/components';
 
 /**
  * Internal dependencies
@@ -15,6 +15,7 @@ import UrlInput from '../../url-input';
 import BlockControls from '../../block-controls';
 import BlockAlignmentToolbar from '../../block-alignment-toolbar';
 import ColorPalette from '../../color-palette';
+import ConditionalPanelColor from '../../conditional-panel-color';
 import ContrastChecker from '../../contrast-checker';
 import InspectorControls from '../../inspector-controls';
 
@@ -101,18 +102,18 @@ class ButtonBlock extends Component {
 							checked={ !! clear }
 							onChange={ this.toggleClear }
 						/>
-						<PanelColor title={ __( 'Background Color' ) } colorValue={ color } >
+						<ConditionalPanelColor title={ __( 'Background Color' ) } colorValue={ color } >
 							<ColorPalette
 								value={ color }
 								onChange={ ( colorValue ) => setAttributes( { color: colorValue } ) }
 							/>
-						</PanelColor>
-						<PanelColor title={ __( 'Text Color' ) } colorValue={ textColor } >
+						</ConditionalPanelColor>
+						<ConditionalPanelColor title={ __( 'Text Color' ) } colorValue={ textColor } >
 							<ColorPalette
 								value={ textColor }
 								onChange={ ( colorValue ) => setAttributes( { textColor: colorValue } ) }
 							/>
-						</PanelColor>
+						</ConditionalPanelColor>
 						{ this.nodeRef && <ContrastCheckerWithFallbackStyles
 							node={ this.nodeRef }
 							textColor={ textColor }
diff --git a/blocks/library/paragraph/index.js b/blocks/library/paragraph/index.js
index 361cabb39fc28f..ea57fa59115e6d 100644
--- a/blocks/library/paragraph/index.js
+++ b/blocks/library/paragraph/index.js
@@ -11,7 +11,6 @@ import { concatChildren, Component, RawHTML } from '@wordpress/element';
 import {
 	Autocomplete,
 	PanelBody,
-	PanelColor,
 	RangeControl,
 	ToggleControl,
 	withFallbackStyles,
@@ -30,6 +29,7 @@ import BlockControls from '../../block-controls';
 import RichText from '../../rich-text';
 import InspectorControls from '../../inspector-controls';
 import ColorPalette from '../../color-palette';
+import ConditionalPanelColor from '../../conditional-panel-color';
 import ContrastChecker from '../../contrast-checker';
 
 const { getComputedStyle } = window;
@@ -133,18 +133,18 @@ class ParagraphBlock extends Component {
 							allowReset
 						/>
 					</PanelBody>
-					<PanelColor title={ __( 'Background Color' ) } colorValue={ backgroundColor } initialOpen={ false }>
+					<ConditionalPanelColor title={ __( 'Background Color' ) } colorValue={ backgroundColor } initialOpen={ false }>
 						<ColorPalette
 							value={ backgroundColor }
 							onChange={ ( colorValue ) => setAttributes( { backgroundColor: colorValue } ) }
 						/>
-					</PanelColor>
-					<PanelColor title={ __( 'Text Color' ) } colorValue={ textColor } initialOpen={ false }>
+					</ConditionalPanelColor>
+					<ConditionalPanelColor title={ __( 'Text Color' ) } colorValue={ textColor } initialOpen={ false }>
 						<ColorPalette
 							value={ textColor }
 							onChange={ ( colorValue ) => setAttributes( { textColor: colorValue } ) }
 						/>
-					</PanelColor>
+					</ConditionalPanelColor>
 					{ this.nodeRef && <ContrastCheckerWithFallbackStyles
 						node={ this.nodeRef }
 						textColor={ textColor }