diff --git a/x-pack/plugins/lens/public/xy_visualization/color_assignment.ts b/x-pack/plugins/lens/public/xy_visualization/color_assignment.ts
index 68c47e11acfc0..e5764eaf0e8c0 100644
--- a/x-pack/plugins/lens/public/xy_visualization/color_assignment.ts
+++ b/x-pack/plugins/lens/public/xy_visualization/color_assignment.ts
@@ -5,9 +5,11 @@
  */
 
 import { uniq, mapValues } from 'lodash';
-import { PaletteOutput } from 'src/plugins/charts/public';
+import { PaletteOutput, PaletteRegistry } from 'src/plugins/charts/public';
 import { Datatable } from 'src/plugins/expressions';
-import { FormatFactory } from '../types';
+import { AccessorConfig, FormatFactory, FramePublicAPI } from '../types';
+import { getColumnToLabelMap } from './state_helpers';
+import { LayerConfig } from './types';
 
 const isPrimitive = (value: unknown): boolean => value != null && typeof value !== 'object';
 
@@ -87,3 +89,48 @@ export function getColorAssignments(
     };
   });
 }
+
+export function getAccessorColorConfig(
+  colorAssignments: ColorAssignments,
+  frame: FramePublicAPI,
+  layer: LayerConfig,
+  sortedAccessors: string[],
+  paletteService: PaletteRegistry
+): AccessorConfig[] {
+  const layerContainsSplits = Boolean(layer.splitAccessor);
+  const currentPalette: PaletteOutput = layer.palette || { type: 'palette', name: 'default' };
+  const totalSeriesCount = colorAssignments[currentPalette.name].totalSeriesCount;
+  return sortedAccessors.map((accessor) => {
+    const currentYConfig = layer.yConfig?.find((yConfig) => yConfig.forAccessor === accessor);
+    if (layerContainsSplits) {
+      return {
+        columnId: accessor as string,
+        triggerIcon: 'disabled',
+      };
+    }
+    const columnToLabel = getColumnToLabelMap(layer, frame.datasourceLayers[layer.layerId]);
+    const rank = colorAssignments[currentPalette.name].getRank(
+      layer,
+      columnToLabel[accessor] || accessor,
+      accessor
+    );
+    const customColor =
+      currentYConfig?.color ||
+      paletteService.get(currentPalette.name).getColor(
+        [
+          {
+            name: columnToLabel[accessor] || accessor,
+            rankAtDepth: rank,
+            totalSeriesAtDepth: totalSeriesCount,
+          },
+        ],
+        { maxDepth: 1, totalSeries: totalSeriesCount },
+        currentPalette.params
+      );
+    return {
+      columnId: accessor as string,
+      triggerIcon: customColor ? 'color' : 'disabled',
+      color: customColor ? customColor : undefined,
+    };
+  });
+}
diff --git a/x-pack/plugins/lens/public/xy_visualization/visualization.tsx b/x-pack/plugins/lens/public/xy_visualization/visualization.tsx
index 872eb179e6a5c..f0dcaf589b1c4 100644
--- a/x-pack/plugins/lens/public/xy_visualization/visualization.tsx
+++ b/x-pack/plugins/lens/public/xy_visualization/visualization.tsx
@@ -10,24 +10,18 @@ import { render } from 'react-dom';
 import { Position } from '@elastic/charts';
 import { I18nProvider } from '@kbn/i18n/react';
 import { i18n } from '@kbn/i18n';
-import { PaletteOutput, PaletteRegistry } from 'src/plugins/charts/public';
+import { PaletteRegistry } from 'src/plugins/charts/public';
 import { DataPublicPluginStart } from 'src/plugins/data/public';
 import { getSuggestions } from './xy_suggestions';
 import { LayerContextMenu, XyToolbar, DimensionEditor } from './xy_config_panel';
-import {
-  Visualization,
-  OperationMetadata,
-  VisualizationType,
-  AccessorConfig,
-  FramePublicAPI,
-} from '../types';
+import { Visualization, OperationMetadata, VisualizationType, AccessorConfig } from '../types';
 import { State, SeriesType, visualizationTypes, LayerConfig } from './types';
-import { getColumnToLabelMap, isHorizontalChart } from './state_helpers';
+import { isHorizontalChart } from './state_helpers';
 import { toExpression, toPreviewExpression, getSortedAccessors } from './to_expression';
 import { LensIconChartBarStacked } from '../assets/chart_bar_stacked';
 import { LensIconChartMixedXy } from '../assets/chart_mixed_xy';
 import { LensIconChartBarHorizontal } from '../assets/chart_bar_horizontal';
-import { ColorAssignments, getColorAssignments } from './color_assignment';
+import { getAccessorColorConfig, getColorAssignments } from './color_assignment';
 
 const defaultIcon = LensIconChartBarStacked;
 const defaultSeriesType = 'bar_stacked';
@@ -328,7 +322,11 @@ export const getXyVisualization = ({
   renderDimensionEditor(domElement, props) {
     render(
       <I18nProvider>
-        <DimensionEditor {...props} />
+        <DimensionEditor
+          {...props}
+          formatFactory={data.fieldFormats.deserialize}
+          paletteService={paletteService}
+        />
       </I18nProvider>,
       domElement
     );
@@ -375,51 +373,6 @@ export const getXyVisualization = ({
   },
 });
 
-function getAccessorColorConfig(
-  colorAssignments: ColorAssignments,
-  frame: FramePublicAPI,
-  layer: LayerConfig,
-  sortedAccessors: string[],
-  paletteService: PaletteRegistry
-): AccessorConfig[] {
-  const layerContainsSplits = Boolean(layer.splitAccessor);
-  const currentPalette: PaletteOutput = layer.palette || { type: 'palette', name: 'default' };
-  const totalSeriesCount = colorAssignments[currentPalette.name].totalSeriesCount;
-  return sortedAccessors.map((accessor) => {
-    const currentYConfig = layer.yConfig?.find((yConfig) => yConfig.forAccessor === accessor);
-    if (layerContainsSplits) {
-      return {
-        columnId: accessor as string,
-        triggerIcon: 'disabled',
-      };
-    }
-    const columnToLabel = getColumnToLabelMap(layer, frame.datasourceLayers[layer.layerId]);
-    const rank = colorAssignments[currentPalette.name].getRank(
-      layer,
-      columnToLabel[accessor] || accessor,
-      accessor
-    );
-    const customColor =
-      currentYConfig?.color ||
-      paletteService.get(currentPalette.name).getColor(
-        [
-          {
-            name: columnToLabel[accessor] || accessor,
-            rankAtDepth: rank,
-            totalSeriesAtDepth: totalSeriesCount,
-          },
-        ],
-        { maxDepth: 1, totalSeries: totalSeriesCount },
-        currentPalette.params
-      );
-    return {
-      columnId: accessor as string,
-      triggerIcon: customColor ? 'color' : 'disabled',
-      color: customColor ? customColor : undefined,
-    };
-  });
-}
-
 function validateLayersForDimension(
   dimension: string,
   layers: LayerConfig[],
diff --git a/x-pack/plugins/lens/public/xy_visualization/xy_config_panel.test.tsx b/x-pack/plugins/lens/public/xy_visualization/xy_config_panel.test.tsx
index 99fbfa058a2de..7b84b990f963a 100644
--- a/x-pack/plugins/lens/public/xy_visualization/xy_config_panel.test.tsx
+++ b/x-pack/plugins/lens/public/xy_visualization/xy_config_panel.test.tsx
@@ -14,6 +14,8 @@ import { FramePublicAPI } from '../types';
 import { State } from './types';
 import { Position } from '@elastic/charts';
 import { createMockFramePublicAPI, createMockDatasource } from '../editor_frame_service/mocks';
+import { chartPluginMock } from 'src/plugins/charts/public/mocks';
+import { EuiColorPicker } from '@elastic/eui';
 
 describe('XY Config panels', () => {
   let frame: FramePublicAPI;
@@ -322,6 +324,8 @@ describe('XY Config panels', () => {
           accessor="bar"
           groupId="left"
           state={{ ...state, layers: [{ ...state.layers[0], seriesType: 'bar_horizontal' }] }}
+          formatFactory={jest.fn()}
+          paletteService={chartPluginMock.createPaletteRegistry()}
         />
       );
 
@@ -343,6 +347,8 @@ describe('XY Config panels', () => {
           accessor="bar"
           groupId="left"
           state={state}
+          formatFactory={jest.fn()}
+          paletteService={chartPluginMock.createPaletteRegistry()}
         />
       );
 
@@ -353,5 +359,82 @@ describe('XY Config panels', () => {
 
       expect(options!.map(({ label }) => label)).toEqual(['Auto', 'Left', 'Right']);
     });
+
+    test('sets the color of a dimension to the color from palette service if not set explicitly', () => {
+      const state = testState();
+      const component = mount(
+        <DimensionEditor
+          layerId={state.layers[0].layerId}
+          frame={{
+            ...frame,
+            activeData: {
+              first: {
+                type: 'datatable',
+                columns: [],
+                rows: [{ bar: 123 }],
+              },
+            },
+          }}
+          setState={jest.fn()}
+          accessor="bar"
+          groupId="left"
+          state={{
+            ...state,
+            layers: [
+              {
+                seriesType: 'bar',
+                layerId: 'first',
+                splitAccessor: undefined,
+                xAccessor: 'foo',
+                accessors: ['bar'],
+              },
+            ],
+          }}
+          formatFactory={jest.fn()}
+          paletteService={chartPluginMock.createPaletteRegistry()}
+        />
+      );
+
+      expect(component.find(EuiColorPicker).prop('color')).toEqual('black');
+    });
+
+    test('uses the overwrite color if set', () => {
+      const state = testState();
+      const component = mount(
+        <DimensionEditor
+          layerId={state.layers[0].layerId}
+          frame={{
+            ...frame,
+            activeData: {
+              first: {
+                type: 'datatable',
+                columns: [],
+                rows: [{ bar: 123 }],
+              },
+            },
+          }}
+          setState={jest.fn()}
+          accessor="bar"
+          groupId="left"
+          state={{
+            ...state,
+            layers: [
+              {
+                seriesType: 'bar',
+                layerId: 'first',
+                splitAccessor: undefined,
+                xAccessor: 'foo',
+                accessors: ['bar'],
+                yConfig: [{ forAccessor: 'bar', color: 'red' }],
+              },
+            ],
+          }}
+          formatFactory={jest.fn()}
+          paletteService={chartPluginMock.createPaletteRegistry()}
+        />
+      );
+
+      expect(component.find(EuiColorPicker).prop('color')).toEqual('red');
+    });
   });
 });
diff --git a/x-pack/plugins/lens/public/xy_visualization/xy_config_panel.tsx b/x-pack/plugins/lens/public/xy_visualization/xy_config_panel.tsx
index a22530c5743b4..cd8a5993d3ecb 100644
--- a/x-pack/plugins/lens/public/xy_visualization/xy_config_panel.tsx
+++ b/x-pack/plugins/lens/public/xy_visualization/xy_config_panel.tsx
@@ -5,7 +5,7 @@
  */
 
 import './xy_config_panel.scss';
-import React, { useState } from 'react';
+import React, { useMemo, useState } from 'react';
 import { i18n } from '@kbn/i18n';
 import { Position } from '@elastic/charts';
 import { debounce } from 'lodash';
@@ -22,10 +22,12 @@ import {
   EuiToolTip,
   EuiIcon,
 } from '@elastic/eui';
+import { PaletteRegistry } from 'src/plugins/charts/public';
 import {
   VisualizationLayerWidgetProps,
   VisualizationToolbarProps,
   VisualizationDimensionEditorProps,
+  FormatFactory,
 } from '../types';
 import {
   State,
@@ -48,6 +50,7 @@ import { AxisSettingsPopover } from './axis_settings_popover';
 import { TooltipWrapper } from './tooltip_wrapper';
 import { getAxesConfiguration } from './axes_configuration';
 import { PalettePicker } from '../shared_components';
+import { getAccessorColorConfig, getColorAssignments } from './color_assignment';
 
 type UnwrapArray<T> = T extends Array<infer P> ? P : T;
 type AxesSettingsConfigKeys = keyof AxesSettingsConfig;
@@ -445,7 +448,12 @@ export function XyToolbar(props: VisualizationToolbarProps<State>) {
 }
 const idPrefix = htmlIdGenerator()();
 
-export function DimensionEditor(props: VisualizationDimensionEditorProps<State>) {
+export function DimensionEditor(
+  props: VisualizationDimensionEditorProps<State> & {
+    formatFactory: FormatFactory;
+    paletteService: PaletteRegistry;
+  }
+) {
   const { state, setState, layerId, accessor } = props;
   const index = state.layers.findIndex((l) => l.layerId === layerId);
   const layer = state.layers[index];
@@ -556,12 +564,37 @@ const ColorPicker = ({
   setState,
   layerId,
   accessor,
-}: VisualizationDimensionEditorProps<State>) => {
+  frame,
+  formatFactory,
+  paletteService,
+}: VisualizationDimensionEditorProps<State> & {
+  formatFactory: FormatFactory;
+  paletteService: PaletteRegistry;
+}) => {
   const index = state.layers.findIndex((l) => l.layerId === layerId);
   const layer = state.layers[index];
   const disabled = !!layer.splitAccessor;
 
-  const [color, setColor] = useState(getSeriesColor(layer, accessor));
+  const overwriteColor = getSeriesColor(layer, accessor);
+  const currentColor = useMemo(() => {
+    if (overwriteColor || !frame.activeData) return overwriteColor;
+
+    const colorAssignments = getColorAssignments(
+      state.layers,
+      { tables: frame.activeData },
+      formatFactory
+    );
+    const mappedAccessors = getAccessorColorConfig(
+      colorAssignments,
+      frame,
+      layer,
+      [accessor],
+      paletteService
+    );
+    return mappedAccessors[0].color;
+  }, [overwriteColor, frame, paletteService, state.layers, accessor, formatFactory, layer]);
+
+  const [color, setColor] = useState(currentColor);
 
   const handleColor: EuiColorPickerProps['onChange'] = (text, output) => {
     setColor(text);
@@ -596,9 +629,9 @@ const ColorPicker = ({
     <EuiColorPicker
       data-test-subj="indexPattern-dimension-colorPicker"
       compressed
-      isClearable
+      isClearable={Boolean(overwriteColor)}
       onChange={handleColor}
-      color={disabled ? '' : color}
+      color={disabled ? '' : color || currentColor}
       disabled={disabled}
       placeholder={i18n.translate('xpack.lens.xyChart.seriesColor.auto', {
         defaultMessage: 'Auto',