diff --git a/test/automation/src/settings.ts b/test/automation/src/settings.ts
index 49cd35a5daa4b..79c3f19988c87 100644
--- a/test/automation/src/settings.ts
+++ b/test/automation/src/settings.ts
@@ -53,4 +53,21 @@ export class SettingsEditor {
 		await this.quickaccess.runCommand('workbench.action.openSettingsJson');
 		await this.editor.waitForEditorFocus('settings.json', 1);
 	}
+
+	async openUserSettingsUI(): Promise<void> {
+		await this.quickaccess.runCommand('workbench.action.openSettings2');
+		await this.code.waitForElement('.settings-editor');
+	}
+
+	async searchSettingsUI(query: string): Promise<void> {
+		await this.openUserSettingsUI();
+		await this.code.waitAndClick('.settings-editor .suggest-input-container .monaco-editor textarea');
+		if (process.platform === 'darwin') {
+			await this.code.dispatchKeybinding('cmd+a');
+		} else {
+			await this.code.dispatchKeybinding('ctrl+a');
+		}
+		await this.code.dispatchKeybinding('Delete');
+		await this.code.waitForTypeInEditor('.settings-editor .suggest-input-container .monaco-editor textarea', query);
+	}
 }
diff --git a/test/smoke/src/areas/preferences/preferences.test.ts b/test/smoke/src/areas/preferences/preferences.test.ts
index 68a6dffdd100b..a6e2d7eaa6f7c 100644
--- a/test/smoke/src/areas/preferences/preferences.test.ts
+++ b/test/smoke/src/areas/preferences/preferences.test.ts
@@ -19,7 +19,7 @@ export function setup(logger: Logger) {
 			await app.code.waitForElements('.line-numbers', false, elements => !!elements.length);
 
 			await app.workbench.settingsEditor.addUserSetting('editor.lineNumbers', '"off"');
-			await app.code.waitForElements('.line-numbers', false, result => !result || result.length === 0);
+			await app.code.waitForElements('.line-numbers', false, elements => !elements || elements.length === 0);
 		});
 
 		it('changes "workbench.action.toggleSidebarPosition" command key binding and verifies it', async function () {
@@ -33,4 +33,34 @@ export function setup(logger: Logger) {
 			await app.workbench.activitybar.waitForActivityBar(ActivityBarPosition.RIGHT);
 		});
 	});
+
+	describe('Settings editor', () => {
+
+		// Shared before/after handling
+		installAllHandlers(logger);
+
+		it('shows a modified indicator on a modified setting', async function () {
+			const app = this.app as Application;
+
+			await app.workbench.settingsEditor.searchSettingsUI('@id:editor.tabSize');
+			await app.code.waitForSetValue('.settings-editor .setting-item-contents .setting-item-control input', '6');
+			await app.code.waitForElement('.settings-editor .setting-item-contents .setting-item-modified-indicator');
+			await app.code.waitForSetValue('.settings-editor .setting-item-contents .setting-item-control input', '4');
+		});
+
+		it('turns off editor line numbers and verifies the live change', async function () {
+			const app = this.app as Application;
+
+			await app.workbench.editors.newUntitledFile();
+			await app.code.dispatchKeybinding('enter');
+			await app.code.waitForElements('.line-numbers', false, elements => !!elements.length);
+
+			await app.workbench.settingsEditor.searchSettingsUI('editor.lineNumbers');
+			await app.code.waitAndClick('.settings-editor .monaco-list-rows .setting-item-control select', 2, 2);
+			await app.code.waitAndClick('.context-view .option-text', 2, 2);
+
+			await app.workbench.editors.selectTab('Untitled-1');
+			await app.code.waitForElements('.line-numbers', false, elements => !elements || elements.length === 0);
+		});
+	});
 }