diff --git a/amp.php b/amp.php
index 3b35a572800..4aa75ffdfef 100644
--- a/amp.php
+++ b/amp.php
@@ -18,6 +18,7 @@
 require_once( AMP__DIR__ . '/back-compat/back-compat.php' );
 require_once( AMP__DIR__ . '/includes/amp-helper-functions.php' );
 require_once( AMP__DIR__ . '/includes/admin/functions.php' );
+require_once( AMP__DIR__ . '/includes/admin/class-amp-customizer.php' );
 require_once( AMP__DIR__ . '/includes/settings/class-amp-customizer-settings.php' );
 require_once( AMP__DIR__ . '/includes/settings/class-amp-customizer-design-settings.php' );
 
@@ -152,24 +153,16 @@ function amp_render_post( $post_id ) {
  * preview page isn't flagged as an AMP template, the core panels will be re-added and the AMP panel
  * hidden.
  *
- * @internal This callback must be hooked before priority 10 on 'plugins_loaded' to properly unhook
- *           the core panels.
- *
  * @since 0.4
  */
 function _amp_bootstrap_customizer() {
-	/**
-	 * Filter whether to enable the AMP template customizer functionality.
-	 *
-	 * @param bool $enable Whether to enable the AMP customizer. Default true.
-	 */
-	$amp_customizer_enabled = apply_filters( 'amp_customizer_is_enabled', true );
-
-	if ( true === $amp_customizer_enabled ) {
-		amp_init_customizer();
-	}
+	// Drop core panels (menus, widgets) from the AMP customizer
+	// `customize_loaded_components` runs super early so we need to call this regardless of whether the AMP customizer is enabled or not
+	add_filter( 'customize_loaded_components', array( 'AMP_Template_Customizer', '_unregister_core_panels' ) );
+
+	add_action( 'after_setup_theme', 'amp_maybe_init_customizer' );
 }
-add_action( 'plugins_loaded', '_amp_bootstrap_customizer', 9 );
+add_action( 'plugins_loaded', '_amp_bootstrap_customizer', 9 ); // Should be hooked before priority 10 on 'plugins_loaded' to properly unhook core panels.
 
 /**
  * Redirects the old AMP URL to the new AMP URL.
diff --git a/includes/admin/functions.php b/includes/admin/functions.php
index 4a6f00033e4..fd21e348385 100644
--- a/includes/admin/functions.php
+++ b/includes/admin/functions.php
@@ -9,11 +9,17 @@
 /**
  * Sets up the AMP template editor for the Customizer.
  */
-function amp_init_customizer() {
-	require_once( AMP__DIR__ . '/includes/admin/class-amp-customizer.php' );
+function amp_maybe_init_customizer() {
+	/**
+	 * Filter whether to enable the AMP template customizer functionality.
+	 *
+	 * @param bool $enable Whether to enable the AMP customizer. Default true.
+	 */
+	$amp_customizer_enabled = apply_filters( 'amp_customizer_is_enabled', true );
 
-	// Drop core panels (menus, widgets) from the AMP customizer
-	add_filter( 'customize_loaded_components', array( 'AMP_Template_Customizer', '_unregister_core_panels' ) );
+	if ( true !== $amp_customizer_enabled ) {
+		return;
+	}
 
 	// Fire up the AMP Customizer
 	add_action( 'customize_register', array( 'AMP_Template_Customizer', 'init' ), 500 );
diff --git a/readme.md b/readme.md
index 220d0523cea..92608c02aea 100644
--- a/readme.md
+++ b/readme.md
@@ -16,6 +16,20 @@ The plugin ships with a default template that looks nice and clean and we tried
 
 You can tweak small pieces of the template or the entire thing depending on your needs.
 
+### AMP Customizer
+
+The plugin ships with its own Customizer that you can use to tweak various parts of the default template like colors.
+
+#### Disabling the AMP Customizer
+
+If you're using a completely custom template, you may want to disable the AMP Customizer:
+
+```
+add_filter( 'amp_customizer_is_enabled', '__return_false' );
+``` 
+
+Note that this needs to be called before the `after_setup_theme` hook to work.
+
 ### Where Do I Put My Code?
 
 The code snippets below and any other code-level customizations should happen in one of the following locations.