diff --git a/includes/class-simple-local-avatars.php b/includes/class-simple-local-avatars.php
index 3a52bae..f7fe83d 100644
--- a/includes/class-simple-local-avatars.php
+++ b/includes/class-simple-local-avatars.php
@@ -276,10 +276,12 @@ public function get_avatar_data( $args, $id_or_email ) {
 
 		// Local only mode
 		if ( ! $simple_local_avatar_url ) {
-			$default_url = $this->get_default_avatar_url( $args['size'] );
+			$default_url    = $this->get_default_avatar_url( $args['size'] );
+			$avatar_default = get_option( 'avatar_default' );
+
 			if ( ! empty( $this->options['only'] ) ) {
 				$args['url'] = $default_url;
-			} else {
+			} elseif ( 'simple_local_avatar' === $avatar_default ) {
 				$args['default'] = $default_url;
 			}
 		}