diff --git a/src/Microsoft.IdentityModel.Tokens/Base64UrlEncoder.cs b/src/Microsoft.IdentityModel.Tokens/Base64UrlEncoder.cs
index a96179ca15..d30ec32dc8 100644
--- a/src/Microsoft.IdentityModel.Tokens/Base64UrlEncoder.cs
+++ b/src/Microsoft.IdentityModel.Tokens/Base64UrlEncoder.cs
@@ -207,10 +207,12 @@ internal static unsafe byte[] UnsafeDecode(ReadOnlyMemory<char> str)
 
                 if (needReplace)
                 {
+                    Span<char> remaining = charsSpan;
                     int pos;
-                    while ((pos = charsSpan.IndexOfAny(base64UrlCharacter62, base64UrlCharacter63)) >= 0)
+                    while ((pos = remaining.IndexOfAny(base64UrlCharacter62, base64UrlCharacter63)) >= 0)
                     {
-                        charsSpan[pos] = charsSpan[pos] == base64UrlCharacter62 ? base64Character62 : base64Character63;
+                        remaining[pos] = (remaining[pos] == base64UrlCharacter62) ? base64Character62 : base64Character63;
+                        remaining = remaining.Slice(pos + 1);
                     }
                 }