diff --git a/parser.go b/parser.go
index f4386fba..e24c5842 100644
--- a/parser.go
+++ b/parser.go
@@ -140,23 +140,33 @@ func (p *Parser) ParseUnverified(tokenString string, claims Claims) (token *Toke
 	}
 
 	// parse Claims
-	var claimBytes []byte
 	token.Claims = claims
 
-	if claimBytes, err = p.DecodeSegment(parts[1]); err != nil {
+	claimBytes, err := p.DecodeSegment(parts[1])
+	if err != nil {
 		return token, parts, newError("could not base64 decode claim", ErrTokenMalformed, err)
 	}
-	dec := json.NewDecoder(bytes.NewBuffer(claimBytes))
-	if p.useJSONNumber {
-		dec.UseNumber()
-	}
-	// JSON Decode.  Special case for map type to avoid weird pointer behavior
-	if c, ok := token.Claims.(MapClaims); ok {
-		err = dec.Decode(&c)
+
+	// If `useJSONNumber` is enabled then we must use *json.Decoder to decode
+	// the claims. However, this comes with a performance penalty so only use
+	// it if we must and, otherwise, simple use json.Unmarshal.
+	if !p.useJSONNumber {
+		// JSON Unmarshal. Special case for map type to avoid weird pointer behavior.
+		if c, ok := token.Claims.(MapClaims); ok {
+			err = json.Unmarshal(claimBytes, &c)
+		} else {
+			err = json.Unmarshal(claimBytes, &claims)
+		}
 	} else {
-		err = dec.Decode(&claims)
+		dec := json.NewDecoder(bytes.NewBuffer(claimBytes))
+		dec.UseNumber()
+		// JSON Decode. Special case for map type to avoid weird pointer behavior.
+		if c, ok := token.Claims.(MapClaims); ok {
+			err = dec.Decode(&c)
+		} else {
+			err = dec.Decode(&claims)
+		}
 	}
-	// Handle decode error
 	if err != nil {
 		return token, parts, newError("could not JSON decode claim", ErrTokenMalformed, err)
 	}