diff --git a/Cargo.lock b/Cargo.lock
index 32e1dfb5..ca73211a 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -23,6 +23,12 @@ version = "0.13.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8"
 
+[[package]]
+name = "base64"
+version = "0.21.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a4a4ddaa51a5bc52a6948f74c06d20aaaddb71924eab79b8c97a8c556e942d6a"
+
 [[package]]
 name = "basic-toml"
 version = "0.1.1"
@@ -473,7 +479,7 @@ version = "0.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "300a51053b1cb55c80b7a9fde4120726ddf25ca241a1cbb926626f62fb136bff"
 dependencies = [
- "base64",
+ "base64 0.13.1",
  "bitflags",
  "serde",
 ]
@@ -559,7 +565,7 @@ dependencies = [
 name = "serde_with"
 version = "2.3.3"
 dependencies = [
- "base64",
+ "base64 0.21.0",
  "chrono",
  "doc-comment",
  "expect-test",
diff --git a/serde_with/CHANGELOG.md b/serde_with/CHANGELOG.md
index 52dc9759..9225efb3 100644
--- a/serde_with/CHANGELOG.md
+++ b/serde_with/CHANGELOG.md
@@ -7,6 +7,19 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ## [Unreleased]
 
+This breaking release should not impact most users.
+It only affects custom character sets used for base64 of which there are no instances on GitHub.
+
+### Changed
+
+* Upgrade base64 to v0.21 (#543)
+    Thanks to @jeff-hiner for submitting the PR.
+
+    Remove support for custom character sets.
+    This is technically a breaking change.
+    A code search on GitHub revealed no instances of anyone using that, and `serde_with` ships with many predefined character sets.
+    The removal means that future base64 upgrade will no longer be breaking changes.
+
 ## [2.3.3] - 2023-04-27
 
 ### Changed
diff --git a/serde_with/Cargo.toml b/serde_with/Cargo.toml
index 2a377a83..4e1896f3 100644
--- a/serde_with/Cargo.toml
+++ b/serde_with/Cargo.toml
@@ -60,7 +60,7 @@ time_0_3 = ["dep:time_0_3"]
 
 # When adding new optional dependencies update the documentation in feature-flags.md
 [dependencies]
-base64 = {version = "0.13.0", optional = true, default-features = false}
+base64 = {version = "0.21.0", optional = true, default-features = false}
 chrono_0_4 = {package = "chrono", version = "0.4.20", optional = true, default-features = false, features = ["serde"]}
 doc-comment = {version = "0.3.3", optional = true}
 hex = {version = "0.4.3", optional = true, default-features = false}
diff --git a/serde_with/src/base64.rs b/serde_with/src/base64.rs
index 1b221d3a..0e52e0c4 100644
--- a/serde_with/src/base64.rs
+++ b/serde_with/src/base64.rs
@@ -12,7 +12,7 @@ use crate::prelude::*;
 /// It works on any type implementing `AsRef<[u8]>` for serialization and `TryFrom<Vec<u8>>` for deserialization.
 ///
 /// The type allows customizing the character set and the padding behavior.
-/// The `CHARSET` is a type implementing [`CharacterSet`].
+/// The `ALPHABET` is a type implementing [`Alphabet`].
 /// `PADDING` specifies if serializing should emit padding.
 /// Deserialization always supports padded and unpadded formats.
 /// [`formats::Padded`] emits padding and [`formats::Unpadded`] leaves it off.
@@ -62,54 +62,71 @@ use crate::prelude::*;
 
 // The padding might be better as `const PADDING: bool = true`
 // https://blog.rust-lang.org/inside-rust/2021/09/06/Splitting-const-generics.html#featureconst_generics_default/
-pub struct Base64<CHARSET: CharacterSet = Standard, PADDING: formats::Format = formats::Padded>(
-    PhantomData<(CHARSET, PADDING)>,
+pub struct Base64<ALPHABET: Alphabet = Standard, PADDING: formats::Format = formats::Padded>(
+    PhantomData<(ALPHABET, PADDING)>,
 );
 
-impl<T, CHARSET> SerializeAs<T> for Base64<CHARSET, formats::Padded>
+impl<T, ALPHABET> SerializeAs<T> for Base64<ALPHABET, formats::Padded>
 where
     T: AsRef<[u8]>,
-    CHARSET: CharacterSet,
+    ALPHABET: Alphabet,
 {
     fn serialize_as<S>(source: &T, serializer: S) -> Result<S::Ok, S::Error>
     where
         S: Serializer,
     {
-        ::base64::encode_config(source, ::base64::Config::new(CHARSET::charset(), true))
-            .serialize(serializer)
+        use ::base64::Engine as _;
+
+        ::base64::engine::GeneralPurpose::new(
+            &ALPHABET::charset(),
+            ::base64::engine::general_purpose::PAD,
+        )
+        .encode(source)
+        .serialize(serializer)
     }
 }
 
-impl<T, CHARSET> SerializeAs<T> for Base64<CHARSET, formats::Unpadded>
+impl<T, ALPHABET> SerializeAs<T> for Base64<ALPHABET, formats::Unpadded>
 where
     T: AsRef<[u8]>,
-    CHARSET: CharacterSet,
+    ALPHABET: Alphabet,
 {
     fn serialize_as<S>(source: &T, serializer: S) -> Result<S::Ok, S::Error>
     where
         S: Serializer,
     {
-        ::base64::encode_config(source, ::base64::Config::new(CHARSET::charset(), false))
-            .serialize(serializer)
+        use ::base64::Engine as _;
+
+        ::base64::engine::GeneralPurpose::new(
+            &ALPHABET::charset(),
+            ::base64::engine::general_purpose::NO_PAD,
+        )
+        .encode(source)
+        .serialize(serializer)
     }
 }
 
-impl<'de, T, CHARSET, FORMAT> DeserializeAs<'de, T> for Base64<CHARSET, FORMAT>
+// Our decoders uniformly do not care about padding.
+const PAD_INDIFFERENT: ::base64::engine::GeneralPurposeConfig =
+    ::base64::engine::GeneralPurposeConfig::new()
+        .with_decode_padding_mode(::base64::engine::DecodePaddingMode::Indifferent);
+
+impl<'de, T, ALPHABET, FORMAT> DeserializeAs<'de, T> for Base64<ALPHABET, FORMAT>
 where
     T: TryFrom<Vec<u8>>,
-    CHARSET: CharacterSet,
+    ALPHABET: Alphabet,
     FORMAT: formats::Format,
 {
     fn deserialize_as<D>(deserializer: D) -> Result<T, D::Error>
     where
         D: Deserializer<'de>,
     {
-        struct Helper<T, CHARSET>(PhantomData<(T, CHARSET)>);
+        struct Helper<T, ALPHABET>(PhantomData<(T, ALPHABET)>);
 
-        impl<'de, T, CHARSET> Visitor<'de> for Helper<T, CHARSET>
+        impl<'de, T, ALPHABET> Visitor<'de> for Helper<T, ALPHABET>
         where
             T: TryFrom<Vec<u8>>,
-            CHARSET: CharacterSet,
+            ALPHABET: Alphabet,
         {
             type Value = T;
 
@@ -121,11 +138,12 @@ where
             where
                 E: DeError,
             {
-                let bytes = ::base64::decode_config(
-                    value,
-                    ::base64::Config::new(CHARSET::charset(), false),
-                )
-                .map_err(DeError::custom)?;
+                use ::base64::Engine as _;
+
+                let bytes =
+                    ::base64::engine::GeneralPurpose::new(&ALPHABET::charset(), PAD_INDIFFERENT)
+                        .decode(value)
+                        .map_err(DeError::custom)?;
 
                 let length = bytes.len();
                 bytes.try_into().map_err(|_e: T::Error| {
@@ -136,25 +154,32 @@ where
             }
         }
 
-        deserializer.deserialize_str(Helper::<T, CHARSET>(PhantomData))
+        deserializer.deserialize_str(Helper::<T, ALPHABET>(PhantomData))
     }
 }
 
-/// A base64 character set from [this list](base64::CharacterSet).
-pub trait CharacterSet {
-    /// Return a specific character set.
-    ///
-    /// Return one enum variant of the [`base64::CharacterSet`](base64::CharacterSet) enum.
-    fn charset() -> ::base64::CharacterSet;
+mod sealed {
+    pub trait Sealed {}
+    impl Sealed for super::Standard {}
+    impl Sealed for super::UrlSafe {}
+    impl Sealed for super::Crypt {}
+    impl Sealed for super::Bcrypt {}
+    impl Sealed for super::ImapMutf7 {}
+    impl Sealed for super::BinHex {}
 }
 
+/// A base64 alphabet
+pub trait Alphabet: sealed::Sealed {
+    /// Return a specific alphabet.
+    fn charset() -> ::base64::alphabet::Alphabet;
+}
 /// The standard character set (uses `+` and `/`).
 ///
 /// See [RFC 3548](https://tools.ietf.org/html/rfc3548#section-3).
 pub struct Standard;
-impl CharacterSet for Standard {
-    fn charset() -> ::base64::CharacterSet {
-        ::base64::CharacterSet::Standard
+impl Alphabet for Standard {
+    fn charset() -> ::base64::alphabet::Alphabet {
+        ::base64::alphabet::STANDARD
     }
 }
 
@@ -162,9 +187,9 @@ impl CharacterSet for Standard {
 ///
 /// See [RFC 3548](https://tools.ietf.org/html/rfc3548#section-3).
 pub struct UrlSafe;
-impl CharacterSet for UrlSafe {
-    fn charset() -> ::base64::CharacterSet {
-        ::base64::CharacterSet::UrlSafe
+impl Alphabet for UrlSafe {
+    fn charset() -> ::base64::alphabet::Alphabet {
+        ::base64::alphabet::URL_SAFE
     }
 }
 
@@ -172,17 +197,17 @@ impl CharacterSet for UrlSafe {
 ///
 /// Not standardized, but folk wisdom on the net asserts that this alphabet is what crypt uses.
 pub struct Crypt;
-impl CharacterSet for Crypt {
-    fn charset() -> ::base64::CharacterSet {
-        ::base64::CharacterSet::Crypt
+impl Alphabet for Crypt {
+    fn charset() -> ::base64::alphabet::Alphabet {
+        ::base64::alphabet::CRYPT
     }
 }
 
 /// The bcrypt character set (uses `./ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789`).
 pub struct Bcrypt;
-impl CharacterSet for Bcrypt {
-    fn charset() -> ::base64::CharacterSet {
-        ::base64::CharacterSet::Bcrypt
+impl Alphabet for Bcrypt {
+    fn charset() -> ::base64::alphabet::Alphabet {
+        ::base64::alphabet::BCRYPT
     }
 }
 
@@ -190,9 +215,9 @@ impl CharacterSet for Bcrypt {
 ///
 /// See [RFC 3501](https://tools.ietf.org/html/rfc3501#section-5.1.3).
 pub struct ImapMutf7;
-impl CharacterSet for ImapMutf7 {
-    fn charset() -> ::base64::CharacterSet {
-        ::base64::CharacterSet::ImapMutf7
+impl Alphabet for ImapMutf7 {
+    fn charset() -> ::base64::alphabet::Alphabet {
+        ::base64::alphabet::IMAP_MUTF7
     }
 }
 
@@ -200,8 +225,8 @@ impl CharacterSet for ImapMutf7 {
 ///
 /// See [BinHex 4.0 Definition](http://files.stairways.com/other/binhex-40-specs-info.txt).
 pub struct BinHex;
-impl CharacterSet for BinHex {
-    fn charset() -> ::base64::CharacterSet {
-        ::base64::CharacterSet::BinHex
+impl Alphabet for BinHex {
+    fn charset() -> ::base64::alphabet::Alphabet {
+        ::base64::alphabet::BIN_HEX
     }
 }