diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 888711e..f6b018a 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -16,8 +16,8 @@ jobs:
       matrix:
         include:
           - pair:
-              elixir: 1.5.3
-              otp: 19.3.6.13
+              elixir: "1.11"
+              otp: "21.3"
           - pair:
               elixir: 1.13.4
               otp: 25.0.4
diff --git a/lib/mox.ex b/lib/mox.ex
index 27a1830..27e034b 100644
--- a/lib/mox.ex
+++ b/lib/mox.ex
@@ -269,6 +269,9 @@ defmodule Mox do
   """
   @type t() :: module()
 
+  @timeout 30000
+  @this {:global, Mox.Server}
+
   defmodule UnexpectedCallError do
     defexception [:message]
   end
@@ -289,7 +292,9 @@ defmodule Mox do
 
   """
   @spec set_mox_private(term()) :: :ok
-  def set_mox_private(_context \\ %{}), do: Mox.Server.set_mode(self(), :private)
+  def set_mox_private(_context \\ %{}) do
+    NimbleOwnership.set_mode_to_private(@this)
+  end
 
   @doc """
   Sets the Mox to global mode.
@@ -311,7 +316,9 @@ defmodule Mox do
             "If you want to use Mox in global mode, remove \"async: true\" when using ExUnit.Case"
   end
 
-  def set_mox_global(_context), do: Mox.Server.set_mode(self(), :global)
+  def set_mox_global(_context) do
+    NimbleOwnership.set_mode_to_shared(@this, self())
+  end
 
   @doc """
   Chooses the Mox mode based on context.
@@ -668,7 +675,7 @@ defmodule Mox do
       raise ArgumentError, "unknown function #{name}/#{arity} for mock #{inspect(mock)}"
     end
 
-    case Mox.Server.add_expectation(self(), key, value) do
+    case add_expectation(self(), key, value) do
       :ok ->
         :ok
 
@@ -681,7 +688,7 @@ defmodule Mox do
         You cannot define expectations/stubs in a process that has been allowed
         """
 
-      {:error, {:not_global_owner, global_pid}} ->
+      {:error, {:not_shared_owner, global_pid}} ->
         inspected = inspect(self())
 
         raise ArgumentError, """
@@ -726,11 +733,17 @@ defmodule Mox do
       raise ArgumentError, "owner_pid and allowed_pid must be different"
     end
 
-    case Mox.Server.allow(mock, owner_pid, allowed_pid_or_function) do
+    case NimbleOwnership.allow(@this, owner_pid, allowed_pid_or_function, mock, @timeout) do
       :ok ->
         mock
 
-      {:error, {:already_allowed, actual_pid}} ->
+      {:error, %NimbleOwnership.Error{reason: :not_allowed}} ->
+        # Init the mock and re-allow.
+        _ = get_and_update!(owner_pid, mock, &{&1, %{}})
+        allow(mock, owner_pid, allowed_via)
+        mock
+
+      {:error, %NimbleOwnership.Error{reason: {:already_allowed, actual_pid}}} ->
         raise ArgumentError, """
         cannot allow #{inspect(allowed_pid_or_function)} to use #{inspect(mock)} \
         from #{inspect(owner_pid)} \
@@ -742,14 +755,14 @@ defmodule Mox do
         are allowing the same process
         """
 
-      {:error, :expectations_defined} ->
+      {:error, %NimbleOwnership.Error{reason: :already_an_owner}} ->
         raise ArgumentError, """
         cannot allow #{inspect(allowed_pid_or_function)} to use \
         #{inspect(mock)} from #{inspect(owner_pid)} \
         because the process has already defined its own expectations/stubs
         """
 
-      {:error, :in_global_mode} ->
+      {:error, %NimbleOwnership.Error{reason: :cant_allow_in_shared_mode}} ->
         # Already allowed
         mock
     end
@@ -767,10 +780,9 @@ defmodule Mox do
   @spec verify_on_exit!(term()) :: :ok
   def verify_on_exit!(_context \\ %{}) do
     pid = self()
-    Mox.Server.verify_on_exit(pid)
 
     ExUnit.Callbacks.on_exit(Mox, fn ->
-      verify_mock_or_all!(pid, :all, :on_exit)
+      verify_mock_or_all!(pid, :all)
     end)
   end
 
@@ -780,7 +792,7 @@ defmodule Mox do
   """
   @spec verify!() :: :ok
   def verify! do
-    verify_mock_or_all!(self(), :all, :test)
+    verify_mock_or_all!(self(), :all)
   end
 
   @doc """
@@ -789,11 +801,18 @@ defmodule Mox do
   @spec verify!(t()) :: :ok
   def verify!(mock) do
     validate_mock!(mock)
-    verify_mock_or_all!(self(), mock, :test)
+    verify_mock_or_all!(self(), mock)
   end
 
-  defp verify_mock_or_all!(pid, mock, test_or_on_exit) do
-    pending = Mox.Server.verify(pid, mock, test_or_on_exit)
+  defp verify_mock_or_all!(owner_pid, mock_or_all) do
+    all_expectations = NimbleOwnership.get_owned(@this, owner_pid, _default = %{}, @timeout)
+
+    pending =
+      for {_mock, expected_funs} <- all_expectations,
+          {{module, _, _} = key, {count, [_ | _] = calls, _stub}} <- expected_funs,
+          module == mock_or_all or mock_or_all == :all do
+        {key, count, length(calls)}
+      end
 
     messages =
       for {{module, name, arity}, total, pending} <- pending do
@@ -804,7 +823,8 @@ defmodule Mox do
 
     if messages != [] do
       raise VerificationError,
-            "error while verifying mocks for #{inspect(pid)}:\n\n" <> Enum.join(messages, "\n")
+            "error while verifying mocks for #{inspect(owner_pid)}:\n\n" <>
+              Enum.join(messages, "\n")
     end
 
     :ok
@@ -839,9 +859,7 @@ defmodule Mox do
 
   @doc false
   def __dispatch__(mock, name, arity, args) do
-    all_callers = [self() | caller_pids()]
-
-    case Mox.Server.fetch_fun_to_dispatch(all_callers, {mock, name, arity}) do
+    case fetch_fun_to_dispatch([self() | caller_pids()], {mock, name, arity}) do
       :no_expectation ->
         mfa = Exception.format_mfa(mock, name, arity)
 
@@ -891,4 +909,84 @@ defmodule Mox do
       pids when is_list(pids) -> pids
     end
   end
+
+  ## Ownership
+
+  @doc false
+  def start_link_ownership do
+    case NimbleOwnership.start_link(name: @this) do
+      {:error, {:already_started, _}} -> :ignore
+      other -> other
+    end
+  end
+
+  defp add_expectation(owner_pid, {mock, _, _} = key, expectation) do
+    case ensure_pid_can_add_expectation(owner_pid, mock) do
+      :ok ->
+        update_fun = &{:ok, init_or_merge_expectations(&1, key, expectation)}
+        :ok = get_and_update!(owner_pid, mock, update_fun)
+
+      {:error, reason} ->
+        {:error, reason}
+    end
+  end
+
+  defp fetch_fun_to_dispatch(caller_pids, {mock, _, _} = key) do
+    parent = self()
+
+    with {:ok, owner_pid} <- fetch_owner_from_callers(caller_pids, mock) do
+      get_and_update!(owner_pid, mock, fn expectations ->
+        case expectations[key] do
+          nil ->
+            {:no_expectation, expectations}
+
+          {total, [], nil} ->
+            {{:out_of_expectations, total}, expectations}
+
+          {_, [], stub} ->
+            {{ok_or_remote(parent), stub}, expectations}
+
+          {total, [call | calls], stub} ->
+            new_expectations = put_in(expectations[key], {total, calls, stub})
+            {{ok_or_remote(parent), call}, new_expectations}
+        end
+      end)
+    end
+  end
+
+  # Make sure that the owner_pid is either the owner or that the mock
+  # isn't owned yet.
+  defp ensure_pid_can_add_expectation(owner_pid, mock) do
+    case NimbleOwnership.fetch_owner(@this, [owner_pid], mock, @timeout) do
+      :error -> :ok
+      {tag, ^owner_pid} when tag in [:ok, :shared_owner] -> :ok
+      {:shared_owner, other_owner} -> {:error, {:not_shared_owner, other_owner}}
+      {:ok, other_owner} -> {:error, {:currently_allowed, other_owner}}
+    end
+  end
+
+  defp fetch_owner_from_callers(caller_pids, mock) do
+    # If the mock doesn't have an owner, it can't have expectations so we return :no_expectation.
+    case NimbleOwnership.fetch_owner(@this, caller_pids, mock, @timeout) do
+      {tag, owner_pid} when tag in [:shared_owner, :ok] -> {:ok, owner_pid}
+      :error -> :no_expectation
+    end
+  end
+
+  defp get_and_update!(owner_pid, mock, update_fun) do
+    case NimbleOwnership.get_and_update(@this, owner_pid, mock, update_fun, @timeout) do
+      {:ok, return} -> return
+      {:error, %NimbleOwnership.Error{} = error} -> raise error
+    end
+  end
+
+  defp init_or_merge_expectations(current_exps, key, {n, calls, stub} = new_exp)
+       when is_map(current_exps) or is_nil(current_exps) do
+    Map.update(current_exps || %{}, key, new_exp, fn {current_n, current_calls, _current_stub} ->
+      {current_n + n, current_calls ++ calls, stub}
+    end)
+  end
+
+  defp ok_or_remote(source) when node(source) == node(), do: :ok
+  defp ok_or_remote(_source), do: :remote
 end
diff --git a/lib/mox/application.ex b/lib/mox/application.ex
index cb7b52d..638fca2 100644
--- a/lib/mox/application.ex
+++ b/lib/mox/application.ex
@@ -4,7 +4,10 @@ defmodule Mox.Application do
   use Application
 
   def start(_, _) do
-    children = [Mox.Server]
+    children = [
+      %{id: Mox, type: :worker, start: {Mox, :start_link_ownership, []}}
+    ]
+
     Supervisor.start_link(children, name: Mox.Supervisor, strategy: :one_for_one)
   end
 end
diff --git a/lib/mox/server.ex b/lib/mox/server.ex
deleted file mode 100644
index 1520770..0000000
--- a/lib/mox/server.ex
+++ /dev/null
@@ -1,317 +0,0 @@
-defmodule Mox.Server do
-  @moduledoc false
-
-  use GenServer
-  @timeout 30000
-  @this {:global, __MODULE__}
-
-  # API
-
-  def start_link(_options) do
-    case GenServer.start_link(__MODULE__, :ok, name: @this) do
-      {:error, {:already_started, _}} ->
-        :ignore
-
-      other ->
-        other
-    end
-  end
-
-  def add_expectation(owner_pid, key, value) do
-    GenServer.call(@this, {:add_expectation, owner_pid, key, value}, @timeout)
-  end
-
-  def fetch_fun_to_dispatch(caller_pids, key) do
-    GenServer.call(@this, {:fetch_fun_to_dispatch, caller_pids, key, self()}, @timeout)
-  end
-
-  def verify(owner_pid, for, test_or_on_exit) do
-    GenServer.call(@this, {:verify, owner_pid, for, test_or_on_exit}, @timeout)
-  end
-
-  def verify_on_exit(pid) do
-    GenServer.call(@this, {:verify_on_exit, pid}, @timeout)
-  end
-
-  def allow(mock, owner_pid, pid_or_function) do
-    GenServer.call(@this, {:allow, mock, owner_pid, pid_or_function}, @timeout)
-  end
-
-  def set_mode(owner_pid, mode) do
-    GenServer.call(@this, {:set_mode, owner_pid, mode})
-  end
-
-  # Callbacks
-
-  def init(:ok) do
-    {:ok,
-     %{
-       expectations: %{},
-       allowances: %{},
-       deps: %{},
-       mode: :private,
-       global_owner_pid: nil,
-       lazy_calls: false
-     }}
-  end
-
-  def handle_call(msg, _from, state) do
-    # The global process may have terminated and we did not receive
-    # the DOWN message yet, so we always check accordingly if it is alive.
-    with %{mode: :global, global_owner_pid: global_owner_pid} <- state,
-         false <- Process.alive?(global_owner_pid) do
-      handle_call(msg, reset_global_mode(state))
-    else
-      _ -> handle_call(msg, state)
-    end
-  end
-
-  def handle_info({:DOWN, _, _, pid, _}, state) do
-    state =
-      case state.global_owner_pid do
-        ^pid -> reset_global_mode(state)
-        _ -> state
-      end
-
-    state =
-      case state.deps do
-        %{^pid => {:DOWN, _}} -> down(state, pid)
-        %{} -> state
-      end
-
-    {:noreply, state}
-  end
-
-  # handle_call
-
-  def handle_call(
-        {:add_expectation, owner_pid, {mock, _, _} = key, expectation},
-        %{mode: :private} = state
-      ) do
-    if allowance = state.allowances[owner_pid][mock] do
-      {:reply, {:error, {:currently_allowed, allowance}}, state}
-    else
-      state = maybe_add_and_monitor_pid(state, owner_pid)
-
-      state =
-        update_in(state, [:expectations, pid_map(owner_pid)], fn owned_expectations ->
-          Map.update(owned_expectations, key, expectation, &merge_expectation(&1, expectation))
-        end)
-
-      {:reply, :ok, state}
-    end
-  end
-
-  def handle_call(
-        {:add_expectation, owner_pid, {_mock, _, _} = key, expectation},
-        %{mode: :global, global_owner_pid: global_owner_pid} = state
-      ) do
-    if owner_pid != global_owner_pid do
-      {:reply, {:error, {:not_global_owner, global_owner_pid}}, state}
-    else
-      state =
-        update_in(state, [:expectations, pid_map(owner_pid)], fn owned_expectations ->
-          Map.update(owned_expectations, key, expectation, &merge_expectation(&1, expectation))
-        end)
-
-      {:reply, :ok, state}
-    end
-  end
-
-  def handle_call(
-        {:fetch_fun_to_dispatch, caller_pids, {mock, _, _} = key, source},
-        %{mode: :private, lazy_calls: lazy_calls} = state
-      ) do
-    state = maybe_revalidate_lazy_calls(lazy_calls, state)
-
-    owner_pid =
-      Enum.find_value(caller_pids, List.first(caller_pids), fn caller_pid ->
-        cond do
-          state.allowances[caller_pid][mock] -> state.allowances[caller_pid][mock]
-          state.expectations[caller_pid][key] -> caller_pid
-          true -> false
-        end
-      end)
-
-    case state.expectations[owner_pid][key] do
-      nil ->
-        {:reply, :no_expectation, state}
-
-      {total, [], nil} ->
-        {:reply, {:out_of_expectations, total}, state}
-
-      {_, [], stub} ->
-        {:reply, {ok_or_remote(source), stub}, state}
-
-      {total, [call | calls], stub} ->
-        new_state = put_in(state.expectations[owner_pid][key], {total, calls, stub})
-        {:reply, {ok_or_remote(source), call}, new_state}
-    end
-  end
-
-  def handle_call(
-        {:fetch_fun_to_dispatch, _caller_pids, {_mock, _, _} = key, source},
-        %{mode: :global} = state
-      ) do
-    case state.expectations[state.global_owner_pid][key] do
-      nil ->
-        {:reply, :no_expectation, state}
-
-      {total, [], nil} ->
-        {:reply, {:out_of_expectations, total}, state}
-
-      {_, [], stub} ->
-        {:reply, {ok_or_remote(source), stub}, state}
-
-      {total, [call | calls], stub} ->
-        new_state = put_in(state.expectations[state.global_owner_pid][key], {total, calls, stub})
-        {:reply, {ok_or_remote(source), call}, new_state}
-    end
-  end
-
-  def handle_call({:verify, owner_pid, mock, test_or_on_exit}, state) do
-    expectations = state.expectations[owner_pid] || %{}
-
-    pending =
-      for {{module, _, _} = key, {count, [_ | _] = calls, _stub}} <- expectations,
-          module == mock or mock == :all do
-        {key, count, length(calls)}
-      end
-
-    state =
-      if test_or_on_exit == :on_exit do
-        down(state, owner_pid)
-      else
-        state
-      end
-
-    {:reply, pending, state}
-  end
-
-  def handle_call({:verify_on_exit, pid}, state) do
-    state = maybe_add_and_monitor_pid(state, pid, :on_exit, fn {_, deps} -> {:on_exit, deps} end)
-    {:reply, :ok, state}
-  end
-
-  def handle_call({:allow, _, _, _}, %{mode: :global} = state) do
-    {:reply, {:error, :in_global_mode}, state}
-  end
-
-  def handle_call({:allow, mock, owner_pid, pid}, %{mode: :private} = state) do
-    %{allowances: allowances, expectations: expectations} = state
-    owner_pid = state.allowances[owner_pid][mock] || owner_pid
-    allowance = allowances[pid][mock]
-
-    cond do
-      Map.has_key?(expectations, pid) ->
-        {:reply, {:error, :expectations_defined}, state}
-
-      allowance && allowance != owner_pid ->
-        {:reply, {:error, {:already_allowed, allowance}}, state}
-
-      true ->
-        state =
-          maybe_add_and_monitor_pid(state, owner_pid, :DOWN, fn {on, deps} ->
-            {on, [{pid, mock} | deps]}
-          end)
-
-        state =
-          state
-          |> put_in([:allowances, pid_map(pid), mock], owner_pid)
-          |> update_in([:lazy_calls], &(&1 or match?(fun when is_function(fun, 0), pid)))
-
-        {:reply, :ok, state}
-    end
-  end
-
-  def handle_call({:set_mode, owner_pid, :global}, state) do
-    state = maybe_add_and_monitor_pid(state, owner_pid)
-    {:reply, :ok, %{state | mode: :global, global_owner_pid: owner_pid}}
-  end
-
-  def handle_call({:set_mode, _owner_pid, :private}, state) do
-    {:reply, :ok, %{state | mode: :private, global_owner_pid: nil}}
-  end
-
-  # Helper functions
-
-  defp reset_global_mode(state) do
-    %{state | mode: :private, global_owner_pid: nil}
-  end
-
-  defp down(state, pid) do
-    {{_, deps}, state} = pop_in(state.deps[pid])
-    {_, state} = pop_in(state.expectations[pid])
-    {_, state} = pop_in(state.allowances[pid])
-
-    Enum.reduce(deps, state, fn {pid, mock}, acc ->
-      acc.allowances[pid][mock] |> pop_in() |> elem(1)
-    end)
-  end
-
-  defp pid_map(pid) do
-    Access.key(pid, %{})
-  end
-
-  defp maybe_add_and_monitor_pid(state, pid) do
-    maybe_add_and_monitor_pid(state, pid, :DOWN, & &1)
-  end
-
-  defp maybe_add_and_monitor_pid(state, pid, on, fun) do
-    case state.deps do
-      %{^pid => entry} ->
-        put_in(state.deps[pid], fun.(entry))
-
-      _ ->
-        Process.monitor(pid)
-        state = put_in(state.deps[pid], {on, []})
-        state
-    end
-  end
-
-  defp merge_expectation({current_n, current_calls, _current_stub}, {n, calls, stub}) do
-    {current_n + n, current_calls ++ calls, stub}
-  end
-
-  defp ok_or_remote(source) when node(source) == node(), do: :ok
-  defp ok_or_remote(_source), do: :remote
-
-  defp maybe_revalidate_lazy_calls(false, state), do: state
-
-  defp maybe_revalidate_lazy_calls(true, state) do
-    state.allowances
-    |> Enum.reduce({[], [], false}, fn
-      {key, value}, {result, resolved, unresolved} when is_function(key, 0) ->
-        case key.() do
-          pid when is_pid(pid) ->
-            {[{pid, value} | result], [{key, pid} | resolved], unresolved}
-
-          _ ->
-            {[{key, value} | result], resolved, true}
-        end
-
-      kv, {result, resolved, unresolved} ->
-        {[kv | result], resolved, unresolved}
-    end)
-    |> fix_resolved(state)
-  end
-
-  defp fix_resolved({_, [], _}, state), do: state
-
-  defp fix_resolved({allowances, fun_to_pids, lazy_calls}, state) do
-    fun_to_pids = Map.new(fun_to_pids)
-
-    deps =
-      Map.new(state.deps, fn {pid, {fun, deps}} ->
-        deps =
-          Enum.map(deps, fn
-            {fun, mock} when is_function(fun, 0) -> {Map.get(fun_to_pids, fun, fun), mock}
-            other -> other
-          end)
-
-        {pid, {fun, deps}}
-      end)
-
-    %{state | deps: deps, allowances: Map.new(allowances), lazy_calls: lazy_calls}
-  end
-end
diff --git a/mix.exs b/mix.exs
index 2c218ed..f01b98b 100644
--- a/mix.exs
+++ b/mix.exs
@@ -7,7 +7,7 @@ defmodule Mox.MixProject do
     [
       app: :mox,
       version: @version,
-      elixir: "~> 1.5",
+      elixir: "~> 1.11",
       elixirc_paths: elixirc_paths(Mix.env()),
       start_permanent: Mix.env() == :prod,
       name: "Mox",
@@ -30,6 +30,7 @@ defmodule Mox.MixProject do
 
   defp deps do
     [
+      {:nimble_ownership, "~> 0.2.0"},
       {:ex_doc, "~> 0.16", only: :docs}
     ]
   end
diff --git a/mix.lock b/mix.lock
index ab2cb07..ddb068c 100644
--- a/mix.lock
+++ b/mix.lock
@@ -4,5 +4,6 @@
   "makeup": {:hex, :makeup, "1.1.0", "6b67c8bc2882a6b6a445859952a602afc1a41c2e08379ca057c0f525366fc3ca", [:mix], [{:nimble_parsec, "~> 1.2.2 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "0a45ed501f4a8897f580eabf99a2e5234ea3e75a4373c8a52824f6e873be57a6"},
   "makeup_elixir": {:hex, :makeup_elixir, "0.16.0", "f8c570a0d33f8039513fbccaf7108c5d750f47d8defd44088371191b76492b0b", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "28b2cbdc13960a46ae9a8858c4bebdec3c9a6d7b4b9e7f4ed1502f8159f338e7"},
   "makeup_erlang": {:hex, :makeup_erlang, "0.1.1", "3fcb7f09eb9d98dc4d208f49cc955a34218fc41ff6b84df7c75b3e6e533cc65f", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "174d0809e98a4ef0b3309256cbf97101c6ec01c4ab0b23e926a9e17df2077cbb"},
+  "nimble_ownership": {:hex, :nimble_ownership, "0.2.0", "5f09a97ce97a873945be4fe52c583f5649954109e7290b11809d7e3271222f96", [:mix], [], "hexpm", "81fa952d95717ca7ee55231f01374078464436aeab49a848121e5602e84cd97e"},
   "nimble_parsec": {:hex, :nimble_parsec, "1.2.3", "244836e6e3f1200c7f30cb56733fd808744eca61fd182f731eac4af635cc6d0b", [:mix], [], "hexpm", "c8d789e39b9131acf7b99291e93dae60ab48ef14a7ee9d58c6964f59efb570b0"},
 }