Require conn_quota::units stay on a single shard

This makes the API contract more explicit since we now require the
connection quota doesn't change shards.

Related: #10544

src/v/net/conn_quota.cc

@@ -9,17 +9,22 @@
 
 #include "conn_quota.h"
 
+#include "bytes/oncore.h"
 #include "config/configuration.h"
 #include "config/validators.h"
 #include "hashing/xx.h"
 #include "rpc/logger.h"
 #include "seastar/core/coroutine.hh"
 #include "ssx/future-util.h"
+#include "vassert.h"
 #include "vlog.h"
 
+#include <seastar/core/smp.hh>
+
 namespace net {
 
 conn_quota::units::~units() noexcept {
+    oncore_debug_verify(_verify_shard);
     if (_quotas) {
         (*_quotas).get().put(_addr);
     }

src/v/net/conn_quota.h

@@ -17,6 +17,8 @@
 #include "utils/mutex.h"
 #include "vassert.h"
 
+#include <seastar/core/smp.hh>
+
 #include <absl/container/flat_hash_map.h>
 #include <absl/hash/hash.h>
 
@@ -61,12 +63,18 @@ class conn_quota : public ss::peering_sharded_service<conn_quota> {
         units(units const&) = delete;
         units& operator=(units const&) = delete;
         units(units&& rhs) noexcept
-          : _addr(rhs._addr) {
+          : _addr(rhs._addr)
+#ifndef NDEBUG
+          , _verify_shard(rhs._verify_shard)
+#endif
+        {
             _quotas = std::exchange(rhs._quotas, std::nullopt);
         }
+
         units& operator=(units&& rhs) noexcept {
             _quotas = std::exchange(rhs._quotas, std::nullopt);
             _addr = rhs._addr;
+            expression_in_debug_mode(_verify_shard = rhs._verify_shard);
             return *this;
         }
 
@@ -83,6 +91,7 @@ class conn_quota : public ss::peering_sharded_service<conn_quota> {
     private:
         std::optional<std::reference_wrapper<conn_quota>> _quotas;
         ss::net::inet_address _addr;
+        expression_in_debug_mode(oncore _verify_shard);
     };
 
     using config_fn = std::function<conn_quota_config()>;
@@ -235,4 +244,4 @@ class conn_quota : public ss::peering_sharded_service<conn_quota> {
     ss::gate _gate;
 };
 
-} // namespace net
+} // namespace net