diff --git a/src/Containers/Microsoft.NET.Build.Containers/FallbackToHttpMessageHandler.cs b/src/Containers/Microsoft.NET.Build.Containers/FallbackToHttpMessageHandler.cs
index 49b687fcb258..7fcb85735605 100644
--- a/src/Containers/Microsoft.NET.Build.Containers/FallbackToHttpMessageHandler.cs
+++ b/src/Containers/Microsoft.NET.Build.Containers/FallbackToHttpMessageHandler.cs
@@ -79,11 +79,17 @@ internal static bool ShouldAttemptFallbackToHttp(HttpRequestException exception)
         return exception.HttpRequestError == HttpRequestError.SecureConnectionError;
     }
 
+    private static bool RegistryNameContainsPort(string registryName)
+    {
+        // use `container` scheme which does not have a default port.
+        return new Uri($"container://{registryName}").Port != -1;
+    }
+
     private static void FallbackToHttp(string registryName, HttpRequestMessage request)
     {
         var uriBuilder = new UriBuilder(request.RequestUri!);
         uriBuilder.Scheme = "http";
-        if (registryName.IndexOf(':') < 0)
+        if (RegistryNameContainsPort(registryName) == false)
         {
             // registeryName does not contains port number, so reset the port number to -1, otherwise it will be https default port 443
             uriBuilder.Port = -1;
diff --git a/test/Microsoft.NET.Build.Containers.UnitTests/FallbackToHttpMessageHandlerTests.cs b/test/Microsoft.NET.Build.Containers.UnitTests/FallbackToHttpMessageHandlerTests.cs
index 77f878898094..0c6933f94473 100644
--- a/test/Microsoft.NET.Build.Containers.UnitTests/FallbackToHttpMessageHandlerTests.cs
+++ b/test/Microsoft.NET.Build.Containers.UnitTests/FallbackToHttpMessageHandlerTests.cs
@@ -18,9 +18,13 @@ public class FallbackToHttpMessageHandlerTests
         [InlineData("mcr.microsoft.com:443", 443)]
         [InlineData("mcr.microsoft.com:80", 80)]
         [InlineData("mcr.microsoft.com:5555", 5555)]
+        [InlineData("[2408:8120:245:49a0:f041:d7bb:bb13:5b64]", 80)]
+        [InlineData("[2408:8120:245:49a0:f041:d7bb:bb13:5b64]:443", 443)]
+        [InlineData("[2408:8120:245:49a0:f041:d7bb:bb13:5b64]:80", 80)]
+        [InlineData("[2408:8120:245:49a0:f041:d7bb:bb13:5b64]:5555", 5555)]
         public async Task FallBackToHttpPortShouldAsExpected(string registry, int expectedPort)
         {
-            var uri = new UriBuilder($"https://{registry}").Uri;
+            var uri = new Uri($"https://{registry}");
             var handler = new FallbackToHttpMessageHandler(
                 registry,
                 uri.Host,