Implement RFC2812 IP Address matching for HTTP over TLS.

If the hostname looks like an IP Address then do exact matches against GEN_IPADDR Subject Alternative Name entries.  Unlike the GEN_DNS matching of text hostnames, the IP Address form always falls back to the CN text comparison if no other match has been found.

src/Common/src/Interop/Unix/System.Security.Cryptography.Native/Interop.Crypto.cs

@@ -120,6 +120,9 @@ private static extern int SetX509ChainVerifyTime(
             int second,
             [MarshalAs(UnmanagedType.Bool)] bool isDst);
 
+        [DllImport(Libraries.CryptoNative)]
+        internal static extern int CheckX509IpAddress(SafeX509Handle x509, [In]byte[] addressBytes, int addressLen, string hostname, int cchHostname);
+
         [DllImport(Libraries.CryptoNative)]
         internal static extern int CheckX509Hostname(SafeX509Handle x509, string hostname, int cchHostname);
 

src/Native/System.Security.Cryptography.Native/openssl.c

@@ -881,7 +881,6 @@ CheckX509Hostname(
     // RFC2818 says that if ANY dNSName alternative name field is matched then we
     // should ignore the subject common name.
 
-    // TODO (3445): Match an input IP Address against the iPAddress SAN entries
     // TODO (3446): Match using IDNA rules.
 
     if (san)
@@ -937,6 +936,100 @@ CheckX509Hostname(
     return success;
 }
 
+/*
+Function:
+CheckX509IpAddress
+
+Used by System.Net.Security's Unix CertModule to identify if the certificate presented by
+the server is applicable to the hostname (an IP address) requested.
+
+Return values:
+1 if the hostname is a match
+0 if the hostname is not a match
+Any negative number indicates an error in the arguments.
+*/
+int
+CheckX509IpAddress(
+    X509* x509,
+    const unsigned char* addressBytes,
+    int addressBytesLen,
+    const char* hostname,
+    int cchHostname)
+{
+    if (!x509)
+        return -2;
+    if (cchHostname > 0 && !hostname)
+        return -3;
+    if (cchHostname < 0)
+        return -4;
+    if (addressBytesLen < 0)
+        return -5;
+    if (!addressBytes)
+        return -6;
+
+    int subjectNid = NID_commonName;
+    int sanGenType = GEN_IPADD;
+    GENERAL_NAMES* san = X509_get_ext_d2i(x509, NID_subject_alt_name, NULL, NULL);
+    int success = 0;
+
+    if (san)
+    {
+        int i;
+        int count = sk_GENERAL_NAME_num(san);
+
+        for (i = 0; i < count; ++i)
+        {
+            GENERAL_NAME* sanEntry = sk_GENERAL_NAME_value(san, i);
+            ASN1_OCTET_STRING* ipAddr;
+
+            if (sanEntry->type != sanGenType)
+            {
+                continue;
+            }
+
+            ipAddr = sanEntry->d.iPAddress;
+
+            if (!ipAddr || !ipAddr->data || ipAddr->length != addressBytesLen)
+            {
+                continue;
+            }
+
+            if (!memcmp(addressBytes, ipAddr->data, (size_t)addressBytesLen))
+            {
+                success = 1;
+                break;
+            }
+        }
+
+        GENERAL_NAMES_free(san);
+    }
+
+    if (!success)
+    {
+        // This is a shared/interor pointer, do not free!
+        X509_NAME* subject = X509_get_subject_name(x509);
+
+        if (subject)
+        {
+            int i = -1;
+
+            while ((i = X509_NAME_get_index_by_NID(subject, subjectNid, i)) >= 0)
+            {
+                // Shared/interior pointers, do not free!
+                X509_NAME_ENTRY* nameEnt = X509_NAME_get_entry(subject, i);
+                ASN1_STRING* cn = X509_NAME_ENTRY_get_data(nameEnt);
+
+                if (CheckX509HostnameMatch(cn, hostname, cchHostname, 0))
+                {
+                    success = 1;
+                    break;
+                }
+            }
+        }
+    }
+
+    return success;
+}
 /*
 Function:
 GetX509StackFieldCount

src/System.Net.Security/src/System/Net/CertificateValidationPal.Unix.cs

@@ -41,7 +41,23 @@ internal static SslPolicyErrors VerifyCertificateProperties(
 
                     using (SafeX509Handle certHandle = Interop.libcrypto.X509_dup(certificate.Handle))
                     {
-                        hostnameMatch = Interop.Crypto.CheckX509Hostname(certHandle, hostName, hostName.Length);
+                        IPAddress hostnameAsIp;
+
+                        if (IPAddress.TryParse(hostName, out hostnameAsIp))
+                        {
+                            byte[] addressBytes = hostnameAsIp.GetAddressBytes();
+
+                            hostnameMatch = Interop.Crypto.CheckX509IpAddress(
+                                certHandle,
+                                addressBytes,
+                                addressBytes.Length,
+                                hostName,
+                                hostName.Length);
+                        }
+                        else
+                        {
+                            hostnameMatch = Interop.Crypto.CheckX509Hostname(certHandle, hostName, hostName.Length);
+                        }
                     }
 
                     if (hostnameMatch != 1)