[BACKPORT 2024.1][#23445]yugabyted: Node not starting with DNS name a…

…nd `--secure` option

Summary:
We have to use `DNS.1` for DNS names in Subject alt names while creating server certs instead of `IP.1`. `IP.1` expects only an IP.
Jira: DB-12366

Original commit: b9d2e9d / D37161

Test Plan: ./yb_build.sh --java-test 'org.yb.yugabyted.*'

Reviewers: nikhil

Reviewed By: nikhil

Subscribers: sgarg-yb, yugabyted-dev

Tags: #jenkins-ready

Differential Revision: https://phorge.dev.yugabyte.com/D37170

bin/openssl_proxy.sh

@@ -9,6 +9,7 @@ temp_certs_path=""
 key_path=""
 keyname=""
 hostname=""
+hostname_type=""
 
 show_help() {
   cat >&1 <<-EOT
@@ -47,6 +48,9 @@ node certs generation)
 
     --keyname, --kn, -kn
         Name of the key to be generated.
+
+    --hostname-type, --ht, -ht
+        Hostname type. Accepted vales: IP/DNS
 ---------------------------------------------------------------------------------------------------
 EOT
 }
@@ -107,9 +111,13 @@ generate_node_certs() {
 
     [ req_ext ]
     subjectAltName = @alt_names
-    [alt_names]
-    IP.1 = '"$hostname"'' > "$temp_certs_path"/node.conf
+    [alt_names]' > "$temp_certs_path"/node.conf
 
+    if [[ $hostname_type == "DNS" ]]; then
+        echo 'DNS.1 = '"$hostname"'' >> "$temp_certs_path"/node.conf
+    else
+        echo 'IP.1 = '"$hostname"'' >> "$temp_certs_path"/node.conf
+    fi
 
     openssl genrsa -out "$temp_certs_path"/node."$hostname".key
     chmod 400 "$temp_certs_path"/node."$hostname".key
@@ -169,6 +177,10 @@ while [[ $# -gt 0 ]]; do
         hostname="$2"
         shift
     ;;
+    --hostname_type|--ht|-ht)
+        hostname_type="$2"
+        shift
+    ;;
     generate-key)
         key_generation=true
     ;;

bin/yugabyted

@@ -1917,8 +1917,12 @@ class ControlScript(object):
                     "root-ca certs database. Removing...")
                 shutil.rmtree(node_certs_dir)
 
+            if self.configs.saved_data.get("dns_enabled"):
+                hostname_type = "DNS"
+            else:
+                hostname_type = "IP"
             status = OpenSSLProxy.generate_node_server_certs(root_certs_dir=root_certs_dir,
-                hostname=hostname, server_cert_dir=node_certs_dir)
+                hostname=hostname, server_cert_dir=node_certs_dir, hostname_type=hostname_type)
 
             if not status:
                 status_details = [
@@ -9015,9 +9019,10 @@ class OpenSSLProxy(object):
 
     # Generate node server certificates
     @staticmethod
-    def generate_node_server_certs(root_certs_dir, server_cert_dir, hostname, timeout=60):
+    def generate_node_server_certs(root_certs_dir, server_cert_dir, hostname, hostname_type,
+                                    timeout=60):
         cmd = OpenSSLProxy.cmd_args + ['generate-server-cert', '-rcp', root_certs_dir,
-            '-scp', server_cert_dir, '-hn', hostname]
+            '-scp', server_cert_dir, '-hn', hostname, '-ht', hostname_type]
         out, err, ret = run_process(cmd, timeout=timeout, log_cmd=True)
 
         return (0 == ret)