Fix vmware#206 and vmware#201 Warn about GOSC DNS failures on Ubuntu …

…and default gateway failures

Signed-off-by: Qi Zhang <[email protected]>

linux/guest_customization/check_dns.yml

@@ -45,13 +45,17 @@
             dest_path: "{{ current_test_log_folder }}/resolvectl_dns.txt"
 
         - name: "Get output of 'resolvectl dns'"
-          command: cat "{{ current_test_log_folder }}/resolvectl_dns.txt"
+          command: "cat {{ current_test_log_folder }}/resolvectl_dns.txt"
           register: dns_servers
+          failed_when: False
 
         - name: "Set fact of guest DNS servers"
           set_fact:
             guest_dns_servers: "{{ dns_servers.stdout_lines | select('match', '^Link') |
                                regex_findall('\\b(?:[0-9]{1,3}\\.){3}[0-9]{1,3}\\b') }}"
+          when:
+            - dns_servers is defined
+            - dns_servers.stdout_lines is defined
       when: resolvectl_dns_result | bool
 
     # Run command 'resolvectl domain' to get DNS servers
@@ -73,13 +77,17 @@
             dest_path: "{{ current_test_log_folder }}/resolvectl_domain.txt"
 
         - name: "Get output of 'resolvectl domain'"
-          command: cat "{{ current_test_log_folder }}/resolvectl_domain.txt"
+          command: "cat {{ current_test_log_folder }}/resolvectl_domain.txt"
           register: dns_domains
+          failed_when: False
 
         - name: "Set fact of guest DNS searching domains"
           set_fact:
             guest_dns_search: "{{ dns_domains.stdout_lines | select('match', '^Link') |
                               regex_findall('\\b(?:[a-zA-Z0-9-]{1,}\\.){1,}[a-zA-Z0-9-]{1,}\\b') }}"
+          when:
+            - dns_domains is defined
+            - dns_domains.stdout_lines is defined
       when: resolvectl_domain_result | bool
   when: resolvectl_status_result
 
@@ -104,24 +112,33 @@
     - name: "Get DNS servers from /etc/resolv.conf"
       shell: cat "{{ current_test_log_folder }}/etc/resolv.conf" | grep -i "^nameserver\s*" | awk '{print $2}'
       register: dns_servers
+      failed_when: False
 
     - name: "Get guest DNS servers from shell command output"
       set_fact:
         guest_dns_servers: "{{ dns_servers.stdout_lines }}"
+      when:
+        - dns_servers is defined
+        - dns_servers.stdout_lines is defined
 
     - name: "Get DNS search domains from /etc/resolv.conf"
       shell: cat "{{ current_test_log_folder }}/etc/resolv.conf" | grep -i "^search" | awk '{i=2;while(i<=NF) {print $i; i++ }}'
       register: dns_search
+      failed_when: False
 
-    - name: "Save guest DNS search list to tmp variable"
-      set_fact:
-        tmp_dns_search: "{{ dns_search.stdout_lines }}"
+    - block:
+        - name: "Save guest DNS search list to tmp variable"
+          set_fact:
+            tmp_dns_search: "{{ dns_search.stdout_lines }}"
 
-    # In Amazon Linux 2 guest, a period after each DNS search domain would be added, so remove it
-    - name: "Get guest DNS search list"
-      set_fact:
-        guest_dns_search: "{{ guest_dns_search + [item | regex_replace('\\.$','')] }}"
-      with_items: "{{ tmp_dns_search }}"
+        # In Amazon Linux 2 guest, a period after each DNS search domain would be added, so remove it
+        - name: "Get guest DNS search list"
+          set_fact:
+            guest_dns_search: "{{ guest_dns_search + [item | regex_replace('\\.$','')] }}"
+          with_items: "{{ tmp_dns_search }}"
+      when:
+        - dns_search is defined
+        - dns_search.stdout_lines is defined
   when:
     - not resolvectl_status_result
     - guest_os_ansible_distribution != 'Ubuntu'
@@ -146,18 +163,28 @@
         - name: "Get DNS servers from command systemd-resolve"
           shell: cat "{{ current_test_log_folder }}/systemd_resolve_status.txt" | sed -n '/DNS Servers/,/DNS Domain/p' | head -n -1
           register: dns_servers
+          failed_when: False
+
         - name: "Set fact of DNS servers from shell command output"
           set_fact:
             guest_dns_servers: "{{ (guest_dns_servers + [item.strip().split(':')[1].strip()]) if 'DNS Servers' in item else (guest_dns_servers + [item.strip()]) }}"
           with_items: "{{ dns_servers.stdout_lines }}"
+          when:
+            - dns_servers is defined
+            - dns_servers.stdout_lines is defined
 
         - name: "Get DNS search domains from command systemd-resolve"
           shell: cat "{{ current_test_log_folder }}/systemd_resolve_status.txt" | sed -n '/DNS Domain/,/DNSSEC NTA/p' | head -n -1
           register: dns_domains
+          failed_when: False
+
         - name: "Set fact of DNS search domains from shell command output"
           set_fact:
             guest_dns_search: "{{ (guest_dns_search + [item.strip().split(':')[1].strip()]) if 'DNS Domain' in item else (guest_dns_search + [item.strip()]) }}"
           with_items: "{{ dns_domains.stdout_lines }}"
+          when:
+            - dns_domains is defined
+            - dns_domains.stdout_lines is defined
       when: systemd_resolve_result
   when:
     - not resolvectl_status_result
@@ -207,6 +234,11 @@
     gosc_check_error_msg: "VM DNS servers are {{ guest_dns_servers }}, not expected DNS servers {{ linux_gosc_spec['dns_servers'] }}"
 
 # Check guest DNS search domains
+- debug:
+    msg:
+      - "The expected search domains: {{ linux_gosc_spec['dns_suffix'] }}"
+      - "The actual DNS search domains got in guest: {{ guest_dns_search }}"
+
 - name: "Check DNS search domains are customized"
   set_fact:
     dns_suffix_success: "{{ linux_gosc_spec['dns_suffix'] | difference(guest_dns_search) | length == 0 }}"

linux/guest_customization/check_etc_hosts.yml

@@ -9,6 +9,7 @@
     etc_hosts_success: False
     expected_ips: ['::1']
     ipv6_hostname_check: False
+    actual_ips: []
 
 - include_tasks: ../../common/vm_guest_file_operation.yml
   vars:
@@ -18,11 +19,15 @@
 
 - name: "Get IP bindings for hostname {{ linux_gosc_spec['hostname'] }} in /etc/hosts"
   shell: grep "{{ linux_gosc_spec['hostname'] }}.{{ linux_gosc_spec['domain'] }} *{{ linux_gosc_spec['hostname'] }}" {{ hosts_file_path }} | awk '{print $1}'
-  changed_when: False
+  failed_when: False
   register: etc_hosts_result
+
 - name: "Save hostname binding list"
   set_fact:
     actual_ips: "{{ etc_hosts_result.stdout_lines }}"
+  when:
+    - etc_hosts_result is defined
+    - etc_hosts_result.stdout_lines is defined
 
 # Perl GOSC:
 #   1. When ip assignment is dhcp, hostname will be bound to 127.0.1.1 and ::1

linux/guest_customization/check_hostname_and_domain.yml

@@ -3,6 +3,8 @@
 ---
 - name: "Initialize hostname and domain check result"
   set_fact:
+    guest_hostname: ""
+    guest_dns_domain: ""
     hostname_success: ""
     dns_domain_success: ""
 
@@ -27,14 +29,18 @@
     dest_path: "{{ current_test_log_folder }}/hostname_f.txt"
 
 - name: "Get guest FQDN"
-  command: cat "{{ current_test_log_folder }}/hostname_f.txt"
-  changed_when: False
+  command: "cat {{ current_test_log_folder }}/hostname_f.txt"
+  failed_when: False
   register: result
 
 - name: "Save hostname and domain check result"
   set_fact:
     guest_hostname: "{{ result.stdout.split('.')[0] }}"
     guest_dns_domain: "{{ result.stdout.split('.')[1:] | join('.') }}"
+  when:
+    - result is defined
+    - result.stdout is defined
+    - result.stdout.split('.') | length >= 2
 
 # Check hostname
 - name: Check guest hostname is customized

linux/guest_customization/check_network_config.yml

@@ -3,15 +3,14 @@
 ---
 - name: "Initialize GOSC network check result"
   set_fact:
+    ip_command: "{{ '/usr/sbin/ip' if guest_os_ansible_distribution != 'Ubuntu' else '/sbin/ip' }}"
+    guest_iface_name: ""
+    guest_ipv4_address: ""
+    guest_ipv4_netmask: ""
+    guest_ipv4_gateway: ""
     guest_ipv4_success: ""
     guest_netmask_success: ""
     guest_gateway_success: ""
-    ip_command: "/usr/sbin/ip"
-
-- name: "Set fact of ip command when guest OS is Ubuntu"
-  set_fact:
-    ip_command: "/sbin/ip"
-  when: guest_os_ansible_distribution in ["Ubuntu"]
 
 # Get IP address and netmask
 - include_tasks: ../../common/vm_shell_in_guest.yml
@@ -27,15 +26,19 @@
     dest_path: "{{ current_test_log_folder }}/ip_addr_show.txt"
 
 - name: "Get IPv4 address"
-  shell: cat "{{ current_test_log_folder }}/ip_addr_show.txt" | grep -v lo | grep -v virbr | awk '{print $1, $2, $3}'
-  changed_when: False
+  shell: "cat {{ current_test_log_folder }}/ip_addr_show.txt | grep -v lo | grep -v virbr | awk '{print $1, $2, $3}'"
+  failed_when: False
   register: result
 
 - name: "Save GOSC network facts from command output"
   set_fact:
-    guest_iface_name: "{{ result.stdout.split(' ')[0] }}"
-    guest_ipv4_address: "{{ result.stdout.split(' ')[-1] | ipaddr('address') }}"
-    guest_ipv4_netmask: "{{ result.stdout.split(' ')[-1] | ipaddr('netmask') }}"
+    guest_iface_name: "{{ result.stdout.split()[0] }}"
+    guest_ipv4_address: "{{ result.stdout.split()[-1] | ipaddr('address') }}"
+    guest_ipv4_netmask: "{{ result.stdout.split()[-1] | ipaddr('netmask') }}"
+  when:
+    - result is defined
+    - result.stdout is defined
+    - result.stdout.split() | length >= 3
 
 # Get default gateway
 - include_tasks: ../../common/vm_shell_in_guest.yml
@@ -51,12 +54,17 @@
     dest_path: "{{ current_test_log_folder }}/ip_route_show.txt"
 
 - name: "Get default gateway"
-  shell: cat "{{ current_test_log_folder }}/ip_route_show.txt" | grep "^default"
-  changed_when: False
+  shell: "cat {{ current_test_log_folder }}/ip_route_show.txt | grep '^default'"
+  failed_when: False
   register: result
+
 - name: "Save GOSC network gateway"
   set_fact:
-    guest_ipv4_gateway: "{{ result.stdout.split(' ')[2] }}"
+    guest_ipv4_gateway: "{{ result.stdout.split()[2] }}"
+  when:
+    - result is defined
+    - result.stdout is defined
+    - result.stdout.split() | length >= 3
 
 # Check IP address, netmaks, gateway settings for GOSC with DHCP IP
 - block:
@@ -186,7 +194,8 @@
         dest_path: "{{ current_test_log_folder }}/list_network_files.txt"
 
     - name: "Look for network file on Photon OS"
-      command: cat "{{ current_test_log_folder }}/list_network_files.txt"
+      command: "cat {{ current_test_log_folder }}/list_network_files.txt"
+      failed_when: False
       register: network_files
 
     - include_tasks: ../../common/vm_guest_file_operation.yml
@@ -197,4 +206,8 @@
       with_items: "{{ network_files.stdout_lines }}"
       loop_control:
         loop_var: "src_network_file"
+      when:
+        - network_files is defined
+        - network_files.stdout_lines is defined 
+        - network_files.stdout_lines | length > 0
   when: guest_os_ansible_distribution == "VMware Photon OS"

linux/guest_customization/check_timezone_and_hwclock.yml

@@ -3,9 +3,12 @@
 ---
 - name: "Initialize timezone check result"
   set_fact:
+    guest_timezone: ""
     timezone_success: ""
+
 - name: "Initialize hardware clock check result for perl GOSC"
   set_fact:
+    guest_hwclockUTC: ""
     hwclock_success: ""
   when: gosc_workflow == "perl"
 
@@ -27,10 +30,14 @@
     - name: "Get time zone from 'timedatectl status'"
       shell: grep -i "^ *time zone:" "{{ current_test_log_folder }}/timedatectl_status.txt" | awk '{print $3}'
       register: tz_result
+      failed_when: False
 
     - name: "Save guest timezone setting after GOSC"
       set_fact:
         guest_timezone: "{{ tz_result.stdout }}"
+      when:
+        - tz_result is defined
+        - tz_result.stdout is defined
 
     - block:
         - name: "Get hwclock from 'timedatectl status'"
@@ -61,13 +68,16 @@
         dest_path: "{{ current_test_log_folder }}/readlink_etc_localtime.txt"
 
     - name: "Get timezone info from /etc/localtime"
-      command: cat "{{ current_test_log_folder }}/readlink_etc_localtime.txt"
+      command: "cat {{ current_test_log_folder }}/readlink_etc_localtime.txt"
       register: tz_result
-      changed_when: False
+      failed_when: False
 
     - name: "Save guest timezone setting after GOSC"
       set_fact:
         guest_timezone: "{{ tz_result.stdout.replace('/usr/share/zoneinfo/','') }}"
+      when:
+        - tz_result is defined
+        - tz_result.stdout is defined
 
     # Get hwclock by running command "hwclock -D" when test perl GOSC
     - block:

linux/guest_customization/linux_gosc_start.yml

@@ -44,6 +44,7 @@
 # Do not fail GOSC immediately if state keyword doesn't exist
 - include_tasks: ../../common/vm_wait_gosc_completed.yml
   vars:
+    get_guest_reset_time_retries: 10
     check_gosc_state_keyword: False
 
 # Traditional GOSC will reboot guest OS (except for Photon) after all customization done

linux/guest_customization/linux_gosc_verify.yml

@@ -43,9 +43,69 @@
 - name: "Print GOSC validation errors"
   debug: var=gosc_validation_errors
 
-- name: "Assert {{ gosc_workflow }} guest customization checks all PASS"
-  assert:
-    that:
-      - "{{ item.value }}"
-    fail_msg: "{{ gosc_validation_errors[item.key] }}"
-  with_dict: "{{ gosc_validation_result }}"
+- name: "Get failed customization items"
+  set_fact:
+    gosc_failed_items: "{{ gosc_validation_result | dict2items | selectattr('value', 'equalto', false) | map(attribute='key') }}"
+
+# Warn about below known issues but not fail the task
+- block:
+    - name: "Ignore DNS customization failures"
+      set_fact:
+        ignored_gosc_failed_items: ['dns_servers_success', 'dns_suffix_success']
+
+    - name: "Ignore DNS customization failures when performing cloud-init GOSC with DHCP IP on Ubuntu"
+      debug:
+        msg: "{{ (gosc_validation_errors | dict2items | selectattr('key', 'in', ignored_gosc_failed_items) | map(attribute='value') | flatten) + ['WARNING: DNS customization failed due to https://bugs.launchpad.net/ubuntu/+source/cloud-init/+bug/1776452. Ignore this failure.'] }}"
+      when: (gosc_failed_items | length) > (gosc_failed_items | difference(ignored_gosc_failed_items) | length)
+
+    - name: "Update failed customization items"
+      set_fact:
+        gosc_failed_items: "{{ gosc_failed_items | difference(ignored_gosc_failed_items) }}"
+  when:
+    - gosc_failed_items | length > 0
+    - guest_os_ansible_distribution == "Ubuntu"
+    - enable_cloudinit_gosc
+    - ip_assignment == "dhcp"
+
+# User cannot customize the vCenter Server Appliance guest operating system to set a virtual NIC as a primary.
+# The issue was fixed on vCenter Server 6.5 U3, 6.7U2, and later releases.
+- block:
+    - name: "Ignore default gateway customization failure"
+      set_fact:
+        ignored_gosc_failed_items: ['guest_gateway_success']
+
+    - name: "cloud-init GOSC with static IP couldn't set default gateway"
+      debug:
+        msg: "{{ (gosc_validation_errors | dict2items | selectattr('key', 'in', ignored_gosc_failed_items) | map(attribute='value') | flatten) + ['WARNING: default gateway customization failure has been fixed on vCenter Server 6.5 U3 and 6.7U2. See https://docs.vmware.com/en/VMware-vSphere/6.5/rn/vsphere-vcenter-server-65u3-release-notes.html#guest-os-issues-resolved and https://docs.vmware.com/en/VMware-vSphere/6.7/rn/vsphere-vcenter-server-67u2-release-notes.html#guest-os-issues-resolved. Ignore this failure.'] }}"
+      when: (gosc_failed_items | length) > (gosc_failed_items | difference(ignored_gosc_failed_items) | length)
+
+    - name: "Update failed customization items"
+      set_fact:
+        gosc_failed_items: "{{ gosc_failed_items | difference(ignored_gosc_failed_items) }}"
+  when:
+    - gosc_failed_items | length > 0
+    - enable_cloudinit_gosc
+    - ip_assignment == "static"
+    - (vcenter_version is version('6.5.0', '==') and vcenter_build | int < 14020092) or (vcenter_version is version('6.7.0', '==') and vcenter_build | int < 13010631)
+
+- block:
+    - name: "Ignore GOSC state keyword failure"
+      set_fact:
+        ignored_gosc_failed_items: ['gosc_state_keyword_found']
+
+    - name: "GOSC doesn't send complete state code to vmware.log when vCenter Server version is 6.5 U3 and earlier"
+      debug:
+        msg: "{{ (gosc_validation_errors | dict2items | selectattr('key', 'in', ignored_gosc_failed_items) | map(attribute='value') | flatten) + ['WARNING: No GOSC complete state keywork in vmware.log from vCenter Server 6.5 U3 and earlier. Ignore this failure.'] }}"
+      when: (gosc_failed_items | length) > (gosc_failed_items | difference(ignored_gosc_failed_items) | length)
+
+    - name: "Update failed customization items"
+      set_fact:
+        gosc_failed_items: "{{ gosc_failed_items | difference(ignored_gosc_failed_items) }}"
+  when:
+    - gosc_failed_items | length > 0
+    - vcenter_version is version('6.5.0', '==')
+
+- name: "GOS customization failed"
+  fail:
+    msg: "{{ gosc_validation_errors | dict2items | selectattr('key', 'in', gosc_failed_items) | map(attribute='value') | flatten }}"
+  when: gosc_failed_items | length > 0