Merge pull request #78 from Microsoft/feature/inspec-profile

Restructured Integration Testing Suite using InSpec Profile

.kitchen.yml

@@ -5,78 +5,72 @@ driver:
 
 provisioner:
   product_name: chef
-  product_version: 13.8
-  always_update_cookbooks: true
+  product_version: latest
+  install_strategy: always
 
 verifier:
   name: inspec
   sudo: true
+  inspec_tests:
+  - test/integration/default
 
 platforms:
-  - name: apex/macos-10.13.3
-  - name: apex/macos-10.12.6
-  - name: apex/macos-10.11.6
+- name: apex/macos-10.11.6
+- name: apex/macos-10.12.6
+- name: apex/macos-10.13.3
 
 suites:
-  - name: default
-    provisioner:
-      multiple_converge: 2
-      enforce_idempotency: true
-    run_list:
-      - recipe[macos_test::new_users]
-      - recipe[macos::disable_software_updates]
-      - recipe[macos_test::preferences]
-    verifier:
-      inspec_tests:
-        - test/smoke/default
+- name: default
+  run_list:
+  - recipe[macos::disable_software_updates]
+  - recipe[macos_test::preferences]
+  verifier:
+    controls:
+    - dock-appearance
+    - show-all-files
+    - updates-disabled
 
-  - name: keep_awake
-    provisioner:
-      multiple_converge: 2
-      enforce_idempotency: false
-    run_list:
-      - recipe[macos::keep_awake]
-    verifier:
-      inspec_tests:
-        - test/smoke/keep_awake
+- name: power-management
+  run_list:
+  - recipe[macos::keep_awake]
+  verifier:
+    controls:
+    - screensaver-disabled
+    - remote-administration
+    - no-sleep
 
-  - name: machine_name
-    provisioner:
-      multiple_converge: 2
-      enforce_idempotency: true
-    run_list:
-      - recipe[macos_test::machine_name]
-    verifier:
-      inspec_tests:
-        - test/smoke/machine_name
+- name: machine-name
+  run_list:
+  - recipe[macos_test::machine_name]
+  verifier:
+    controls:
+    - nonstandard-computer-name
 
-  - name: xcode
-    run_list:
-      - recipe[macos_test::xcode]
-    verifier:
-      inspec_tests:
-        - test/smoke/xcode
+- name: spotlight
+  run_list:
+  - recipe[macos_test::spotlight]
+  verifier:
+    controls:
+    - indexing-and-searching
 
-  - name: spotlight
-    provisioner:
-      multiple_converge: 2
-      enforce_idempotency: true
-    run_list:
-      - recipe[macos_test::spotlight]
-    verifier:
-      inspec_tests:
-        - test/smoke/spotlight
+- name: xcode
+  run_list:
+  - recipe[macos_test::xcode]
+  verifier:
+    controls:
+    - xcode-and-simulators
 
-  - name: certificate
-    run_list:
-      - recipe[macos_test::certificate]
-    verifier:
-      inspec_tests:
-        - test/smoke/certificate
+- name: certificate
+  run_list:
+  - recipe[macos_test::certificate]
+  verifier:
+    controls:
+    - certificate-install
 
-  - name: users
-    run_list:
-      - recipe[macos_test::new_users]
-    verifier:
-      inspec_tests:
-        - test/smoke/users
+- name: users
+  run_list:
+  - recipe[macos_test::new_users]
+  verifier:
+    controls:
+    - admin-user
+    - standard-user

TESTING.md

@@ -31,7 +31,7 @@ $ foodcritic .
 - [RSpec](http://rspec.info/)
 - [ChefDK](https://downloads.chef.io/chefdk) (optional)
 
-For unit tests, we focus on testing the library files, which are usually writtin
+For unit tests, we focus on testing the library files, which are written
 in pure Ruby and tested with RSpec. The library files contain most of the core
 business logic for each of the custom resources and are used as either mixins
 for the custom resources or contain classes that act as helpers, except with

test/cookbooks/macos_test/metadata.rb

@@ -1,4 +1,8 @@
 name 'macos_test'
+maintainer 'Microsoft'
+maintainer_email '[email protected]'
+description 'The testing/example cookbook for the macOS cookbook recipes and custom resources.'
+chef_version '~> 13.0' if respond_to?(:chef_version)
 version '1.0.1'
 
 depends 'macos'

test/cookbooks/macos_test/recipes/xcode.rb

@@ -1,13 +1,13 @@
-if node['platform_version'].match?(/10\.13/) || node['platform_version'].match?(/10\.12/)
+if node['platform_version'].match? Regexp.union ['10.12', '10.13']
   execute 'Disable Gatekeeper' do
-    command 'spctl --master-disable'
+    command ['spctl', '--master-disable']
   end
 
   xcode '9.2' do
     ios_simulators %w(11 10)
   end
 
-elsif node['platform_version'].match?(/10\.11/)
+elsif node['platform_version'].match? Regexp.union '10.11'
 
   xcode '8.2.1' do
     ios_simulators %w(10 9)

test/integration/default/README.md

@@ -0,0 +1,6 @@
+# macOS Cookbook InSpec Profile
+
+The macOS Cookbook InSpec profile tests the custom resources and the recipes
+that utilize them to ascertain their full functionality and dependability. We
+are always looking for better and more creative ways to test the code, and
+all types of contributions are welcomed.

test/integration/default/controls/desktop_and_screensaver.rb

@@ -0,0 +1,35 @@
+title 'desktop and screen saver'
+
+user_home = os_env('HOME').content
+test_user = 'vagrant'
+
+control 'screensaver-disabled' do
+  title 'idletime is set to zero'
+  desc 'Verify that when the computer is not in use, moving images or patterns do not appear'
+
+  def hardware_uuid
+    system_profiler_command = command('/usr/sbin/system_profiler SPHardwareDataType')
+    hardware_data = ::Psych.load(system_profiler_command.stdout)
+    hardware_data['Hardware']['Hardware Overview']['Hardware UUID']
+  end
+
+  describe command('/usr/sbin/system_profiler SPHardwareDataType') do
+    its('stdout') { should_not be nil }
+  end
+
+  describe command("/usr/libexec/PlistBuddy -c 'Print :idleTime' #{user_home}/Library/Preferences/ByHost/com.apple.screensaver.#{hardware_uuid}.plist") do
+    its('stdout') { should match(/0/) }
+  end
+
+  describe command("su #{test_user} -c 'defaults -currentHost read com.apple.screensaver idleTime'") do
+    its('stdout') { should match(/0/) }
+  end
+
+  describe command("su #{test_user} -c 'defaults -currentHost read-type com.apple.screensaver idleTime'") do
+    its('stdout') { should match(/integer/) }
+  end
+
+  describe command("file --brief --mime #{user_home}/Library/Preferences/ByHost/com.apple.screensaver.#{hardware_uuid}.plist") do
+    its('stdout') { should match(/binary/) }
+  end
+end

test/integration/default/controls/dock.rb

@@ -0,0 +1,16 @@
+user_home = os_env('HOME').content
+
+title 'dock'
+
+control 'dock-appearance' do
+  title 'how the dock looks'
+  desc 'Verify changes are made to the dock by modifying the plist'
+
+  describe command("/usr/libexec/PlistBuddy -c 'Print :orientation' #{user_home}/Library/Preferences/com.apple.dock.plist") do
+    its('stdout') { should match 'left' }
+  end
+
+  describe command("/usr/libexec/PlistBuddy -c 'Print :DisableAllAnimations' #{user_home}/Library/Preferences/com.apple.dock.plist") do
+    its('stdout') { should match 'true' }
+  end
+end

test/integration/default/controls/energy_saver.rb

@@ -0,0 +1,67 @@
+title 'energy saver'
+
+control 'remote-administration' do
+  title 'power management settings for managed macs'
+  desc '
+    Verify all sleep is disabled the machine automatically
+    restarts after a freeze or power outage using the systemsetup and
+    pmset commands
+  '
+
+  describe command('systemsetup -getremoteappleevents') do
+    its('stdout') { should include 'Remote Apple Events: On' }
+  end
+
+  describe command('systemsetup -getwaitforstartupafterpowerfailure') do
+    its('stdout') { should match(/getwaitforstartupafterpowerfailure: 0 seconds/) }
+  end
+
+  describe command('systemsetup -getremotelogin') do
+    its('stdout') { should match(/On/) }
+  end
+
+  describe command('pmset -g') do
+    its('stdout') { should match(/hibernatemode\s+0/) }
+  end
+
+  describe command('pmset -g') do
+    its('stdout') { should match %r{hibernatefile\s+/var/vm/sleepimage} }
+  end
+
+  describe command('pmset -g') do
+    its('stdout') { should match(/ttyskeepawake\s+1/) }
+  end
+end
+
+control 'no-sleep' do
+  title 'macOS never goes to sleep'
+  desc 'Verify that the system will not fall asleep using the pmset and systemsetup commands'
+
+  describe command('systemsetup -getcomputersleep') do
+    its('stdout') { should match(/Never/) }
+  end
+
+  describe command('systemsetup -getdisplaysleep') do
+    its('stdout') { should match(/Never/) }
+  end
+
+  describe command('systemsetup -getharddisksleep') do
+    its('stdout') { should match(/after 10 minutes/) }
+  end
+
+  describe command('systemsetup -getrestartfreeze') do
+    its('stdout') { should match(/On/) }
+  end
+
+  describe command('pmset -g') do
+    its('stdout') { should match(/sleep\s+0/) }
+  end
+
+  describe command('pmset -g') do
+    its('stdout') { should match(/disksleep\s+10/) }
+  end
+
+  describe command('pmset -g') do
+    its('stdout') { should match(/displaysleep\s+0/) }
+  end
+end

test/integration/default/controls/general.rb

@@ -0,0 +1,13 @@
+user_home = os_env('HOME').content
+
+title 'general'
+
+control 'show-all-files' do
+  title 'enable visibility of invisble files and folders'
+  desc 'Verify changes to the finder plist show hidden files using PlistBuddy'
+
+  finder_plist = "#{user_home}/Library/Preferences/com.apple.finder.plist"
+  describe command("/usr/libexec/PlistBuddy -c 'Print :AppleShowAllFiles' #{finder_plist}") do
+    its('stdout') { should match 'true' }
+  end
+end

test/integration/default/controls/security.rb

@@ -0,0 +1,17 @@
+title 'security'
+
+control 'certificate-install' do
+  title 'installation and discovery of a certificate'
+  desc '
+    Verify that a test certificate is able to be installed and can also
+    be discovered via the security-find-certificate command
+  '
+
+  describe file('/Users/vagrant/Test.p12') do
+    it { should exist }
+  end
+
+  describe command('/usr/bin/security find-certificate /Users/vagrant/Library/Keychains/login.keychain') do
+    its('stdout') { should include 'Test' }
+  end
+end