Refactor the SNTApplicationTest unit tests to function correctly (#885)

* Refactor the SNTApplicationTest unit tests to function correctly.

The tests were originally written in a table style and were impacted by the lack of mocking the configurator. This caused issues with static rules to impact the unit tests.

Additionally added improved logging messages for critical binaries and a todo for macOS 13 unit tests.

Added goodbinary and rules.db test files to allstar's ignored paths.

.allstar/binary_artifacts.yaml

@@ -9,8 +9,10 @@ ignorePaths:
 - Source/common/testdata/DirectoryBundle/Contents/MacOS/DirectoryBundle
 - Source/common/testdata/DirectoryBundle/Contents/Resources/BundleExample.app/Contents/MacOS/BundleExample
 - Source/santad/testdata/binaryrules/badbinary
+- Source/santad/testdata/binaryrules/goodbinary
 - Source/santad/testdata/binaryrules/badcert
 - Source/santad/testdata/binaryrules/banned_teamid_allowed_binary
 - Source/santad/testdata/binaryrules/banned_teamid
 - Source/santad/testdata/binaryrules/goodcert
 - Source/santad/testdata/binaryrules/noop
+- Source/santad/testdata/binaryrules/rules.db

Source/santad/DataLayer/SNTRuleTable.m

@@ -105,6 +105,7 @@ + (NSArray *)criticalSystemBinaryPaths {
 
     // This is a Santa-curated list of paths to check on startup. This list will be merged
     // with the set of default muted paths from ES.
+
     NSSet *santaDefinedCriticalPaths = [NSSet setWithArray:@[
       @"/usr/libexec/trustd",
       @"/usr/lib/dyld",
@@ -143,9 +144,9 @@ - (void)setupSystemCriticalBinaries {
     if ([csInfo signingInformationMatches:self.launchdCSInfo]) {
       systemBin = YES;
     } else if (![csInfo signingInformationMatches:self.santadCSInfo]) {
-      LOGE(@"Unable to validate critical system binary. "
+      LOGE(@"Unable to validate critical system binary %@. "
            @"pid 1: %@, santad: %@ and %@: %@ do not match.",
-           self.launchdCSInfo.leafCertificate, self.santadCSInfo.leafCertificate, path,
+           path, self.launchdCSInfo.leafCertificate, self.santadCSInfo.leafCertificate, path,
            csInfo.leafCertificate);
       continue;
     }

Source/santad/SNTApplicationTest.m

@@ -24,6 +24,8 @@
 
 #include "Source/santad/EventProviders/EndpointSecurityTestUtil.h"
 
+NSString *testBinariesPath = @"santa/Source/santad/testdata/binaryrules";
+
 @interface SNTApplicationTest : XCTestCase
 @property id mockSNTDatabaseController;
 @end
@@ -40,12 +42,26 @@ - (void)tearDown {
   [super tearDown];
 }
 
-- (void)checkBinaryExecution:(NSString *)binaryName
-                    testPath:(NSString *)testPath
-                  wantResult:(es_auth_result_t)wantResult {
+- (BOOL)checkBinaryExecution:(NSString *)binaryName
+                  wantResult:(es_auth_result_t)wantResult
+                  clientMode:(NSInteger)clientMode {
   MockEndpointSecurity *mockES = [MockEndpointSecurity mockEndpointSecurity];
   [mockES reset];
 
+  id mockConfigurator = OCMClassMock([SNTConfigurator class]);
+
+  OCMStub([mockConfigurator configurator]).andReturn(mockConfigurator);
+  // Ensure that static rules do not interfere.
+  OCMStub([mockConfigurator staticRules]).andReturn(nil);
+  // Ensure the mode is set.
+  OCMStub([mockConfigurator clientMode]).andReturn(clientMode);
+  OCMStub([mockConfigurator failClosed]).andReturn(NO);
+
+  NSString *baseTestPath = @"santa/Source/santad/testdata/binaryrules";
+  NSString *testPath = [NSString pathWithComponents:@[
+    [[[NSProcessInfo processInfo] environment] objectForKey:@"TEST_SRCDIR"], baseTestPath
+  ]];
+
   OCMStub([self.mockSNTDatabaseController databasePath]).andReturn(testPath);
 
   SNTApplication *app = [[SNTApplication alloc] init];
@@ -88,31 +104,100 @@ - (void)checkBinaryExecution:(NSString *)binaryName
   [mockES triggerHandler:msg.message];
 
   [self waitForExpectations:@[ expectation ] timeout:10.0];
+  NSString *clientModeStr = (clientMode == SNTClientModeLockdown) ? @"LOCKDOWN" : @"MONITOR";
 
-  XCTAssertEqual(got.result, wantResult, @"received unexpected ES response on executing \"%@/%@\"",
-                 testPath, binaryName);
-}
-
-- (void)testRules {
-  NSString *testPath = @"santa/Source/santad/testdata/binaryrules";
-  NSDictionary *testCases = @{
-    @"badbinary" : [NSNumber numberWithInt:ES_AUTH_RESULT_DENY],
-    @"goodbinary" : [NSNumber numberWithInt:ES_AUTH_RESULT_ALLOW],
-    @"noop" : [NSNumber numberWithInt:ES_AUTH_RESULT_ALLOW],
-    @"banned_teamid" : [NSNumber numberWithInt:ES_AUTH_RESULT_DENY],
-    @"banned_teamid_allowed_binary" : [NSNumber numberWithInt:ES_AUTH_RESULT_ALLOW],
-    @"badcert" : [NSNumber numberWithInt:ES_AUTH_RESULT_DENY],
-    @"goodcert" : [NSNumber numberWithInt:ES_AUTH_RESULT_ALLOW],
-  };
-  NSString *fullTestPath = [NSString pathWithComponents:@[
-    [[[NSProcessInfo processInfo] environment] objectForKey:@"TEST_SRCDIR"], testPath
-  ]];
+  XCTAssertEqual(got.result, wantResult,
+                 @"received unexpected ES response on executing \"%@/%@\" in clientMode %@",
+                 testPath, binaryName, clientModeStr);
+}
+
+/**
+ * testRules ensures that we get the expected outcome when the mocks "execute"
+ * our test binaries.
+ **/
+
+- (void)testBinaryWithSHA256BlockRuleIsBlockedInLockdownMode {
+  [self checkBinaryExecution:@"badbinary"
+                  wantResult:ES_AUTH_RESULT_DENY
+                  clientMode:SNTClientModeLockdown];
+}
+
+- (void)testBinaryWithSHA256BlockRuleIsBlockedInMonitorMode {
+  [self checkBinaryExecution:@"badbinary"
+                  wantResult:ES_AUTH_RESULT_DENY
+                  clientMode:SNTClientModeMonitor];
+}
+
+- (void)testBinaryWithSHA256AllowRuleIsNotBlockedInLockdownMode {
+  [self checkBinaryExecution:@"goodbinary"
+                  wantResult:ES_AUTH_RESULT_ALLOW
+                  clientMode:SNTClientModeLockdown];
+}
+
+- (void)testBinaryWithSHA256AllowRuleIsNotBlockedInMonitorMode {
+  [self checkBinaryExecution:@"goodbinary"
+                  wantResult:ES_AUTH_RESULT_ALLOW
+                  clientMode:SNTClientModeMonitor];
+}
+
+- (void)testBinaryWithCertificateAllowRuleIsNotBlockedInLockdownMode {
+  [self checkBinaryExecution:@"goodcert"
+                  wantResult:ES_AUTH_RESULT_ALLOW
+                  clientMode:SNTClientModeLockdown];
+}
+
+- (void)testBinaryWithCertificateAllowRuleIsNotBlockedInMonitorMode {
+  [self checkBinaryExecution:@"goodcert"
+                  wantResult:ES_AUTH_RESULT_ALLOW
+                  clientMode:SNTClientModeMonitor];
+}
+
+- (void)testBinaryWithCertificateBlockRuleIsBlockedInLockdownMode {
+  [self checkBinaryExecution:@"badcert"
+                  wantResult:ES_AUTH_RESULT_DENY
+                  clientMode:SNTClientModeLockdown];
+}
+
+- (void)testBinaryWithCertificateBlockRuleIsNotBlockedInMonitorMode {
+  [self checkBinaryExecution:@"badcert"
+                  wantResult:ES_AUTH_RESULT_DENY
+                  clientMode:SNTClientModeMonitor];
+}
+
+- (void)testBinaryWithTeamIDBlockRuleIsBlockedInLockdownMode {
+  [self checkBinaryExecution:@"banned_teamid"
+                  wantResult:ES_AUTH_RESULT_DENY
+                  clientMode:SNTClientModeLockdown];
+}
+
+- (void)testBinaryWithTeamIDBlockRuleIsBlockedInMonitorMode {
+  [self checkBinaryExecution:@"banned_teamid"
+                  wantResult:ES_AUTH_RESULT_DENY
+                  clientMode:SNTClientModeMonitor];
+}
+
+- (void)testBinaryWithSHA256AllowRuleAndBlockedTeamIDRuleIsAllowedInLockdownMode {
+  [self checkBinaryExecution:@"banned_teamid_allowed_binary"
+                  wantResult:ES_AUTH_RESULT_ALLOW
+                  clientMode:SNTClientModeLockdown];
+}
+
+- (void)testBinaryWithSHA256AllowRuleAndBlockedTeamIDRuleIsAllowedInMonitorMode {
+  [self checkBinaryExecution:@"banned_teamid_allowed_binary"
+                  wantResult:ES_AUTH_RESULT_ALLOW
+                  clientMode:SNTClientModeMonitor];
+}
+
+- (void)testBinaryWithoutBlockOrAllowRuleIsAllowedInLockdownMode {
+  [self checkBinaryExecution:@"noop"
+                  wantResult:ES_AUTH_RESULT_DENY
+                  clientMode:SNTClientModeLockdown];
+}
 
-  for (NSString *binary in testCases) {
-    [self checkBinaryExecution:binary
-                      testPath:fullTestPath
-                    wantResult:[testCases[binary] intValue]];
-  }
+- (void)testBinaryWithoutBlockOrAllowRuleIsAllowedInMonitorMode {
+  [self checkBinaryExecution:@"noop"
+                  wantResult:ES_AUTH_RESULT_ALLOW
+                  clientMode:SNTClientModeMonitor];
 }
 
 @end