Merge pull request Azure#11633 from Azure/Filtering-script-fix

Filtering Script exception list bug fix

.script/tests/asimParsersTest/ASimFilteringTest.py

@@ -18,7 +18,7 @@
 # Workspace ID for the Log Analytics workspace where the ASim filtering tests will be performed.
 WORKSPACE_ID = "e9beceee-7d61-429f-a177-ee5e2b7f481a"
 # Timespan for the parser query
-TIME_SPAN_IN_DAYS = 7
+TIME_SPAN_IN_DAYS = 2
 
 # exclusion_file_path refers to the CSV file path containing a list of parsers. Despite failing tests, these parsers will not cause the overall workflow to fail
 exclusion_file_path = '.script/tests/asimParsersTest/ExclusionListForASimTests.csv'
@@ -309,6 +309,7 @@ def main():
                 if parser_file['EquivalentBuiltInParser'] in read_exclusion_list_from_csv():
                     print(f"{YELLOW}The parser {parser_file_path} is listed in the exclusions file. Therefore, this workflow run will not fail because of it. To allow this parser to cause the workflow to fail, please remove its name from the exclusions list file located at: {exclusion_file_path}{RESET}")
                     sys.stdout.flush()
+                    continue
                 # Check for exception cases where the failure can be ignored
                 # Check if the failure message and schema match the exception cases
                 if len(result.failures) == 1:

.script/tests/asimParsersTest/VerifyASimParserTemplate.py

@@ -125,6 +125,10 @@ def extract_and_check_properties(Parser_file, Union_Parser__file, FileType, Pars
     if match:
         event_product = match.group(1)
         results.append((event_product, '"EventProduct" field is mapped in parser', 'Pass'))
+    # if equivalent_built_in_parser end with Native, then use 'EventProduct' as SchemaName + 'NativeTable'
+    elif equivalent_built_in_parser.endswith('_Native'):
+        event_product = 'NativeTable'
+        results.append((event_product, '"EventProduct" field is not required since this is a native table parser. Static value will be used for "EventProduct".', 'Pass'))
     # If 'EventProduct' was not found in the KQL query, add to results
     else:
         results.append((f'{RED}EventProduct{RESET}', f'{RED}"EventProduct" field not mapped in parser. Please map it in parser query.{RESET}', f'{RED}Fail{RESET}'))
@@ -136,6 +140,10 @@ def extract_and_check_properties(Parser_file, Union_Parser__file, FileType, Pars
     if match:
         event_vendor = match.group(1)
         results.append((event_vendor, '"EventVendor" field is mapped in parser', 'Pass'))
+    # if equivalent_built_in_parser end with Native, then use 'EventVendor' as 'Microsoft'
+    elif equivalent_built_in_parser.endswith('_Native'):
+        event_vendor = 'Microsoft'
+        results.append((event_vendor, '"EventVendor" field is not required since this is a native table parser. Static value will be used for "EventVendor".', 'Pass'))
     # If 'EventVendor' was not found in the KQL query, add to results
     else:
         results.append((f'{RED}EventVendor{RESET}', f'{RED}"EventVendor" field not mapped in parser. Please map it in parser query.{RESET}', f'{RED}Fail{RESET}'))

.script/tests/asimParsersTest/ingestASimSampleData.py

@@ -259,12 +259,18 @@ def extract_event_vendor_product(parser_query,parser_file):
     match = re.search(r'EventVendor\s*=\s*[\'"]([^\'"]+)[\'"]', parser_query)
     if match:
         event_vendor = match.group(1)
+    # if equivalent_built_in_parser end with Native, then use 'EventVendor' as 'Microsoft'
+    elif equivalent_built_in_parser.endswith('_Native'):
+        event_vendor = 'Microsoft'
     else:
         print(f'EventVendor field not mapped in parser. Please map it in parser query.{parser_file}')
 
     match = re.search(r'EventProduct\s*=\s*[\'"]([^\'"]+)[\'"]', parser_query)
     if match:
         event_product = match.group(1)
+    # if equivalent_built_in_parser end with Native, then use 'EventProduct' as SchemaName + 'NativeTable'
+    elif equivalent_built_in_parser.endswith('_Native'):
+        event_product = 'NativeTable'
     else:
         print(f'Event Product field not mapped in parser. Please map it in parser query.{parser_file}')
     return event_vendor, event_product ,schema_name   
@@ -332,6 +338,7 @@ def convert_data_type(schema_result, data_result):
     parser_query = asim_parser.get('ParserQuery', '')
     normalization = asim_parser.get('Normalization', {})
     schema = normalization.get('Schema')
+    equivalent_built_in_parser = asim_parser.get('EquivalentBuiltInParser')
     event_vendor, event_product, schema_name = extract_event_vendor_product(parser_query, file)
 
     SampleDataFile = f'{event_vendor}_{event_product}_{schema}_IngestedLogs.csv'