diff --git a/.github/workflows/detection-template-schema-validations.yaml b/.github/workflows/detection-template-schema-validations.yaml index ece93382c47..78591e45237 100644 --- a/.github/workflows/detection-template-schema-validations.yaml +++ b/.github/workflows/detection-template-schema-validations.yaml @@ -8,7 +8,7 @@ on: workflow_dispatch: jobs: DetectionTemplateSchemaValidation: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 env: buildConfiguration: Release dotnetSdkVersion: 3.1.401 diff --git a/.github/workflows/non-ascii-validations.yaml b/.github/workflows/non-ascii-validations.yaml index ca37bec3a71..ddca1038d0b 100644 --- a/.github/workflows/non-ascii-validations.yaml +++ b/.github/workflows/non-ascii-validations.yaml @@ -9,7 +9,7 @@ on: workflow_dispatch: jobs: NonAsciiValidations: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 env: buildConfiguration: Release dotnetSdkVersion: 3.1.401 diff --git a/.script/tests/asimParsersTest/ASimFilteringTest.py b/.script/tests/asimParsersTest/ASimFilteringTest.py index dd98dcfc2a2..c9085d672fa 100644 --- a/.script/tests/asimParsersTest/ASimFilteringTest.py +++ b/.script/tests/asimParsersTest/ASimFilteringTest.py @@ -18,7 +18,7 @@ # Workspace ID for the Log Analytics workspace where the ASim filtering tests will be performed. WORKSPACE_ID = "e9beceee-7d61-429f-a177-ee5e2b7f481a" # Timespan for the parser query -TIME_SPAN_IN_DAYS = 7 +TIME_SPAN_IN_DAYS = 2 # exclusion_file_path refers to the CSV file path containing a list of parsers. Despite failing tests, these parsers will not cause the overall workflow to fail exclusion_file_path = '.script/tests/asimParsersTest/ExclusionListForASimTests.csv' @@ -309,6 +309,7 @@ def main(): if parser_file['EquivalentBuiltInParser'] in read_exclusion_list_from_csv(): print(f"{YELLOW}The parser {parser_file_path} is listed in the exclusions file. Therefore, this workflow run will not fail because of it. To allow this parser to cause the workflow to fail, please remove its name from the exclusions list file located at: {exclusion_file_path}{RESET}") sys.stdout.flush() + continue # Check for exception cases where the failure can be ignored # Check if the failure message and schema match the exception cases if len(result.failures) == 1: diff --git a/.script/tests/asimParsersTest/VerifyASimParserTemplate.py b/.script/tests/asimParsersTest/VerifyASimParserTemplate.py index 2b78f95f9a7..11ace0107d7 100644 --- a/.script/tests/asimParsersTest/VerifyASimParserTemplate.py +++ b/.script/tests/asimParsersTest/VerifyASimParserTemplate.py @@ -125,6 +125,10 @@ def extract_and_check_properties(Parser_file, Union_Parser__file, FileType, Pars if match: event_product = match.group(1) results.append((event_product, '"EventProduct" field is mapped in parser', 'Pass')) + # if equivalent_built_in_parser end with Native, then use 'EventProduct' as SchemaName + 'NativeTable' + elif equivalent_built_in_parser.endswith('_Native'): + event_product = 'NativeTable' + results.append((event_product, '"EventProduct" field is not required since this is a native table parser. Static value will be used for "EventProduct".', 'Pass')) # If 'EventProduct' was not found in the KQL query, add to results else: results.append((f'{RED}EventProduct{RESET}', f'{RED}"EventProduct" field not mapped in parser. Please map it in parser query.{RESET}', f'{RED}Fail{RESET}')) @@ -136,6 +140,10 @@ def extract_and_check_properties(Parser_file, Union_Parser__file, FileType, Pars if match: event_vendor = match.group(1) results.append((event_vendor, '"EventVendor" field is mapped in parser', 'Pass')) + # if equivalent_built_in_parser end with Native, then use 'EventVendor' as 'Microsoft' + elif equivalent_built_in_parser.endswith('_Native'): + event_vendor = 'Microsoft' + results.append((event_vendor, '"EventVendor" field is not required since this is a native table parser. Static value will be used for "EventVendor".', 'Pass')) # If 'EventVendor' was not found in the KQL query, add to results else: results.append((f'{RED}EventVendor{RESET}', f'{RED}"EventVendor" field not mapped in parser. Please map it in parser query.{RESET}', f'{RED}Fail{RESET}')) diff --git a/.script/tests/asimParsersTest/ingestASimSampleData.py b/.script/tests/asimParsersTest/ingestASimSampleData.py index 8de99b8aa1d..a92c5384dba 100644 --- a/.script/tests/asimParsersTest/ingestASimSampleData.py +++ b/.script/tests/asimParsersTest/ingestASimSampleData.py @@ -259,12 +259,18 @@ def extract_event_vendor_product(parser_query,parser_file): match = re.search(r'EventVendor\s*=\s*[\'"]([^\'"]+)[\'"]', parser_query) if match: event_vendor = match.group(1) + # if equivalent_built_in_parser end with Native, then use 'EventVendor' as 'Microsoft' + elif equivalent_built_in_parser.endswith('_Native'): + event_vendor = 'Microsoft' else: print(f'EventVendor field not mapped in parser. Please map it in parser query.{parser_file}') match = re.search(r'EventProduct\s*=\s*[\'"]([^\'"]+)[\'"]', parser_query) if match: event_product = match.group(1) + # if equivalent_built_in_parser end with Native, then use 'EventProduct' as SchemaName + 'NativeTable' + elif equivalent_built_in_parser.endswith('_Native'): + event_product = 'NativeTable' else: print(f'Event Product field not mapped in parser. Please map it in parser query.{parser_file}') return event_vendor, event_product ,schema_name @@ -332,6 +338,7 @@ def convert_data_type(schema_result, data_result): parser_query = asim_parser.get('ParserQuery', '') normalization = asim_parser.get('Normalization', {}) schema = normalization.get('Schema') + equivalent_built_in_parser = asim_parser.get('EquivalentBuiltInParser') event_vendor, event_product, schema_name = extract_event_vendor_product(parser_query, file) SampleDataFile = f'{event_vendor}_{event_product}_{schema}_IngestedLogs.csv'