Merge branch 'master' into patch-30

.github/workflows/codeql-analysis.yml

@@ -49,6 +49,7 @@ jobs:
         # By default, queries listed here will override any specified in a config file.
         # Prefix the list here with "+" to use these queries and those in the config file.
         # queries: ./path/to/local/query, your-org/your-repo/queries@main
+        queries: security-extended,security-and-quality
 
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)

.github/workflows/convertKqlFuncYamlToArmTemplate.yaml

@@ -2,7 +2,7 @@
 # The script generates deployable ARM templates based on ASim parsers YAML files and pushes them to the pull request branch.
 name: Convert Kql function yaml to ARM template
 on:
-  pull_request_target:
+  pull_request:
     types: [opened, edited, reopened, synchronize, labeled]
     paths:
        - 'Parsers/ASimDns/Parsers/**'
@@ -11,6 +11,8 @@ on:
        - 'Parsers/ASimProcessEvent/Parsers/**'
 jobs:
   kqlFuncYaml2Arm:
+    # The workflow should not run on forked repositories for security reasons
+    if: ${{ !github.event.pull_request.head.repo.fork }}
     runs-on: ubuntu-latest
     steps:
       - name: Checkout pull request branch

.github/workflows/runAsimTesters.yaml

@@ -1,6 +1,6 @@
 name: Run ASIM testers on "eco-connector-test" workspace
 on:
-  pull_request_target:
+  pull_request:
     types: [opened, edited, reopened, synchronize, labeled]
     paths:
     - 'Parsers/ASimDns/Parsers/**'
@@ -17,6 +17,8 @@ permissions:
 
 jobs: 
   runAsimTesters:
+      # The workflow should not run on forked repositories for security reasons
+      if: ${{ !github.event.pull_request.head.repo.fork }}
       runs-on: ubuntu-latest
       steps:
       - name: Checkout pull request branch

.script/dataConnectorValidator.ts

@@ -74,7 +74,7 @@ function getConnectorCategory(dataTypes : any, instructionSteps:[])
     {
         return ConnectorCategory.AzureFunction;
     }
-    else if((dataTypes[0].name.includes("meraki") || dataTypes[0].name.includes("vcenter")) && JSON.stringify(instructionSteps).includes("\"type\":\"InstallAgent\""))
+    else if((dataTypes[0].name.includes("meraki") || dataTypes[0].name.includes("vCenter")) && JSON.stringify(instructionSteps).includes("\"type\":\"InstallAgent\""))
     {
         return ConnectorCategory.SysLog;
     }

.script/idChangeValidator.ts

@@ -3,48 +3,78 @@ import { GetPRDetails } from "./utils/gitWrapper";
 import { ExitCode } from "./utils/exitCode";
 import * as logger from "./utils/logger";
 import gitP, { SimpleGit } from 'simple-git/promise';
+import { readFileSync } from 'fs';
 
-const workingDir:string = process.cwd();
-const guidRegex:string = "[0-9a-fA-F]{8}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{12}";
-const templateIdRegex:string = `(id: ${guidRegex}(.|\n)*){2}`;
+const workingDir: string = process.cwd();
+const guidRegex: string = "[0-9a-fA-F]{8}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{12}";
+const templateIdRegex: string = `(id: ${guidRegex}(.|\n)*){2}`;
 const git: SimpleGit = gitP(workingDir);
 
 export async function IsIdHasChanged(filePath: string): Promise<ExitCode> {
 
-  if(filePath.includes("Detections") || filePath.includes("Analytic Rules"))
-  {
-    filePath = workingDir + '/' + filePath;
-    const pr = await GetPRDetails();
-    console.log(filePath);
-
-    if (typeof pr === "undefined") {
-      console.log("Azure DevOps CI for a Pull Request wasn't found. If issue persists - please open an issue");
-      return ExitCode.ERROR;
-    }
-
-    let options = [pr.targetBranch, pr.sourceBranch, filePath];
-    let diffSummary = await git.diff(options);
-    let idHasChanged = diffSummary.search(templateIdRegex) > 0;
-    if (idHasChanged){
-        throw new Error();
+    var skipValidationCheckFilePath = workingDir + "/.script/tests/idChangeValidatorTest/SkipIdValidationsTemplates.json";
+    console.log("skipValidationCheckFilePath: " + skipValidationCheckFilePath);
+    var skipIdsFile = JSON.parse(readFileSync(skipValidationCheckFilePath, 'utf8'));
+    console.log(skipIdsFile + " " + typeof (skipIdsFile));
+
+    if (filePath.includes("Detections") || filePath.includes("Analytic Rules")) {
+        filePath = workingDir + '/' + filePath;
+        const pr = await GetPRDetails();
+        console.log(filePath);
+
+        if (typeof pr === "undefined") {
+            console.log("Azure DevOps CI for a Pull Request wasn't found. If issue persists - please open an issue");
+            return ExitCode.ERROR;
+        }
+
+        let options = [pr.targetBranch, pr.sourceBranch, filePath];
+        let diffSummary = await git.diff(options);
+        let idPosition = diffSummary.search(templateIdRegex);
+        let idHasChanged = idPosition > 0;
+
+        if (idHasChanged) {
+
+            //const regexp = new RegExp('[0-9a-fA-F]{8}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{12}', 'g');
+            //console.log(typeof (regexp));
+
+            const regex = RegExp('[0-9a-fA-F]{8}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{12}', 'g');
+            let array1;
+            var oldId: string = "";
+            var newId: string = "";
+
+            while ((array1 = regex.exec(diffSummary)) !== null) {
+                if (oldId == "") {
+                    oldId = array1[0];
+                } else {
+                    newId = array1[0];
+                }
+            }
+            console.log(`Found ${oldId} and ${newId}.`);
+
+            if (skipIdsFile.indexOf(newId) > -1) {
+                console.log(filePath + " is skipped from this validation.");
+                return ExitCode.SUCCESS;
+            } else {
+                throw new Error();
+            }
+        }
     }
-  }
-  return ExitCode.SUCCESS;
+    return ExitCode.SUCCESS;
 }
 
 let fileKinds = ["Modified"];
 let fileTypeSuffixes = ["yaml", "yml", "json"];
 let filePathFolderPrefixes = ["Detections", "Solutions"];
 let CheckOptions = {
-  onCheckFile: (filePath: string) => {
-    return IsIdHasChanged(filePath);
-  },
-  onExecError: async (e: any, filePath: string) => {
-    console.log(`${e}: Id of file - "${filePath}" has changed, please make sure you do not change any file id.`);
-  },
-  onFinalFailed: async () => {
-    logger.logError("An error occurred, please open an issue");
-  }
+    onCheckFile: (filePath: string) => {
+        return IsIdHasChanged(filePath);
+    },
+    onExecError: async (e: any, filePath: string) => {
+        console.log(`${e}: Id of file - "${filePath}" has changed, please make sure you do not change any file id.`);
+    },
+    onFinalFailed: async () => {
+        logger.logError("An error occurred, please open an issue");
+    }
 };
 
 runCheckOverChangedFiles(CheckOptions, fileKinds, fileTypeSuffixes, filePathFolderPrefixes);

.script/tests/KqlvalidationsTests/CustomTables/AliCloud_CL.json

@@ -0,0 +1,117 @@
+{
+   "name": "AliCloud_CL",
+   "Properties": [
+       {
+           "Name": "TimeGenerated",
+           "Type": "DateTime"
+       },
+       {
+           "Name": "EventVendor",
+           "Type": "String"
+       },
+       {
+           "Name": "EventProduct",
+           "Type": "String"
+       },
+       {
+           "Name": "EventCount",
+           "Type": "String"
+       },
+       {
+           "Name": "CreatedAt",
+           "Type": "DateTime"
+       },
+       {
+           "Name": "EventEndTime",
+           "Type": "DateTime"
+       },
+       {
+           "Name": "ContentTopic",
+           "Type": "String"
+       },
+       {
+           "Name": "SourceName",
+           "Type": "String"
+       },
+       {
+           "Name": "AcsRegion",
+           "Type": "String"
+       },
+       {
+           "Name": "EventSubType",
+           "Type": "String"
+       },
+       {
+           "Name": "EventResult",
+           "Type": "String"
+       },
+       {
+           "Name": "EventOriginalUid",
+           "Type": "String"
+       },
+       {
+           "Name": "EventOriginalType",
+           "Type": "String"
+       },
+       {
+           "Name": "AdditionalEventData",
+           "Type": "String"
+       },
+       {
+           "Name": "EventRW",
+           "Type": "String"
+       },
+       {
+           "Name": "EventSource",
+           "Type": "String"
+       },
+       {
+           "Name": "EventVersion",
+           "Type": "String"
+       },
+       {
+           "Name": "RequestId",
+           "Type": "String"
+       },
+       {
+           "Name": "RequestParameterJson",
+           "Type": "String"
+       },
+       {
+           "Name": "RequestParameters",
+           "Type": "String"
+       },
+       {
+           "Name": "ServiceName",
+           "Type": "String"
+       },
+       {
+           "Name": "SrcIpAddr",
+           "Type": "String"
+       },
+       {
+           "Name": "UserAgent",
+           "Type": "String"
+       },
+       {
+           "Name": "UserIdentity",
+           "Type": "String"
+       },
+       {
+           "Name": "Type",
+           "Type": "String"
+       },
+       {
+           "Name": "apiVersion",
+           "Type": "Datetime"
+       },
+       {
+           "Name": "TenanId",
+           "Type": "String"
+       },
+       {
+           "Name": "SourseSystem",
+           "Type": "String"
+       }
+   ]
+}