Merge pull request #7 from demisto/master

Pulled latest changes from demisto/content

.circleci/config.yml

@@ -63,6 +63,9 @@ parameters:
   nightly:
     type: string
     default: ""
+  legacy_nightly:
+    type: string
+    default: ""
   instance_tests:
     type: string
     default: ""
@@ -122,6 +125,7 @@ references:
       TIME_TO_LIVE_PARAMETER: << pipeline.parameters.time_to_live >>
       PULL_REQUEST_NUMBER: << pipeline.parameters.pr_number >>
       NIGHTLY_PARAMETER: << pipeline.parameters.nightly >>
+      LEGACY_NIGHTLY_PARAMETER: << pipeline.parameters.legacy_nightly >>
       INSTANCE_TESTS_PARAMETER: << pipeline.parameters.instance_tests >>
       DEMISTO_SDK_NIGHTLY_PARAMETER: << pipeline.parameters.demisto_sdk_nightly >>
       FORCE_PACK_UPLOAD_PARAMETER: << pipeline.parameters.force_pack_upload >>
@@ -168,7 +172,7 @@ references:
         echo 'source /home/circleci/project/.circleci/content_release_vars.sh' >> $BASH_ENV
         chmod +x .circleci/gitlab-ci-env-variables.sh
         ./.circleci/gitlab-ci-env-variables.sh
-        if [ -n "${NIGHTLY_PARAMETER}" ];
+        if [ -n "${NIGHTLY_PARAMETER}" ] || [ -n "${LEGACY_NIGHTLY_PARAMETER}" ];
         then
             echo 'export NIGHTLY=true' >> $BASH_ENV
         fi
@@ -221,7 +225,7 @@ references:
         git config diff.renameLimit 6000
 
         if [ -n "${DEMISTO_SDK_NIGHTLY}" ]; then
-          pip3 install git+https://github.com/demisto/demisto-sdk@master
+          pip3 install git+https://github.com/demisto/demisto-sdk.git@"$SDK_REF"
         fi
 
         echo "========== Build Parameters =========="
@@ -522,6 +526,12 @@ references:
               SERVER_URL=$(jq -r 'select(.[].Role == "Server Master") | .[].InstanceDNS' $ENV_RESULTS_PATH)
               python3 ./Utils/comment_on_pr.py -p $PULL_REQUEST_NUMBER -c "Instance is ready. Server link: https://$SERVER_URL, Build link: $CIRCLE_BUILD_URL"
 
+  change_file_ids: &change_file_ids
+    run:
+      name: Change File IDs to prevent conflicts in XSOAR
+      command:
+        python3 ./Tests/scripts/sdk_nightly_change_json_file_fields.py Packs/HelloWorld/Classifiers/classifier-mapper-incoming-HelloWorldTest.json name
+
   upload_entities_to_cortex_xsoar: &upload_entities_to_cortex_xsoar
     run:
         name: Upload entities to Cortex XSOAR
@@ -549,6 +559,17 @@ references:
           - Create Instances
 
   nightly_jobs: &nightly_jobs
+    - Setup Environment:
+        context: nightly_env
+    - Run Unit Testing And Lint:
+        context: nightly_env
+        requires:
+          - Setup Environment
+    - Run Validations:
+        requires:
+          - Setup Environment
+
+  legacy_nightly_jobs: &legacy_nightly_jobs
     - Setup Environment:
         context: nightly_env
     - Create Instances:
@@ -1192,11 +1213,6 @@ jobs:
             echo "export DEMISTO_API_KEY=$(cat $SECRET_CONF_PATH | jq -r '.temp_apikey')" >> $BASH_ENV
             echo "export DEMISTO_BASE_URL=\"https://localhost:$(cat $ENV_RESULTS_PATH | jq -r '.[0].TunnelPort')\"" >> $BASH_ENV
             echo "Server URL: $DEMISTO_BASE_URL"
-      - run:
-          name: Install SDK
-          command: |
-            echo "Installing SDK from $SDK_REF"
-            pip3 install git+https://github.com/demisto/demisto-sdk.git@"$SDK_REF"
       - *wait_until_server_ready
       - run:
           name: Unlock HelloWorld integration and playbook
@@ -1222,6 +1238,7 @@ jobs:
       - run:
           name: Run lint
           command: demisto-sdk lint -i Packs/HelloWorld/
+      - *change_file_ids
       - *upload_entities_to_cortex_xsoar
       - *destroy_instances
       - *store_artifacts
@@ -1306,6 +1323,7 @@ workflows:
           - << pipeline.parameters.force_pack_upload >>
           - << pipeline.parameters.bucket_upload >>
           - << pipeline.parameters.demisto_sdk_nightly >>
+          - << pipeline.parameters.legacy_nightly >>
           - << pipeline.parameters.nightly >>
           - << pipeline.parameters.instance_tests >>
     jobs:
@@ -1367,6 +1385,14 @@ workflows:
     jobs:
       *nightly_jobs
 
+  legacy_nightly_trigger:
+    # considered legacy since we now run the nightly (test playbooks against live xsoar instances) in gitlab
+    # use to trigger a fullblown nightly in circle including the jobs to test against xsoar instances
+    # will initiate when using the trigger script with the appropriate option.
+    when: << pipeline.parameters.legacy_nightly >>
+    jobs:
+      *legacy_nightly_jobs
+
   sdk_nightly:
     triggers:
       - schedule:

.circleci/content_release_vars.sh

@@ -1,3 +1,3 @@
 export CONTENT_VERSION="21.7.0"
 export SERVER_VERSION="6.0.0"
-export GIT_SHA1="74720f776b2c872e28d2cfb012d48403fb1b832c" # guardrails-disable-line disable-secrets-detection
+export GIT_SHA1="74720f776b2c872e28d2cfb012d48403fb1b832c" # guardrails-disable-line disable-secrets-detection

.github/workflows/check-contribution-form-filled.yml

@@ -9,7 +9,8 @@ permissions:
 jobs:
   contribution_form_filled_verification:
     runs-on: ubuntu-latest
-    if: github.repository == 'demisto/content' && github.event.pull_request.head.repo.fork == true && contains(github.head_ref, 'xsoar-bot-contrib-ContributionTestPack') == false && contains(github.event.pull_request.labels.*.name, 'Contribution Form Filled') == false
+    # This status check only runs for: forked PRS that don't contain the '[Marketplace Contribution]' in their title, the head branch is not a test branch and the PR don't have the 'Contribution Form Filled' label.
+    if: github.repository == 'demisto/content' && github.event.pull_request.head.repo.fork == true && contains(github.head_ref, 'xsoar-bot-contrib-ContributionTestPack') == false && contains(github.event.pull_request.labels.*.name, 'Contribution Form Filled') == false && contains(github.event.pull_request.title, '[Marketplace Contribution]') == false
     steps:
       - name: Checkout
         uses: actions/checkout@v2
@@ -32,4 +33,4 @@ jobs:
         run: |
           echo "Checking if contribution form needs to be filled for PR: $PR_NUMBER"
           cd Utils/github_workflow_scripts
-          pipenv run ./check_if_needs_to_fill_contribution_form.py --pr_number $PR_NUMBER --github_token $GITHUB_TOKEN
+          pipenv run ./check_if_needs_to_fill_contribution_form.py --pr_number $PR_NUMBER --github_token $GITHUB_TOKEN

.gitlab/ci/on-push.yml

@@ -106,6 +106,7 @@ create-instances:
     - !reference [.open-ssh-tunnel]
     - python3 ./Tests/scripts/wait_until_server_ready.py "$INSTANCE_ROLE"
     - ./Tests/scripts/install_content_and_test_integrations.sh "$INSTANCE_ROLE" || EXIT_CODE=$?
+    - cp -f $ARTIFACTS_FOLDER/conf.json Tests/conf.json
     - ./Tests/scripts/run_tests.sh "$INSTANCE_ROLE" || EXIT_CODE=$?
     - |
       if [ -f ./Tests/failed_tests.txt ]; then

.vscode/extensions.json

@@ -4,6 +4,7 @@
 
     // List of extensions which should be recommended for users of this workspace.
     "recommendations": [
+        "cortexxsoarext.xsoar",
         "ms-python.python",
         "ms-python.vscode-pylance",
         "ms-toolsai.jupyter",

Packs/AWS-EC2/.pack-ignore

@@ -5,4 +5,4 @@ ignore=BA108,BA109
 ignore=RM106
 
 [file:playbook-IP_Whitelist_-_AWS_Security_Group.yml]
-ignore=PB113
+ignore=PB114

Packs/AbnormalSecurity/.secrets-ignore

@@ -0,0 +1,3 @@
+https://api.abnormalplatform.com/v1
+100.101.102.103
+[email protected]