Skip to content

Commit

Permalink
Merge branch 'contrib/Mathieu4141_feedly/create-integration' into fee…
Browse files Browse the repository at this point in the history
…dly/create-integration
  • Loading branch information
MichaelYochpaz authored Oct 22, 2023
2 parents 16cc419 + f365923 commit 025b445
Show file tree
Hide file tree
Showing 856 changed files with 19,684 additions and 3,958 deletions.
10 changes: 5 additions & 5 deletions .circleci/config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -148,7 +148,7 @@ references:
echo 'export PYTHONPATH="/home/circleci/project:${PYTHONPATH}"' >> $BASH_ENV
echo 'export DEMISTO_README_VALIDATION=true' >> $BASH_ENV
echo 'export ENV_RESULTS_PATH=/home/circleci/project/artifacts/env_results.json' >> $BASH_ENV
echo 'export GCS_PRODUCTION_BUCKET="marketplace-dist"' >> $BASH_ENV
echo 'export GCS_PRODUCTION_BUCKET="${TEST_XDR_PREFIX}marketplace-dist"' >> $BASH_ENV
echo 'source /home/circleci/project/.circleci/content_release_vars.sh' >> $BASH_ENV
chmod +x .circleci/gitlab-ci-env-variables.sh
./.circleci/gitlab-ci-env-variables.sh
Expand Down Expand Up @@ -210,7 +210,7 @@ references:
[ ! -d ~/.ssh ] && mkdir -p ~/.ssh
chmod 700 ~/.ssh
ssh-keyscan github.com >> ~/.ssh/known_hosts
if [[ "$(echo "$GCS_MARKET_BUCKET" | tr '[:upper:]' '[:lower:]')" != "marketplace-dist" ]]; then
if [[ "$(echo "$GCS_MARKET_BUCKET" | tr '[:upper:]' '[:lower:]')" != "${TEST_XDR_PREFIX}marketplace-dist" ]]; then
echo "Skipping the -Validate Files and Yaml- step when uploading to a test bucket."
exit 0
fi
Expand All @@ -231,7 +231,7 @@ references:
when: always
no_output_timeout: 5h
command: |
if [[ "$(echo "$GCS_MARKET_BUCKET" | tr '[:upper:]' '[:lower:]')" != "marketplace-dist" ]]; then
if [[ "$(echo "$GCS_MARKET_BUCKET" | tr '[:upper:]' '[:lower:]')" != "${TEST_XDR_PREFIX}marketplace-dist" ]]; then
echo "Skipping validations when uploading to a test bucket."
exit 0
fi
Expand All @@ -257,7 +257,7 @@ references:
command: |
EXIT_CODE=0
if [[ -f $ARTIFACTS_FOLDER/coverage_report/.coverage ]]; then
demisto-sdk coverage-analyze -i $ARTIFACTS_FOLDER/coverage_report/.coverage --report-dir $ARTIFACTS_FOLDER/coverage_report --report-type all --previous-coverage-report-url https://storage.googleapis.com/marketplace-dist-dev/code-coverage-reports/coverage-min.json || EXIT_CODE=1
demisto-sdk coverage-analyze -i $ARTIFACTS_FOLDER/coverage_report/.coverage --report-dir $ARTIFACTS_FOLDER/coverage_report --report-type all --previous-coverage-report-url https://storage.googleapis.com/${TEST_XDR_PREFIX}marketplace-dist-dev/code-coverage-reports/coverage-min.json || EXIT_CODE=1
# Checks if the $XSOAR_BOT_TEST_CONTENT exist. for security reasons only non forked pr's have access to it.
if [[ -n $XSOAR_BOT_TEST_CONTENT && -e $ARTIFACTS_FOLDER/coverage_report/html/index.html ]]; then
echo "Adding unit tests coverage comment to the pr"
Expand Down Expand Up @@ -384,7 +384,7 @@ jobs:
command: |
# Download index.zip
INDEX_PATH=$(mktemp)
gsutil cp gs://marketplace-dist/content/packs/index.zip $INDEX_PATH
gsutil cp gs://${TEST_XDR_PREFIX}marketplace-dist/content/packs/index.zip $INDEX_PATH
echo "successfully downloaded index.zip into $INDEX_PATH"
UNZIP_PATH=$(mktemp -d)
Expand Down
2 changes: 1 addition & 1 deletion .github/CODEOWNERS
Validating CODEOWNERS rules …
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@

# PANW Products
/Packs/Palo_Alto_Networks_Enterprise_DLP/ @DeanArbel
/Packs/PAN-OS/Integrations/ @GuyAfik
/Packs/PAN-OS/Integrations/ @GuyAfik @jlevypaloalto
/Packs/PrismaCloudCompute/Integrations/ @GuyAfik
/Packs/PrismaSaasSecurity/Integrations/ @GuyAfik

Expand Down
5 changes: 3 additions & 2 deletions .github/content_roles.json
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,8 @@
"CONTRIBUTION_SECURITY_REVIEWER": "melamedbn",
"ON_CALL_DEVS": [
"tkatzir",
"mlainer"
"mgalitzki"
],
"DOC_REVIEWER": "ShirleyDenkberg"
"DOC_REVIEWER": "ShirleyDenkberg",
"TIM_REVIEWER": "MLainer1"
}
155 changes: 77 additions & 78 deletions .github/workflows/pre-commit.yml
Original file line number Diff line number Diff line change
@@ -1,78 +1,77 @@
name: pre-commit
on: pull_request

jobs:
pre-commit:
runs-on: ubuntu-latest
permissions:
checks: write
pull-requests: write

steps:
- name: Install poetry
uses: Gr1N/setup-poetry@v8

- name: Checkout
uses: actions/checkout@v4
with:
fetch-depth: 0

- name: Setup python
uses: actions/setup-python@v4
with:
python-version: '3.10'
cache: 'poetry'

- name: Install dependencies
run: poetry install

- name: Cache Pre commit
id: cache-pre-commit
uses: actions/cache@v3
with:
path: ~/.cache/pre-commit
key: ${{ runner.os }}-pre-commit

- name: run pre-commit
run: |
source .venv/bin/activate
npm install --no-fund --no-audit
demisto-sdk pre-commit -g --unit-test --validate --no-secrets --show-diff-on-failure --verbose
- name: Publish Test Report
uses: mikepenz/action-junit-report@v3
if: always() # always run even if the previous step fails
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
check_name: Unit Test Report
report_paths: |
**/Packs/**/Integrations/**/.report_pytest.xml
github_token: ${{ secrets.GITHUB_TOKEN }}
detailed_summary: true
annotate_only: true

- name: "Check coverage.xml exists"

id: check_files
uses: andstor/file-existence-action@v2
with:
files: "coverage.xml"


- name: Code Coverage Report
if: steps.check_files.outputs.files_exists == 'true'
uses: irongut/[email protected]
with:
filename: "coverage.xml"
badge: true
format: markdown
output: both
thresholds: '70 80'


- name: Append coverage to summary
if: always()
id: append-coverage
run: |
cat code-coverage-results.md >> $GITHUB_STEP_SUMMARY || echo "Missing coverage report"
name: pre-commit
on: pull_request

jobs:
pre-commit:
runs-on: ubuntu-latest
permissions:
checks: write
pull-requests: write

steps:
- name: Install poetry
uses: Gr1N/setup-poetry@v8

- name: Checkout
uses: actions/checkout@v4
with:
fetch-depth: 0

- name: Setup python
uses: actions/setup-python@v4
with:
python-version: '3.10'
cache: 'poetry'

- name: Install dependencies
run: poetry install

- name: Cache Pre commit
id: cache-pre-commit
uses: actions/cache@v3
with:
path: ~/.cache/pre-commit
key: ${{ runner.os }}-pre-commit

- name: run pre-commit
run: |
source .venv/bin/activate
demisto-sdk pre-commit -g --unit-test --validate --no-secrets --show-diff-on-failure --verbose
- name: Publish Test Report
uses: mikepenz/action-junit-report@v3
if: always() # always run even if the previous step fails
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
check_name: Unit Test Report
report_paths: |
**/Packs/**/Integrations/**/.report_pytest.xml
github_token: ${{ secrets.GITHUB_TOKEN }}
detailed_summary: true
annotate_only: true

- name: "Check coverage.xml exists"

id: check_files
uses: andstor/file-existence-action@v2
with:
files: "coverage.xml"


- name: Code Coverage Report
if: steps.check_files.outputs.files_exists == 'true'
uses: irongut/[email protected]
with:
filename: "coverage.xml"
badge: true
format: markdown
output: both
thresholds: '70 80'


- name: Append coverage to summary
if: always()
id: append-coverage
run: |
cat code-coverage-results.md >> $GITHUB_STEP_SUMMARY || echo "Missing coverage report"
42 changes: 27 additions & 15 deletions .gitlab/ci/.gitlab-ci.bucket-upload.yml
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@
if [[ $TEST_UPLOAD == "false" ]]; then
section_start "Upload packs_dependencies.json to GCP" --collapsed
gcloud auth activate-service-account --key-file="$GCS_ARTIFACTS_KEY" > $ARTIFACTS_FOLDER/logs/gauth.out 2>$ARTIFACTS_FOLDER/logs/gauth.err
gsutil cp $ARTIFACTS_FOLDER/packs_dependencies.json "gs://xsoar-ci-artifacts/content-cache-docs/$MARKETPLACE_VERSION/packs_dependencies.json"
gsutil cp $ARTIFACTS_FOLDER/packs_dependencies.json "gs://${TEST_XDR_PREFIX}xsoar-ci-artifacts/content-cache-docs/$MARKETPLACE_VERSION/packs_dependencies.json"
section_end "Upload packs_dependencies.json to GCP"
fi
Expand Down Expand Up @@ -210,6 +210,20 @@ install-packs-in-server-master:
- ${CI_PROJECT_DIR}/artifacts/*
- ${CI_PROJECT_DIR}/pipeline_jobs_folder/*
when: always
rules:
- if: '$CI_COMMIT_BRANCH =~ /pull\/[0-9]+/'
when: never
- if: $TEST_XDR_ENV && $BUCKET_UPLOAD == "true"
when: always
variables:
CLOUD_MACHINES_TYPE: "testing"
GCS_LOCKS_PATH: "content-locks/locks-xsiam-ga-testing"
- if: $TEST_UPLOAD == "true" && $BUCKET_UPLOAD == "true"
when: always
variables:
CLOUD_MACHINES_TYPE: "build"
GCS_LOCKS_PATH: "content-locks/locks-xsiam-ga"
- if: '$BUCKET_UPLOAD == "true"'
variables:
ARTIFACTS_FOLDER: "${ARTIFACTS_FOLDER_MPV2}"
ENV_RESULTS_PATH: "${ARTIFACTS_FOLDER_MPV2}/env_results.json"
Expand All @@ -219,7 +233,6 @@ install-packs-in-server-master:
CLOUD_MACHINES_TYPE: "upload"
CLOUD_MACHINES_COUNT: 1
extends:
- .bucket-upload-rule
- .default-job-settings
script:
- EXIT_CODE=0
Expand All @@ -231,12 +244,6 @@ install-packs-in-server-master:
- python3 ./Tests/scripts/add_secrets_file_to_build.py -sa "$GSM_SERVICE_ACCOUNT" -sf "$SECRET_CONF_PATH" -u "$DEMISTO_USERNAME" -p "$DEMISTO_PASSWORD" --gsm_project_id_dev "$GSM_PROJECT_ID_DEV" --gsm_project_id_prod "$GSM_PROJECT_ID" >> $ARTIFACTS_FOLDER/logs/handle_secrets.log
- section_end "Secrets Fetch"
- section_start "Lock Machine"
- |
if [[ $TEST_UPLOAD == "true" ]]; then
echo "Setting CLOUD_MACHINES_TYPE to build in test upload-flow"
CLOUD_MACHINES_TYPE="build"
GCS_LOCKS_PATH="content-locks/locks-xsiam-ga"
fi
- echo "Authenticating GCP"
- gcloud auth activate-service-account --key-file="$GCS_ARTIFACTS_KEY" > $ARTIFACTS_FOLDER/logs/gauth.out 2>$ARTIFACTS_FOLDER/logs/gauth.err
- echo "Auth done successfully"
Expand Down Expand Up @@ -284,7 +291,7 @@ install-packs-in-xsiam-ga:
INSTANCE_ROLE: "XSIAM"
GCS_QUEUE_FILE: "queue-ga"
GCS_SOURCE_BUCKET: "$GCS_PRODUCTION_V2_BUCKET"
GCS_MACHINES_BUCKET: "marketplace-v2-dist-dev/upload-flow/builds-xsiam"
GCS_MACHINES_BUCKET: "${TEST_XDR_PREFIX}marketplace-v2-dist-dev/upload-flow/builds-xsiam"
CLOUD_SERVERS_FILE: "xsiam_servers_path"
CLOUD_API_KEYS: $XSIAM_API_KEYS
NON_REMOVABLE_PACKS: "Base"
Expand Down Expand Up @@ -322,7 +329,7 @@ upload-packs-to-marketplace:
PACK_ARTIFACTS=$ARTIFACTS_FOLDER/content_packs.zip
PACKS_DEPENDENCIES=$ARTIFACTS_FOLDER/packs_dependencies.json
CI_COMMIT_BRANCH=${CI_COMMIT_BRANCH:-unknown}
GCS_BUILD_BUCKET="marketplace-ci-build"
GCS_BUILD_BUCKET="${TEST_XDR_PREFIX}marketplace-ci-build"
if [[ $GCS_MARKET_BUCKET == $GCS_PRODUCTION_BUCKET ]] && [[ -z $STORAGE_BASE_PATH ]]; then
STORAGE_BASE_PATH="content"
fi
Expand Down Expand Up @@ -402,7 +409,7 @@ upload-packs-to-marketplace-v2:
PACK_ARTIFACTS=$ARTIFACTS_FOLDER/content_packs.zip
PACKS_DEPENDENCIES=$ARTIFACTS_FOLDER/packs_dependencies.json
CI_COMMIT_BRANCH=${CI_COMMIT_BRANCH:-unknown}
GCS_BUILD_BUCKET="marketplace-ci-build"
GCS_BUILD_BUCKET="${TEST_XDR_PREFIX}marketplace-ci-build"
if [[ -z $STORAGE_BASE_PATH ]]; then
if [[ $GCS_MARKET_V2_BUCKET == $GCS_PRODUCTION_V2_BUCKET ]]; then
STORAGE_BASE_PATH="content"
Expand Down Expand Up @@ -472,7 +479,7 @@ upload-packs-to-xpanse-marketplace:
PACK_ARTIFACTS=$ARTIFACTS_FOLDER/content_packs.zip
PACKS_DEPENDENCIES=$ARTIFACTS_FOLDER/packs_dependencies.json
CI_COMMIT_BRANCH=${CI_COMMIT_BRANCH:-unknown}
GCS_BUILD_BUCKET="marketplace-ci-build"
GCS_BUILD_BUCKET="${TEST_XDR_PREFIX}marketplace-ci-build"
if [[ -z $STORAGE_BASE_PATH ]]; then
if [[ $GCS_MARKET_XPANSE_BUCKET == $GCS_PRODUCTION_XPANSE_BUCKET ]]; then
STORAGE_BASE_PATH="content"
Expand Down Expand Up @@ -545,7 +552,7 @@ upload-packs-to-xsoar-saas-marketplace:
PACK_ARTIFACTS=$ARTIFACTS_FOLDER/content_packs.zip
PACKS_DEPENDENCIES=$ARTIFACTS_FOLDER/packs_dependencies.json
CI_COMMIT_BRANCH=${CI_COMMIT_BRANCH:-unknown}
GCS_BUILD_BUCKET="marketplace-ci-build"
GCS_BUILD_BUCKET="${TEST_XDR_PREFIX}marketplace-ci-build"
if [[ -z $STORAGE_BASE_PATH ]]; then
if [[ "$GCS_MARKET_XSOAR_SAAS_BUCKET" == "$GCS_PRODUCTION_XSOAR_SAAS_BUCKET" ]]; then
STORAGE_BASE_PATH="content"
Expand Down Expand Up @@ -630,7 +637,7 @@ xpanse-force-pack-upload:
- PACK_ARTIFACTS=$ARTIFACTS_FOLDER/content_packs.zip
- PACKS_DEPENDENCIES=$ARTIFACTS_FOLDER/packs_dependencies.json
- CI_COMMIT_BRANCH=${CI_COMMIT_BRANCH:-unknown}
- GCS_BUILD_BUCKET="marketplace-ci-build"
- GCS_BUILD_BUCKET="${TEST_XDR_PREFIX}marketplace-ci-build"
- |
if [[ $GCS_MARKET_BUCKET_TO_UPLOAD == $GCS_CURRENT_PRODUCTION_BUCKET ]] && [[ -z $STORAGE_BASE_PATH ]]; then
STORAGE_BASE_PATH="content"
Expand Down Expand Up @@ -708,6 +715,11 @@ sync-buckets-between-projects:
echo "Skipping syncing buckets in test upload-flow"
exit 0
fi
if [[ $TEST_XDR_ENV == "true" ]]; then
echo "Skipping syncing buckets in xdr env when testing"
exit 0
fi
- |
if [[ -z "$GCS_XSOAR_CONTENT_DEV_KEY" ]] || [[ -z "$GCS_XSOAR_CONTENT_PROD_KEY" ]]; then
Expand All @@ -718,7 +730,7 @@ sync-buckets-between-projects:
gcloud auth activate-service-account --key-file="$GCS_XSOAR_CONTENT_DEV_KEY"
echo "Syncing gs://marketplace-xsoar-dev"
gsutil -m rsync -r gs://marketplace-dist gs://marketplace-xsoar-dev
gsutil -m rsync -r gs://marketplace-saas-dist gs://marketplace-xsoar-dev
echo "Syncing gs://marketplace-xsiam-dev"
gsutil -m rsync -r gs://marketplace-v2-dist gs://marketplace-xsiam-dev
echo "Syncing gs://marketplace-xpanse-dev"
Expand Down
2 changes: 1 addition & 1 deletion .gitlab/ci/.gitlab-ci.build-machines-cleanup.yml
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ build-machines-cleanup:
GCS_MARKET_BUCKET: "${GCS_MARKET_V2_BUCKET}"
ENV_RESULTS_PATH: "${ARTIFACTS_FOLDER_MPV2}/env_results.json"
GCS_SOURCE_BUCKET: "$GCS_PRODUCTION_V2_BUCKET"
GCS_MACHINES_BUCKET: "marketplace-v2-dist-dev/upload-flow/builds-xsiam"
GCS_MACHINES_BUCKET: "${TEST_XDR_PREFIX}marketplace-v2-dist-dev/upload-flow/builds-xsiam"
SERVER_TYPE: "XSIAM"
MARKETPLACE_NAME: "marketplacev2"
NON_REMOVABLE_PACKS: "Base"
Expand Down
8 changes: 4 additions & 4 deletions .gitlab/ci/.gitlab-ci.global.yml
Original file line number Diff line number Diff line change
Expand Up @@ -257,7 +257,7 @@
if [[ -e "$ARTIFACTS_FOLDER/logs/lock_file.txt" ]]; then
echo "Job finished, removing lock file for machine ids:${CLOUD_CHOSEN_MACHINE_IDS}"
gcloud auth activate-service-account --key-file="$GCS_ARTIFACTS_KEY" > $ARTIFACTS_FOLDER/logs/gauth.out 2>$ARTIFACTS_FOLDER/logs/gauth.err
gsutil rm "gs://xsoar-ci-artifacts/$GCS_LOCKS_PATH/machines_locks/*-lock-$CI_JOB_ID"
gsutil rm "gs://${TEST_XDR_PREFIX}xsoar-ci-artifacts/$GCS_LOCKS_PATH/machines_locks/*-lock-$CI_JOB_ID"
echo "Finished removing lock file"
else
echo "No lock file found, skipping unlocking"
Expand Down Expand Up @@ -338,7 +338,7 @@
- section_end "Test Infrastructure"
- section_start "Run Unit Testing and Lint"
- |
if [[ -n $FORCE_BUCKET_UPLOAD || -n $BUCKET_UPLOAD ]] && [[ "$(echo "$GCS_MARKET_BUCKET" | tr '[:upper:]' '[:lower:]')" != "marketplace-dist" ]] && [[ $CI_COMMIT_BRANCH != "master" ]]; then
if [[ -n $FORCE_BUCKET_UPLOAD || -n $BUCKET_UPLOAD ]] && [[ "$(echo "$GCS_MARKET_BUCKET" | tr '[:upper:]' '[:lower:]')" != "${TEST_XDR_PREFIX}marketplace-dist" ]] && [[ $CI_COMMIT_BRANCH != "master" ]]; then
echo "Skipping validations when uploading to a test bucket."
else
echo "demisto-sdk version: $(demisto-sdk --version)"
Expand Down Expand Up @@ -380,7 +380,7 @@
python3 Utils/upload_code_coverage_report.py --service_account $GCS_MARKET_KEY --source_file_name $ARTIFACTS_FOLDER/coverage_report/coverage.json --minimal_file_name $ARTIFACTS_FOLDER/coverage_report/coverage-min.json
fi
else
demisto-sdk coverage-analyze -i $ARTIFACTS_FOLDER/coverage_report/.coverage --report-dir $ARTIFACTS_FOLDER/coverage_report --report-type html,xml --previous-coverage-report-url https://storage.googleapis.com/marketplace-dist-dev/code-coverage-reports/coverage-min.json
demisto-sdk coverage-analyze -i $ARTIFACTS_FOLDER/coverage_report/.coverage --report-dir $ARTIFACTS_FOLDER/coverage_report --report-type html,xml --previous-coverage-report-url https://storage.googleapis.com/${TEST_XDR_PREFIX}marketplace-dist-dev/code-coverage-reports/coverage-min.json
fi
fi
fi
Expand Down Expand Up @@ -428,7 +428,7 @@
- |
gcloud auth activate-service-account --key-file="$GCS_MARKET_KEY" >> $ARTIFACTS_FOLDER/logs/auth.out
echo "successfully activated google cloud service account"
gsutil cp "gs://marketplace-dist/content/packs/index.zip" $INDEX_PATH
gsutil cp "gs://${TEST_XDR_PREFIX}marketplace-dist/content/packs/index.zip" $INDEX_PATH
echo "successfully downloaded index.zip"
gcloud auth revoke $GCS_ARTIFACTS_ACCOUNT_NAME
- echo "successfully downloaded index.zip into $INDEX_PATH"
Expand Down
Loading

0 comments on commit 025b445

Please sign in to comment.