Ceph s3 Tests | Separate configuration from Running Tests + GitHub Action

.eslintignore

@@ -1 +1 @@
-src/test/system_tests/s3-tests/**
+src/test/system_tests/ceph_s3_tests/s3-tests/**

.github/workflows/ceph-s3-tests.yaml

@@ -0,0 +1,80 @@
+name: ceph-s3-tests
+on: 
+  workflow_dispatch:
+  schedule:
+    - cron: "00 22,1 * * *" # Runs every day at 22:00 and 1:00 UTC
+
+jobs:
+  ceph-s3-tests:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout noobaa-core
+        uses: actions/checkout@v3
+        with:
+          repository: 'noobaa/noobaa-core'
+          path: 'noobaa-core'
+
+      - name: Deploy minikube
+        run: |
+         cd ./noobaa-core
+         sudo bash ./.travis/deploy_minikube.sh
+
+      - name: Build noobaa tester
+        run: |
+         cd ./noobaa-core
+         make tester TESTER_TAG=noobaa-tester:s3-tests
+         docker tag noobaa:latest noobaa-core:s3-tests
+
+      - name: Checkout noobaa-operator
+        uses: actions/checkout@v3
+        with:
+          repository: 'noobaa/noobaa-operator'
+          path: 'noobaa-operator'
+
+      - name: Change settings for k8s and minikube
+        run: |
+          sudo mv /root/.kube /root/.minikube $HOME
+          sudo chown -R $USER $HOME/.kube $HOME/.minikube
+          sed "s/root/home\/$USER/g" $HOME/.kube/config > tmp; mv tmp $HOME/.kube/config
+
+      - name: Build operator
+        run: |
+         cd ./noobaa-operator
+         make all
+
+      - name: Install noobaa system
+        run: |
+         cd ./noobaa-operator
+         ./build/_output/bin/noobaa-operator crd create
+         ./build/_output/bin/noobaa-operator operator install
+         ./build/_output/bin/noobaa-operator system create  \
+         --db-resources='{ "limits": {"cpu": "200m","memory": "1G"}, "requests": {"cpu": "200m","memory": "1G"}}' \
+         --core-resources='{ "limits": {"cpu": "200m","memory": "1G"}, "requests": {"cpu": "200m","memory": "1G"}}' \
+         --endpoint-resources='{ "limits": {"cpu": "200m","memory": "1G"}, "requests": {"cpu": "200m","memory": "1G"}}' \
+         --noobaa-image='noobaa-core:s3-tests'
+         ./build/_output/bin/noobaa-operator status
+
+      - name: Run Ceph s3-tests
+        run: |
+          set -x
+          kubectl wait --for=condition=available backingstore/noobaa-default-backing-store --timeout=3m
+          # we added th sleep since the test pool is in phase ready and condition available
+          # but the test pool storage is not ready yet, see issue:
+          # https://github.com/noobaa/noobaa-operator/issues/1007
+          sleep 3m
+          cd ./noobaa-core
+          kubectl apply -f ./src/test/system_tests/ceph_s3_tests/test_ceph_s3_job.yml
+          kubectl wait --for=condition=complete job/noobaa-tests-s3 --timeout=30m || TIMEOUT=true
+          kubectl logs job/noobaa-tests-s3 --tail 10000 -f
+          if kubectl logs job/noobaa-tests-s3 | grep -q "Ceph Test Failed:"; then
+            echo "At least one test failed!"
+            exit 1
+          fi
+          if [ ${TIMEOUT} ]; then
+            echo "Timed out waiting for the condition on jobs/noobaa-tests-s3"
+            exit 1
+          fi
+          if kubectl logs job/noobaa-tests-s3 | grep -q "ran 0 tests"; then
+            echo "The s3 tests did not run!"
+            exit 1
+          fi

src/deploy/NVA_build/Tests.Dockerfile

@@ -36,8 +36,8 @@ WORKDIR /root/node_modules/noobaa-core/
 #   Size: ~ 83.9 MB
 #
 ##############################################################
-RUN ./src/test/system_tests/ceph_s3_tests_deploy.sh $(pwd)
-RUN cd ./src/test/system_tests/s3-tests/ && \
+RUN ./src/test/system_tests/ceph_s3_tests/test_ceph_s3_deploy.sh $(pwd)
+RUN cd ./src/test/system_tests/ceph_s3_tests/s3-tests/ && \
     ./bootstrap
 
 ##############################################################

src/test/system_tests/ceph_s3_tests/s3-tests-lists/s3_tests_pending_list.txt

@@ -0,0 +1,93 @@
+s3tests_boto3.functional.test_s3.test_get_bucket_policy_status
+s3tests.functional.test_s3.test_atomic_dual_conditional_write_1mb
+s3tests_boto3.functional.test_s3.test_bucket_listv2_encoding_basic
+s3tests_boto3.functional.test_s3.test_bucket_list_encoding_basic
+s3tests_boto3.functional.test_s3.test_bucket_list_delimiter_none
+s3tests_boto3.functional.test_s3.test_bucket_listv2_delimiter_none
+s3tests_boto3.functional.test_s3.test_bucket_list_delimiter_not_skip_special
+s3tests_boto3.functional.test_s3.test_bucket_list_prefix_basic
+s3tests_boto3.functional.test_s3.test_bucket_list_prefix_delimiter_basic
+s3tests_boto3.functional.test_s3.test_account_usage
+s3tests_boto3.functional.test_s3.test_head_bucket_usage
+s3tests_boto3.functional.test_s3.test_multi_object_delete_key_limit
+s3tests_boto3.functional.test_s3.test_post_object_invalid_signature
+s3tests_boto3.functional.test_s3.test_post_object_invalid_access_key
+s3tests_boto3.functional.test_s3.test_post_object_missing_policy_condition
+s3tests_boto3.functional.test_s3.test_post_object_request_missing_policy_specified_field
+s3tests_boto3.functional.test_s3.test_post_object_expired_policy
+s3tests_boto3.functional.test_s3.test_post_object_invalid_request_field_value
+s3tests_boto3.functional.test_s3.test_bucket_create_exists
+s3tests_boto3.functional.test_s3.test_bucket_create_exists_nonowner
+s3tests_boto3.functional.test_s3.test_bucket_recreate_overwrite_acl
+s3tests_boto3.functional.test_s3.test_bucket_recreate_new_acl
+s3tests_boto3.functional.test_s3.test_list_multipart_upload_owner
+s3tests_boto3.functional.test_s3.test_lifecycle_set
+s3tests_boto3.functional.test_s3.test_lifecycle_expiration
+s3tests_boto3.functional.test_s3.test_lifecycle_expiration_versioning_enabled
+s3tests_boto3.functional.test_s3.test_lifecycle_expiration_tags2
+s3tests_boto3.functional.test_s3.test_lifecycle_expiration_versioned_tags2
+s3tests_boto3.functional.test_s3.test_lifecycle_expiration_noncur_tags1
+s3tests_boto3.functional.test_s3.test_lifecycle_set_date
+s3tests_boto3.functional.test_s3.test_lifecycle_expiration_header_put
+s3tests_boto3.functional.test_s3.test_lifecycle_expiration_header_head
+s3tests_boto3.functional.test_s3.test_lifecycle_expiration_header_tags_head
+s3tests_boto3.functional.test_s3.test_put_obj_enc_conflict_c_s3
+s3tests_boto3.functional.test_s3.test_put_obj_enc_conflict_c_kms
+s3tests_boto3.functional.test_s3.test_put_obj_enc_conflict_s3_kms
+s3tests_boto3.functional.test_s3.test_put_obj_enc_conflict_bad_enc_kms
+s3tests_boto3.functional.test_s3.test_bucket_policy_put_obj_s3_noenc
+s3tests_boto3.functional.test_s3.test_bucket_policy_put_obj_s3_kms
+s3tests_boto3.functional.test_s3.test_bucket_policy_put_obj_kms_noenc
+s3tests_boto3.functional.test_s3.test_bucket_policy_put_obj_kms_s3
+s3tests_boto3.functional.test_s3.test_object_lock_multi_delete_object_with_retention
+s3tests_boto3.functional.test_s3.test_object_lock_put_legal_hold
+s3tests_boto3.functional.test_s3.test_object_lock_get_legal_hold
+s3tests_boto3.functional.test_s3.test_object_lock_changing_mode_from_governance_with_bypass
+s3tests_boto3.functional.test_s3.test_object_lock_changing_mode_from_governance_without_bypass
+s3tests_boto3.functional.test_s3.test_object_lock_changing_mode_from_compliance
+s3tests_boto3.functional.test_s3.test_object_read_unreadable
+s3tests_boto3.functional.test_s3.test_user_policy
+s3tests_boto3.functional.test_s3.test_get_public_acl_bucket_policy_status
+s3tests_boto3.functional.test_s3.test_get_authpublic_acl_bucket_policy_status
+s3tests_boto3.functional.test_s3.test_get_publicpolicy_acl_bucket_policy_status
+s3tests_boto3.functional.test_s3.test_get_nonpublicpolicy_acl_bucket_policy_status
+s3tests_boto3.functional.test_s3.test_get_nonpublicpolicy_deny_bucket_policy_status
+s3tests_boto3.functional.test_s3.test_get_default_public_block
+s3tests_boto3.functional.test_s3.test_put_public_block
+s3tests_boto3.functional.test_s3.test_block_public_put_bucket_acls
+s3tests_boto3.functional.test_s3.test_block_public_object_canned_acls
+s3tests_boto3.functional.test_s3.test_block_public_policy
+s3tests_boto3.functional.test_s3.test_ignore_public_acls
+s3tests_boto3.functional.test_s3.test_sse_s3_default_post_object_authenticated_request
+s3tests_boto3.functional.test_s3.test_sse_kms_default_post_object_authenticated_request
+s3tests_boto3.functional.test_s3select.test_generate_where_clause
+s3tests_boto3.functional.test_s3select.test_generate_projection
+s3tests_boto3.functional.test_s3select.test_count_operation
+s3tests_boto3.functional.test_s3select.test_column_sum_min_max
+s3tests_boto3.functional.test_s3select.test_nullif_expressions
+s3tests_boto3.functional.test_s3select.test_nulliftrue_expressions
+s3tests_boto3.functional.test_s3select.test_is_not_null_expressions
+s3tests_boto3.functional.test_s3select.test_lowerupper_expressions
+s3tests_boto3.functional.test_s3select.test_in_expressions
+s3tests_boto3.functional.test_s3select.test_true_false_in_expressions
+s3tests_boto3.functional.test_s3select.test_like_expressions
+s3tests_boto3.functional.test_s3select.test_truefalselike_expressions
+s3tests_boto3.functional.test_s3select.test_complex_expressions
+s3tests_boto3.functional.test_s3select.test_alias
+s3tests_boto3.functional.test_s3select.test_alias_cyclic_refernce
+s3tests_boto3.functional.test_s3select.test_datetime
+s3tests_boto3.functional.test_s3select.test_true_false_datetime
+s3tests_boto3.functional.test_s3select.test_csv_parser
+s3tests_boto3.functional.test_s3select.test_csv_definition
+s3tests_boto3.functional.test_s3select.test_schema_definition
+s3tests_boto3.functional.test_s3select.test_when_then_else_expressions
+s3tests_boto3.functional.test_s3select.test_coalesce_expressions
+s3tests_boto3.functional.test_s3select.test_cast_expressions
+s3tests_boto3.functional.test_s3select.test_version
+s3tests_boto3.functional.test_s3select.test_trim_expressions
+s3tests_boto3.functional.test_s3select.test_truefalse_trim_expressions
+s3tests_boto3.functional.test_s3select.test_escape_expressions
+s3tests_boto3.functional.test_s3select.test_case_value_expressions
+s3tests_boto3.functional.test_s3select.test_bool_cast_expressions
+s3tests_boto3.functional.test_s3select.test_progress_expressions
+s3tests_boto3.functional.test_s3select.test_output_serial_expressions

src/test/system_tests/ceph_s3_tests/test_ceph_s3.js

@@ -0,0 +1,144 @@
+/* Copyright (C) 2016 NooBaa */
+"use strict";
+
+/**
+ * This script is used as a part of the CI/CD process to run all Ceph S3 tests.
+ * It uses the file setup_ceph_s3_config as prior configuration.
+ * In the past this script was a part of the CI/CD process using run_test_job.sh flow.
+ */
+
+const fs = require('fs');
+const _ = require('lodash');
+const P = require('../../../util/promise');
+const os_utils = require('../../../util/os_utils');
+const dbg = require('../../../util/debug_module')(__filename);
+dbg.set_process_name('test_ceph_s3');
+const argv = require('minimist')(process.argv.slice(2));
+delete argv._;
+const { S3_CEPH_TEST_STEMS, S3_CEPH_TEST_SIGV4, CEPH_TEST, DEFAULT_NUMBER_OF_WORKERS } = require('./test_ceph_s3_constants.js');
+
+let testing_status = {
+    pass: [],
+    fail: [],
+    skip: [],
+    total: 0
+};
+
+let tests_list;
+
+const OUT_OF_SCOPE_TESTS = create_out_of_scope_tests_list() || [];
+//Regexp match will be tested per each entry
+const S3_CEPH_TEST_OUT_OF_SCOPE_REGEXP = new RegExp(`(${OUT_OF_SCOPE_TESTS.join(')|(')})`);
+const S3_CEPH_TEST_STEMS_REGEXP = new RegExp(`(${S3_CEPH_TEST_STEMS.join(')|(')})`);
+
+async function main() {
+    if (argv.help) usage();
+    try {
+        await run_s3_tests();
+    } catch (err) {
+        console.error(`Ceph Test Failed: ${err}`);
+        process.exit(1);
+    }
+    process.exit(0);
+}
+
+function usage() {
+    console.log(`
+Usage:
+--ignore_lists      <path to list>,<path to list>,...
+                    path to list is a text file with the tests in a format:
+                    <directory name>...<directory name>.<file name without extension>.<test name>
+                    for example:
+                    s3tests_boto3.functional.test_s3.test_bucket_listv2_maxkeys_zero
+--concurrency       <integer>
+                    set the number of workers to run the tests. 
+`);
+    process.exit(0);
+}
+
+async function run_s3_tests() {
+    try {
+        await run_all_tests();
+    } catch (err) {
+        console.error('Failed running ceph tests', err);
+        throw new Error('Running Ceph Tests Failed');
+    }
+
+    console.info(`CEPH TEST SUMMARY: Suite contains ${testing_status.total}, ran ${testing_status.pass.length + testing_status.fail.length + testing_status.skip.length} tests, Passed: ${testing_status.pass.length}, Skipped: ${testing_status.skip.length}, Failed: ${testing_status.fail.length}`);
+    if (testing_status.skip.length) {
+        console.warn(`CEPH TEST SUMMARY:  ${testing_status.skip.length} skipped tests ${testing_status.skip.join('\n')}`);
+    }
+    if (testing_status.fail.length) {
+        console.error(`CEPH TEST FAILED TESTS SUMMARY: ${testing_status.fail.length} failed tests \n${testing_status.fail.join('\n')}`);
+        throw new Error('Ceph Tests Returned with Failures');
+    }
+}
+
+async function run_all_tests() {
+    console.info('Running Ceph S3 Tests...');
+    const tests_list_command =
+        `S3TEST_CONF=${CEPH_TEST.test_dir}${CEPH_TEST.ceph_config}  ./${CEPH_TEST.test_dir}${CEPH_TEST.s3_test_dir}virtualenv/bin/nosetests -v --collect-only  2>&1 | awk '{print $1}' | grep test`;
+    try {
+        tests_list = await os_utils.exec(tests_list_command, { ignore_rc: false, return_stdout: true });
+    } catch (err) {
+        console.error('Failed getting tests list');
+        throw new Error(`Failed getting tests list ${err}`);
+    }
+    tests_list = tests_list.split('\n');
+    testing_status.total = tests_list.length;
+    await P.map(_.times(argv.concurrency || DEFAULT_NUMBER_OF_WORKERS), test_worker); //number of concurrent workers is set in the argument of times
+    console.log('Finished Running Ceph S3 Tests');
+}
+
+async function test_worker() {
+    for (;;) {
+        const test = tests_list.shift();
+        if (!test) return;
+        await run_single_test(test);
+    }
+}
+
+async function run_single_test(test) {
+    let ceph_args = `S3TEST_CONF=${CEPH_TEST.test_dir}${CEPH_TEST.ceph_config}`;
+    if (S3_CEPH_TEST_SIGV4.includes(test)) {
+        ceph_args += ` S3_USE_SIGV4=true`;
+    }
+    let base_cmd = `${ceph_args} ./${CEPH_TEST.test_dir}${CEPH_TEST.s3_test_dir}virtualenv/bin/nosetests`;
+    if (!S3_CEPH_TEST_OUT_OF_SCOPE_REGEXP.test(test)) {
+        try {
+            const test_name = test.replace(S3_CEPH_TEST_STEMS_REGEXP, pref => `${pref.slice(0, -1)}:`); //Match against the common test path
+            if (test_name.includes('boto')) {
+                base_cmd = `${ceph_args} ./${CEPH_TEST.test_dir}${CEPH_TEST.s3_test_dir}virtualenv/bin/nosetests -v -s -A 'not fails_on_rgw'`;
+            }
+            const res = await os_utils.exec(`${base_cmd} ${test_name}`, { ignore_rc: false, return_stdout: true });
+            if (res.indexOf('SKIP') >= 0) {
+                console.warn('Test skipped:', test);
+                testing_status.skip.push(test);
+            } else {
+                console.info('Test Passed:', test);
+                testing_status.pass.push(test);
+            }
+        } catch (err) {
+            console.error('Test Failed:', test);
+            testing_status.fail.push(test);
+        }
+    }
+}
+
+function create_out_of_scope_tests_list() {
+    const ignore_lists_paths = argv.ignore_lists?.split(',');
+    if (ignore_lists_paths && ignore_lists_paths.length >= 1) {
+        let out_of_scope_tests_list = [];
+        for (const ignore_lists_path of ignore_lists_paths) {
+            const list_content = fs.readFileSync(ignore_lists_path).toString().trim();
+            if (list_content === "") continue; // in case someone added accidentally an empty list we ignore it
+            const ignore_list = list_content.split("\n");
+            out_of_scope_tests_list = _.concat(out_of_scope_tests_list, ignore_list);
+        }
+        return out_of_scope_tests_list;
+    }
+}
+
+if (require.main === module) {
+    main();
+}

src/test/system_tests/ceph_s3_tests/test_ceph_s3_config_and_run_s3_tests.sh

@@ -0,0 +1,17 @@
+# This script objective is to run Ceph s3 tests
+# We use it inside of K8S job, see noobaa-tests-s3-job
+# since we use "|| true" this job will always end with "Completed" status
+
+#!/bin/bash
+set -x
+
+CEPH_S3_DIRECTORY="src/test/system_tests/ceph_s3_tests"
+CEPH_S3_TESTS_CONFIG="${CEPH_S3_DIRECTORY}/test_ceph_s3_config_setup.js"
+CEPH_S3_RUN_TESTS="${CEPH_S3_DIRECTORY}/test_ceph_s3.js"
+S3_CEPH_TEST_BLACKLIST="${CEPH_S3_DIRECTORY}/s3-tests-lists/s3_tests_black_list.txt"
+S3_CEPH_TEST_PENDING_LIST="${CEPH_S3_DIRECTORY}/s3-tests-lists/s3_tests_pending_list.txt"
+NUMBER_OF_WORKERS=1
+
+cd /root/node_modules/noobaa-core/
+node ./${CEPH_S3_TESTS_CONFIG}
+node ./${CEPH_S3_RUN_TESTS} --ignore_lists ${S3_CEPH_TEST_BLACKLIST},${S3_CEPH_TEST_PENDING_LIST} --concurrency ${NUMBER_OF_WORKERS} || true