From 430b607476d4ec2f5daf79b258a66412becca03d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerardo=20Ravago=20=F0=9F=87=B5=F0=9F=87=AD?= Date: Thu, 19 Oct 2023 10:09:55 -0400 Subject: [PATCH] Initial Import of Cryptofuzz from AWS-LC This directly imports the related CI source for Cryptofuzz from AWS-LC so that the ACCP specific changes show up better in PR and the source history. --- tests/ci/cdk/cdk/accp_github_fuzz_ci_stack.py | 131 ++++++++++ tests/ci/common_fuzz.sh | 144 +++++++++++ tests/ci/common_posix_setup.sh | 244 ++++++++++++++++++ .../dependencies/build_cryptofuzz_modules.sh | 56 ++++ .../dependencies/cryptofuzz_data.zip | Bin 0 -> 17176 bytes .../amazonlinux-2_clang-11x/Dockerfile | 19 ++ .../Dockerfile | 26 ++ tests/ci/run_cryptofuzz.sh | 78 ++++++ 8 files changed, 698 insertions(+) create mode 100644 tests/ci/cdk/cdk/accp_github_fuzz_ci_stack.py create mode 100644 tests/ci/common_fuzz.sh create mode 100644 tests/ci/common_posix_setup.sh create mode 100755 tests/ci/docker_images/dependencies/build_cryptofuzz_modules.sh create mode 100644 tests/ci/docker_images/dependencies/cryptofuzz_data.zip create mode 100644 tests/ci/docker_images/linux-x86/amazonlinux-2_clang-11x/Dockerfile create mode 100644 tests/ci/docker_images/linux-x86/amazonlinux-2_clang-11x_cryptofuzz/Dockerfile create mode 100755 tests/ci/run_cryptofuzz.sh diff --git a/tests/ci/cdk/cdk/accp_github_fuzz_ci_stack.py b/tests/ci/cdk/cdk/accp_github_fuzz_ci_stack.py new file mode 100644 index 00000000..d99cc453 --- /dev/null +++ b/tests/ci/cdk/cdk/accp_github_fuzz_ci_stack.py @@ -0,0 +1,131 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 OR ISC + +from aws_cdk import Duration, Size, Stack, aws_codebuild as codebuild, aws_iam as iam, aws_ec2 as ec2, aws_efs as efs +from constructs import Construct + +from cdk.components import PruneStaleGitHubBuilds +from util.ecr_util import ecr_arn +from util.iam_policies import code_build_batch_policy_in_json, \ + code_build_publish_metrics_in_json +from util.metadata import AWS_ACCOUNT, AWS_REGION, GITHUB_PUSH_CI_BRANCH_TARGETS, GITHUB_REPO_OWNER, GITHUB_REPO_NAME +from util.build_spec_loader import BuildSpecLoader + + +class AwsLcGitHubFuzzCIStack(Stack): + """Define a stack used to batch execute AWS-LC tests in GitHub.""" + + def __init__(self, + scope: Construct, + id: str, + spec_file_path: str, + **kwargs) -> None: + super().__init__(scope, id, **kwargs) + + # Define CodeBuild resource. + git_hub_source = codebuild.Source.git_hub( + owner=GITHUB_REPO_OWNER, + repo=GITHUB_REPO_NAME, + webhook=True, + webhook_filters=[ + codebuild.FilterGroup.in_event_of( + codebuild.EventAction.PULL_REQUEST_CREATED, + codebuild.EventAction.PULL_REQUEST_UPDATED, + codebuild.EventAction.PULL_REQUEST_REOPENED), + codebuild.FilterGroup.in_event_of(codebuild.EventAction.PUSH).and_branch_is( + GITHUB_PUSH_CI_BRANCH_TARGETS), + ], + webhook_triggers_batch_build=True) + + # Define a IAM role for this stack. + code_build_batch_policy = iam.PolicyDocument.from_json( + code_build_batch_policy_in_json([id]) + ) + fuzz_policy = iam.PolicyDocument.from_json(code_build_publish_metrics_in_json()) + inline_policies = {"code_build_batch_policy": code_build_batch_policy, + "fuzz_policy": fuzz_policy} + role = iam.Role(scope=self, + id="{}-role".format(id), + assumed_by=iam.ServicePrincipal("codebuild.amazonaws.com"), + inline_policies=inline_policies) + + # Create the VPC for EFS and CodeBuild + public_subnet = ec2.SubnetConfiguration(name="PublicFuzzingSubnet", subnet_type=ec2.SubnetType.PUBLIC) + private_subnet = ec2.SubnetConfiguration(name="PrivateFuzzingSubnet", subnet_type=ec2.SubnetType.PRIVATE_WITH_EGRESS) + + # Create a VPC with a single public and private subnet in a single AZ. This is to avoid the elastic IP limit + # being used up by a bunch of idle NAT gateways + fuzz_vpc = ec2.Vpc( + scope=self, + id="{}-FuzzingVPC".format(id), + subnet_configuration=[public_subnet, private_subnet], + max_azs=1 + ) + build_security_group = ec2.SecurityGroup( + scope=self, + id="{}-FuzzingSecurityGroup".format(id), + vpc=fuzz_vpc + ) + + build_security_group.add_ingress_rule( + peer=build_security_group, + connection=ec2.Port.all_traffic(), + description="Allow all traffic inside security group" + ) + + efs_subnet_selection = ec2.SubnetSelection(subnet_type=ec2.SubnetType.PRIVATE_WITH_EGRESS) + + # Create the EFS to store the corpus and logs. EFS allows new filesystems to burst to 100 MB/s for the first 2 + # TB of data read/written, after that the rate is limited based on the size of the filesystem. As of late + # 2021 our corpus is less than one GB which results in EFS limiting all reads and writes to the minimum 1 MB/s. + # To have the fuzzing be able to finish in a reasonable amount of time use the Provisioned capacity option. + # For now this uses 100 MB/s which matches the performance used for 2021. Looking at EFS metrics in late 2021 + # during fuzz runs EFS sees 4-22 MB/s of transfers thus 100 MB/s gives lots of buffer and allows ~4-5 fuzz runs + # to start at the same time with no issue. + # https://docs.aws.amazon.com/efs/latest/ug/performance.html + fuzz_filesystem = efs.FileSystem( + scope=self, + id="{}-FuzzingEFS".format(id), + file_system_name="AWS-LC-Fuzz-Corpus", + enable_automatic_backups=True, + encrypted=True, + security_group=build_security_group, + vpc=fuzz_vpc, + vpc_subnets=efs_subnet_selection, + performance_mode=efs.PerformanceMode.GENERAL_PURPOSE, + throughput_mode=efs.ThroughputMode.PROVISIONED, + provisioned_throughput_per_second=Size.mebibytes(100), + ) + + # Define CodeBuild. + fuzz_codebuild = codebuild.Project( + scope=self, + id="FuzzingCodeBuild", + project_name=id, + source=git_hub_source, + role=role, + timeout=Duration.minutes(120), + environment=codebuild.BuildEnvironment(compute_type=codebuild.ComputeType.LARGE, + privileged=True, + build_image=codebuild.LinuxBuildImage.STANDARD_4_0), + build_spec=BuildSpecLoader.load(spec_file_path), + vpc=fuzz_vpc, + security_groups=[build_security_group]) + fuzz_codebuild.enable_batch_builds() + + # CDK raw overrides: https://docs.aws.amazon.com/cdk/latest/guide/cfn_layer.html#cfn_layer_raw + # https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-codebuild-project.html#aws-resource-codebuild-project-properties + # The EFS identifier needs to match tests/ci/common_fuzz.sh, CodeBuild defines an environment variable named + # codebuild_$identifier. + # https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-codebuild-project-projectfilesystemlocation.html + # + # TODO: add this to the CDK project above when it supports EfsFileSystemLocation + cfn_codebuild = fuzz_codebuild.node.default_child + cfn_codebuild.add_override("Properties.FileSystemLocations", [{ + "Identifier": "fuzzing_root", + "Location": "%s.efs.%s.amazonaws.com:/" % (fuzz_filesystem.file_system_id, AWS_REGION), + "MountPoint": "/efs_fuzzing_root", + "Type": "EFS" + }]) + + PruneStaleGitHubBuilds(scope=self, id="PruneStaleGitHubBuilds", project=fuzz_codebuild) diff --git a/tests/ci/common_fuzz.sh b/tests/ci/common_fuzz.sh new file mode 100644 index 00000000..3adb2c12 --- /dev/null +++ b/tests/ci/common_fuzz.sh @@ -0,0 +1,144 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 OR ISC + +source tests/ci/common_posix_setup.sh + +if [ -v CODEBUILD_FUZZING_ROOT ]; then + CORPUS_ROOT="${CODEBUILD_FUZZING_ROOT}/fuzzing" +else + CORPUS_ROOT="${BUILD_ROOT}/mock_efs/fuzzing" +fi +echo "$CORPUS_ROOT" + +if [ -v CODEBUILD_BUILD_ID ]; then + BUILD_ID=$CODEBUILD_BUILD_ID +else + # Generate a random string in bash https://unix.stackexchange.com/questions/230673/how-to-generate-a-random-string + BUILD_ID=$(tr -dc A-Za-z0-9 &1 | tee "$SUMMARY_LOG" + # This gets the status of the fuzz run which determines if we want to fail the build or not, otherwise we'd get the results of tee + if [ "${PIPESTATUS[0]}" == 1 ]; then + FUZZ_RUN_FAILURE=1 + fi + + # The libfuzzer logs are written to the current working directory and need to be moved after the test is done + mv ./*.log "${LOCAL_FUZZ_RUN_LOGS}/." + + if [ "$FUZZ_RUN_FAILURE" == 1 ]; then + FUZZ_TEST_FAILURE_ROOT="${SHARED_FAILURE_ROOT}/${FUZZ_NAME}" + mkdir -p "$FUZZ_TEST_FAILURE_ROOT" + + if [[ "$FUZZ_NAME" == "cryptofuzz" ]]; then + for ARTIFACT in "$LOCAL_ARTIFACTS_FOLDER"/*; do + base64 $ARTIFACT + ARTIFACT_NAME=$(basename "$ARTIFACT") + "${FUZZ_TEST_PATH}" --debug "$ARTIFACT" | tee "${LOCAL_FUZZ_RUN_LOGS}/${ARTIFACT_NAME}.log" + done + fi + + cp -r "$LOCAL_FUZZ_TEST_ROOT" "$SHARED_FAILURE_ROOT" + cp "$FUZZ_TEST_PATH" "${FUZZ_TEST_FAILURE_ROOT}/${FUZZ_NAME}" + + # If this fuzz run has failed the below metrics won't make a lot of sense, it could fail on the first input and + # publish a TestCount of 1 which makes all the metrics look weird + echo "${FUZZ_NAME} failed, see the above output for details. For all the logs see ${SHARED_FAILURE_ROOT} in EFS" + exit 1 + else + echo "Fuzz test ${FUZZ_NAME} finished successfully, not copying run logs and run corpus" + fi + + set -e + + # Step 2 merge any new files from the run corpus and GitHub src corpus into the shared corpus, the first folder is + # where to merge the new corpus (SHARED_FUZZ_TEST_CORPUS), the second two are where to read new inputs from + # (LOCAL_RUN_CORPUS and SRC_CORPUS). + time "${FUZZ_TEST_PATH}" -merge=1 "$SHARED_FUZZ_TEST_CORPUS" "$LOCAL_RUN_CORPUS" "$SRC_CORPUS" + + # Calculate interesting metrics and post results to CloudWatch, this checks the shared (EFS) corpus after the new test + # run corpus has been merged in + FINAL_SHARED_CORPUS_FILE_COUNT=$(find "$SHARED_FUZZ_TEST_CORPUS" -type f | wc -l) + put_metric_count --metric-name SharedCorpusFileCount --value "$FINAL_SHARED_CORPUS_FILE_COUNT" --dimensions "FuzzTest=$FUZZ_NAME" + + RUN_CORPUS_FILE_COUNT=$(find "$LOCAL_RUN_CORPUS" -type f | wc -l) + put_metric_count --metric-name RunCorpusFileCount --value "$RUN_CORPUS_FILE_COUNT" --dimensions "FuzzTest=$FUZZ_NAME,Platform=$PLATFORM" + + TEST_COUNT=$(grep -o "stat::number_of_executed_units: [0-9]*" "$SUMMARY_LOG" | awk '{test_count += $2} END {print test_count}') + put_metric_count --metric-name TestCount --value "$TEST_COUNT" --dimensions "FuzzTest=$FUZZ_NAME,Platform=$PLATFORM" + + TESTS_PER_SECOND=$((TEST_COUNT/TIME_FOR_EACH_FUZZ)) + put_metric --metric-name TestRate --value "$TESTS_PER_SECOND" --unit Count/Second --dimensions "FuzzTest=$FUZZ_NAME,Platform=$PLATFORM" + + FEATURE_COVERAGE=$(grep -o "ft: [0-9]*" "$SUMMARY_LOG" | awk '{print $2}' | sort -n | tail -1) + put_metric_count --metric-name FeatureCoverage --value "$FEATURE_COVERAGE" --dimensions "FuzzTest=$FUZZ_NAME,Platform=$PLATFORM" + + BLOCK_COVERAGE=$(grep -o "cov: [0-9]*" "$SUMMARY_LOG" | awk '{print $2}' | sort -n | tail -1) + put_metric_count --metric-name BlockCoverage --value "$BLOCK_COVERAGE" --dimensions "FuzzTest=$FUZZ_NAME,Platform=$PLATFORM" + + echo "${FUZZ_NAME} starting shared ${ORIGINAL_CORPUS_FILE_COUNT} final shared ${FINAL_SHARED_CORPUS_FILE_COUNT} new files ${RUN_CORPUS_FILE_COUNT} total test count ${TEST_COUNT} test rate ${TESTS_PER_SECOND} code coverage ${BLOCK_COVERAGE} feature coverage ${FEATURE_COVERAGE}" +} diff --git a/tests/ci/common_posix_setup.sh b/tests/ci/common_posix_setup.sh new file mode 100644 index 00000000..57605037 --- /dev/null +++ b/tests/ci/common_posix_setup.sh @@ -0,0 +1,244 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 OR ISC + +if [ -v CODEBUILD_SRC_DIR ]; then + SRC_ROOT="$CODEBUILD_SRC_DIR" +else + SRC_ROOT=$(pwd) +fi +echo "$SRC_ROOT" + +cd ../ +SYS_ROOT=$(pwd) +cd $SRC_ROOT + +BUILD_ROOT="${SRC_ROOT}/test_build_dir" +echo "$BUILD_ROOT" + +PLATFORM=$(uname -m) + +NUM_CPU_THREADS='' +KERNEL_NAME=$(uname -s) +if [[ "${KERNEL_NAME}" == "Darwin" ]]; then + # On MacOS, /proc/cpuinfo does not exist. + NUM_CPU_THREADS=$(sysctl -n hw.ncpu) +else + # Assume KERNEL_NAME is Linux. + NUM_CPU_THREADS=$(grep -c ^processor /proc/cpuinfo) + if [[ $PLATFORM == "aarch64" ]]; then + CPU_PART=$(grep -Po -m 1 'CPU part.*:\s\K.*' /proc/cpuinfo) + NUM_CPU_PART=$(grep -c $CPU_PART /proc/cpuinfo) + # Set capabilities via the static flags for valgrind tests. + # This is because valgrind reports the instruction + # mrs %0, MIDR_EL1 + # which fetches the CPU part number, as illegal. + # For some reason, valgrind also reports SHA512 instructions illegal, + # so the SHA512 capability is not included below. + VALGRIND_STATIC_CAP_FLAGS="-DOPENSSL_STATIC_ARMCAP -DOPENSSL_STATIC_ARMCAP_NEON" + VALGRIND_STATIC_CAP_FLAGS+=" -DOPENSSL_STATIC_ARMCAP_AES -DOPENSSL_STATIC_ARMCAP_PMULL " + VALGRIND_STATIC_CAP_FLAGS+=" -DOPENSSL_STATIC_ARMCAP_SHA1 -DOPENSSL_STATIC_ARMCAP_SHA256 " + if [[ $NUM_CPU_PART == $NUM_CPU_THREADS ]] && [[ ${CPU_PART} =~ 0x[dD]40 ]]; then + VALGRIND_STATIC_CAP_FLAGS+=" -DOPENSSL_STATIC_ARMCAP_SHA3 -DOPENSSL_STATIC_ARMCAP_NEOVERSE_V1" + fi + fi +fi + +# Pick cmake3 if possible. We don't know of any OS that installs a cmake3 +# executable that is not at least version 3.0. +if [[ -x "$(command -v cmake3)" ]] ; then + CMAKE_COMMAND="cmake3" +else + CMAKE_COMMAND="cmake" +fi + +function run_build { + local cflags=("$@") + rm -rf "$BUILD_ROOT" + mkdir -p "$BUILD_ROOT" + cd "$BUILD_ROOT" || exit 1 + + if [[ "${AWSLC_32BIT}" == "1" ]]; then + cflags+=("-DCMAKE_TOOLCHAIN_FILE=${SRC_ROOT}/util/32-bit-toolchain.cmake") + fi + + if [[ -x "$(command -v ninja)" ]]; then + echo "Using Ninja build system (ninja)." + BUILD_COMMAND="ninja" + cflags+=(-GNinja) + elif [[ -x "$(command -v ninja-build)" ]]; then + echo "Using Ninja build system (ninja-build)." + BUILD_COMMAND="ninja-build" + cflags+=(-GNinja) + else + echo "Using Make." + BUILD_COMMAND="make -j${NUM_CPU_THREADS}" + fi + + ${CMAKE_COMMAND} "${cflags[@]}" "$SRC_ROOT" + $BUILD_COMMAND + cd "$SRC_ROOT" +} + +function run_cmake_custom_target { + $BUILD_COMMAND -C "$BUILD_ROOT" "$@" +} + +function build_and_test { + run_build "$@" + run_cmake_custom_target 'run_tests' +} + +function generate_symbols_file { + # read_symbols.go currently only support static libraries + if [ ! -f "$BUILD_ROOT"/crypto/libcrypto.a ]; then + echo "Static library not found: ${BUILD_ROOT}/crypto/libcrypto.a" + exit 1 + fi + + go run "$SRC_ROOT"/util/read_symbols.go -out "$BUILD_ROOT"/symbols_crypto.txt "$BUILD_ROOT"/crypto/libcrypto.a + go run "$SRC_ROOT"/util/read_symbols.go -out "$BUILD_ROOT"/symbols_ssl.txt "$BUILD_ROOT"/ssl/libssl.a + + # The $BUILD_ROOT gets deleted on each run. symbols.txt must be placed elsewhere. + cat "$BUILD_ROOT"/symbols_crypto.txt "$BUILD_ROOT"/symbols_ssl.txt > "$SRC_ROOT"/symbols.txt +} + + +function verify_symbols_prefixed { + go run "$SRC_ROOT"/util/read_symbols.go -out "$BUILD_ROOT"/symbols_final_crypto.txt "$BUILD_ROOT"/crypto/libcrypto.a + go run "$SRC_ROOT"/util/read_symbols.go -out "$BUILD_ROOT"/symbols_final_ssl.txt "$BUILD_ROOT"/ssl/libssl.a + # For grep's basic regular expression language the meta-characters (e.g. "?", + # "|", etc.) are interpreted as literal characters. To keep their + # meta-character semantics, they must be escaped with "\". + # Deciphering the pattern "^_\?\(bignum\|curve25519_x25519\)": + # * "^": anchor at start of line. + # * "_\?": might contain underscore. + # * "\(bignum\|curve25519_x25519\)": match string of either "bignum" or "curve25519_x25519". + # Recall that the option "-v" reverse the pattern matching. So, we are really + # filtering out lines that contain either "bignum" or "curve25519_x25519". + cat "$BUILD_ROOT"/symbols_final_crypto.txt "$BUILD_ROOT"/symbols_final_ssl.txt | grep -v -e '^_\?\(bignum\|curve25519_x25519\)' > "$SRC_ROOT"/symbols_final.txt + # Now filter out every line that has the unique prefix $CUSTOM_PREFIX. If we + # have any lines left, then some symbol(s) weren't prefixed, unexpectedly. + if [ $(grep -c -v ${CUSTOM_PREFIX} "$SRC_ROOT"/symbols_final.txt) -ne 0 ]; then + echo "Symbol(s) missing prefix!" + exit 1 + fi +} + + +function build_prefix_and_test { + CUSTOM_PREFIX=aws_lc_1_1_0 + run_build "$@" + generate_symbols_file + run_build "$@" "-DBORINGSSL_PREFIX=${CUSTOM_PREFIX}" "-DBORINGSSL_PREFIX_SYMBOLS=${SRC_ROOT}/symbols.txt" + verify_symbols_prefixed + run_cmake_custom_target 'run_tests' +} + +function fips_build_and_test { + run_build "$@" -DFIPS=1 + # Upon completion of the build process. The module’s status can be verified by 'tool/bssl isfips'. + # https://csrc.nist.gov/CSRC/media/projects/cryptographic-module-validation-program/documents/security-policies/140sp3678.pdf + # FIPS mode is enabled when 'defined(BORINGSSL_FIPS) && !defined(OPENSSL_ASAN)'. + # https://github.com/aws/aws-lc/blob/220e266d4e415cf0101388b89a2bd855e0e4e203/crypto/fipsmodule/is_fips.c#L22 + expect_fips_mode=1 + for build_flag in "$@" + do + if [[ "${build_flag}" == '-DASAN=1' ]]; then + expect_fips_mode=0 + break + fi + done + module_status=$("${BUILD_ROOT}/tool/bssl" isfips) + [[ "${expect_fips_mode}" == "${module_status}" ]] || { echo >&2 "FIPS Mode validation failed."; exit 1; } + # Run tests. + run_cmake_custom_target 'run_tests' + "${BUILD_ROOT}/util/fipstools/test_fips" +} + +function build_and_test_valgrind { + if [[ $PLATFORM == "aarch64" ]]; then + run_build "$@" -DCMAKE_C_FLAGS="$VALGRIND_STATIC_CAP_FLAGS" + run_cmake_custom_target 'run_tests_valgrind' + + # Disable all capabilities and run again + # (We don't use the env. variable OPENSSL_armcap because it is currently + # restricted to the case of runtime discovery of capabilities + # in cpu_aarch64_linux.c) + run_build "$@" -DCMAKE_C_FLAGS="-DOPENSSL_STATIC_ARMCAP" + run_cmake_custom_target 'run_tests_valgrind' + else + run_build "$@" + run_cmake_custom_target 'run_tests_valgrind' + fi +} + +function build_and_test_ssl_runner_valgrind { + export AWS_LC_GO_TEST_TIMEOUT="60m" + + if [[ $PLATFORM == "aarch64" ]]; then + run_build "$@" -DCMAKE_C_FLAGS="$VALGRIND_STATIC_CAP_FLAGS" + else + run_build "$@" + fi + run_cmake_custom_target 'run_ssl_runner_tests_valgrind' +} + +function build_and_test_with_sde { + run_build "$@" + run_cmake_custom_target 'run_tests_with_sde' +} + +function build_and_run_minimal_test { + run_build "$@" + run_cmake_custom_target 'run_minimal_tests' +} + +# Install local build of AWS-LC for integration testing. +function aws_lc_build() { + AWS_LC_DIR=${1} + BUILD_FOLDER=${2} + INSTALL_FOLDER=${3} + + echo "Building AWS-LC to ${BUILD_FOLDER} and installing to ${INSTALL_FOLDER} with CFlags "${@:4}"" + ${CMAKE_COMMAND} ${AWS_LC_DIR} -GNinja "-B${BUILD_FOLDER}" "-DCMAKE_INSTALL_PREFIX=${INSTALL_FOLDER}" "${@:4}" + ninja -C ${BUILD_FOLDER} install + ls -R ${INSTALL_FOLDER} + rm -rf ${BUILD_FOLDER:?}/* +} + +function print_executable_information { + EXE_NAME=${1} + EXE_ARGUMENT=${2} + LABEL=${3} + + echo "" + echo "${LABEL}:" + if command -v ${EXE_NAME} &> /dev/null + then + ${EXE_NAME} ${EXE_ARGUMENT} + else + echo "${EXE_NAME} not found" + fi +} + +print_executable_information "cmake" "--version" "CMake version" +print_executable_information "cmake3" "--version" "CMake version (cmake3 executable)" +print_executable_information "go" "version" "Go version" +print_executable_information "perl" "--version" "Perl version" +# Ninja executable names are not uniform over operating systems +print_executable_information "ninja-build" "--version" "Ninja version (ninja-build executable)" +print_executable_information "ninja" "--version" "Ninja version (ninja executable)" +print_executable_information "gcc" "--version" "gcc version" +print_executable_information "g++" "--version" "g++ version" +print_executable_information "clang" "--version" "clang version" +print_executable_information "clang++" "--version" "clang++ version" +print_executable_information "cc" "--version" "cc version" +print_executable_information "c++" "--version" "c++ version" +print_executable_information "make" "--version" "Make version" +print_executable_information "rustup" "show" "Rust toolchain" +echo "" +echo "Operating system information:" +uname -a +echo "" +echo "Environment variables:" +env diff --git a/tests/ci/docker_images/dependencies/build_cryptofuzz_modules.sh b/tests/ci/docker_images/dependencies/build_cryptofuzz_modules.sh new file mode 100755 index 00000000..802d8adc --- /dev/null +++ b/tests/ci/docker_images/dependencies/build_cryptofuzz_modules.sh @@ -0,0 +1,56 @@ +#!/bin/bash +set -exo pipefail + +function env { + export "$1"="$2" + echo "export ${1}=\"${2}\"" >> "${FUZZ_ROOT}/fuzz_env.sh" +} +# Recommended flags from https://github.com/guidovranken/cryptofuzz/blob/master/docs/building.md +# Remove-fsanitize=undefined which doesn't fail the build but creates additional noise in build output +export CFLAGS="-fsanitize=address,fuzzer-no-link -O2 -g -Wno-gnu-designator" +export CXXFLAGS="-fsanitize=address,fuzzer-no-link -D_GLIBCXX_DEBUG -O2 -g" + +# Setup base of Cryptofuzz +cd "$FUZZ_ROOT" +MODULES_ROOT="${FUZZ_ROOT}/modules" +git clone --depth 1 https://github.com/guidovranken/cryptofuzz.git +cd cryptofuzz +git rev-parse HEAD +CRYPTOFUZZ_SRC=$(pwd) +python3 gen_repository.py + +mkdir "$MODULES_ROOT" +cd "$MODULES_ROOT" + +# Setup the other crypto libraries for differential fuzzing +# Crypto++ https://github.com/guidovranken/cryptofuzz/blob/master/docs/cryptopp.md +cd "$MODULES_ROOT" +git clone --depth 1 https://github.com/weidai11/cryptopp.git +cd cryptopp/ +git rev-parse HEAD +make libcryptopp.a -j$(nproc) +export CXXFLAGS="$CXXFLAGS -DCRYPTOFUZZ_CRYPTOPP" +env LIBCRYPTOPP_A_PATH `realpath libcryptopp.a` +env CRYPTOPP_INCLUDE_PATH `realpath .` +cd "${CRYPTOFUZZ_SRC}/modules/cryptopp/" +make -j$(nproc) + +# Extract the seed corpus, docker layers are already compressed so this won't use any more space and save time when running +cd "$FUZZ_ROOT" +unzip cryptofuzz_data.zip +rm cryptofuzz_data.zip +env CRYPTOFUZZ_SEED_CORPUS `realpath cryptofuzz_seed_corpus` +env CRYPTOFUZZ_DICT `realpath cryptofuzz-dict.txt` + +# Save final common flags +env CFLAGS "$CFLAGS" +env CXXFLAGS "$CXXFLAGS" +env CRYPTOFUZZ_SRC "$CRYPTOFUZZ_SRC" + +# Cryptofuzz builds its modules into $CRYPTOFUZZ_SRC/modules that includes everything it needs, deleting the module source +# code saves a substantial amount of space in the docker image +rm -rf "$MODULES_ROOT" + +# Prebuild the required libcpu_features to save time +cd "$CRYPTOFUZZ_SRC" +make third_party/cpu_features/build/libcpu_features.a diff --git a/tests/ci/docker_images/dependencies/cryptofuzz_data.zip b/tests/ci/docker_images/dependencies/cryptofuzz_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..2fd98997de387f4eaa00f5a2ec90fe042c491af0 GIT binary patch literal 17176 zcmb`u2RN4f|Nk#L#Fd$d%U+dhkFrS_S=l1jl#rcS_D*DEMz-uNgis>kHX|9CnT5=v z|KjgeT%ba1;!U0KuVP5E_REVG$505($ALad0?K(2sq=#RB5 zht&EW(Nd$KO}n%lFF_(Zp!<7n@V{^qKoArXf*>N`5Cj2_LqI_o0tktMzz9%08iIhM zQBWux1N{~5PxJN9oJySTc#~03+L76R%ra_^XfAUTJ#Bs zlCTcY{vP_mKSP5tC@h$W!V{oSFb)MlU{DY+mOvn4!K6XpP(&mKOoV|@uwQ`&A&XY_ zU%wO1c|~@FTkh+7t)39YIpHFjRZfiWi<8M$G&JkK(6^>aIuqU&+$}Qya{xtUSP+ba zc!2r$133TB1At=@ASjWD2H|jM7y%7NAPHzF5rzOkpco_`4?>{OPy&L0`;`IwrIubf zga8Fy(~;+F@(mnYn$o09sSSP2nZfb#9;_f0+fhGLy#nH91=-@Vqqu{5|75gu}};Y4#xgh*vt&M z-W0(G8cz(px}B0h0_x_6d{J0*&iTM%4$%G{`roty!jgOhN+cj41P~63gM%<=G!lx$ zf{3I!03+bQFd_tp1^){4lc_pHl1*7-HXVid1lgntBF}_#{QNbVQchuz?+Z>$)M1?{ zRFw%zlDL1alYg0}P&5LLgQDT2Wd{jIA&>|N1PzA55CkxWfF|Mycra;DNCM)&XUDED zbNO;T8CNGeU{7!BjH`v%=qnq5NJXZhJ^IM?^l1|K&nM`=CfI@ifQ5}%|{p9?YrUi{c zfRIoE7D_U1C=mxmV31%G27@Ej97SpehWm9bdeTR;(yof{@HdFw1LAgx7uIbq!ujgTsU6M+Zgr<>won1 zAzNQBNp}6`zU4q!|4?`=0YOCJAfy>d8V4Rq>KBP5A(5~_1k(J)Vxb@isci@UlCA$W zUH{x;lolv27AYol_L+a((J{#GeD#5j`cF$AZTSX2u$Tj~zpwefEJauX3QFn`v0x|> z1Ba0O1p!6E;6xaah{ck;oTMZK7L5k~QqBJoOVoi_m?y}vg#`XZtbtp|vmZ2`tuUK- z&};pCCHfa^9E3=K!7y-=kkJqb1_~wMAtbZJA~B@?i%8-{;z$ky{-qZ8|Ho#aeMZZV z^4>mDcT?+8lo|4_?6rkg6-Jz3wzL;htZRc}XBaeyI@dw2|fctxm z{L8$=qmT$J4uJu~@gNKyN7_yhF{FJz27@QNcUz<3;( z)Z8LSPccw1IOxBt`TtBx@7^#!Xk6_bpC8u!nyXwY|9eul`2_2b?!ceNBPD|Uslx(j(ZwRwZnaSn)$CjEoY~p>y50>xDRY$#0&ZUK*^!!lmvTR4 z{L5aq?BaSX5jxH3>XqVUw!gXm>|8_ZS1FF{x~~1%{jVO<<1#7Vklhp`S0WQE5Glfc ztnS>}@_Z^S;PMdIvOd1Qv%aeS<6RHy}J+@rlFJw6J^tSBu=1h z*VDrdxK*CM!)fC++aue^+ez=1QYRyDxqD)3y)$FW_hpFNfcD$YFXT?Q_hqFcXLejZ z4<55-lg*ebkr8+pmUG^{wqs&1J~iSjqxnWZI?{f#K}w(?c@5ebuza%q;T$x_o^5)1 zYOaQ>&tukE`Wo8ky&0eBT&%UV%=r;R{UVE)_OLNG83C#BM3LemOhtx*>*ljmRO^)V zAO^oW7B?8=%No}?|An`kGXQ=+sesIwN6rA zCy9%_C9eIQG~N#U0&NoPlePR~a$kX&W?xE-`?Er%3gmAs_ytl+00epsk%(RfeUXr%{poPJhK$b!yDXsK^GR9n z4s)*I{9{wZCVU47DJroUH^1J8y;`46EYGi%jL02;vxj2xx~M%Go%dzWjkP{H7Q_oP zrZ~af;q@sRu$*qIdioG^ldfWgxR z^ZN<{>|w~2BX2_JrQ@F7QkSh zlf~#qb@Umfy?;F4re(A2hMpOq`5NuyXZnOC>%sP~mUlT%YTxdrLR$v9mNm65-P^sU zJ@mr;t}U@AxaxX0eN*%#`yX84y+aKXo+o8#8=BX5=F>+$sry&14z2ShW!Hy_E@`LYUY_f7RfIIl~Y;;TA2mu-@-tvaNvaK=#uO`StKbO!$2G<6V%U``KJ6rU}&h#HopgFFtMYiF{a{ zC~THxS{+z4<8v4hwOXebPTzMl{cI<~h&tBfFhvoV?eKV#Cm!M;&{bzZ+Z~Yn>{^LO zNu&*Ql=JPvs|0CskTXL1%B5*QvHlQ{F}su;XJ}CcDdJ{3>@(en?=ki{Q)~3g6A{Bnl60;k zUFJC8PISPLus@#HO@5vhk<6c9?XWW@ma2RD_sR`uH(C}o*wb1)(C=d84F+Xb76ohF z8zr!DljStRV^ah&;!lx-YkEiM{LXf=+cKWo?3$o*h%nOwez4{8?Pv2d{wn92IuiCG zT8_whf$uChbeZ1w88R&`n9WY(PHJAz1q$s5ZE_*UNo z(aidRhmS(Sf0xp?WmJHY%J;a*xOUYOn`!pif#>Sgwz-xnfLub(ZoB zcf-{I_J`ufTlKRTkIS-iK?^8}NAr0WL>ACS5j_G~Ds;uQ;u}*tfGXVTw+mW^J*JL4 zxm>H-KG$tG>4sapwE(Vcb1-%7y(iruNHSQ)L1UKOc_iAaCi%IyPicUIx5to`Q|;Ev z567STkIrPST3<|G8*MD(Pm#UHUar>^#4&mj5tIn!&==TcJ6riy?~J{-<2J*)m8VwE zgGBfe$hokB4B{_O+;UZpiTJWXETEpUAAU9^9O667RjN;m_bs<7zKIPD77&T&ZG+P# zU5Z~Ho)iWhF^Bh}fKUG%$U=$JAT1uNnew8k?Pw*r8hb*G&!BB+gyb}H-! z+eCx(PEOi=29DM&dDe3HCYNLkal)9R+xA71p#A2Y(X{AEQ8bs(^mMd3ZQ|1Ohlhkj z!<$kjxQD_E?=-x^`t(2ayHU{L)kV=yBgL+Y0~lxq<&HKFs`@Kh`N zl5KlF!FLNb2TnN8ucAh`AA|a0Wn=S>}wy6YlE0^ff-YqQ*>C8fzyo-aDenCshNhQa*F;cZ$VL>^!)* z8p_zQ%o%%;c1Wh-t+81Q6a6>YktJ)~q8=U7Dv`JCtIvcVs9TaAOB`jtw3dsC-+eTC4-Gh8(dT{@I5b8m0tiuL zoPWuDrQak~HRXMk?^-AJ4&bp$NABn3jN)1pAW32?McM1Ba)Kh=?#wZ|sr*I1 zs2Whc1`PLvCz#OC>QA^2}^yYSWyJe?f9P~a9n{UTD>YbA%n z_qY%1XWPzMb>&BLe>lb(Fz}h-g$#vy8eY)(E^1%7$qj*_NyT+C&|rsN>@ z^wiJ%ea~G_cbd2;>A%y`zG$S-Lq8p=LPyzsPGn0)#El-^k}p^LbRk)#f_iGPM5Zeth$H3atm0Ra)=^WCtdYVr)f{+iQ-+|i zk!tkY_oc*w;|i<_mOIiXBjsc`t)ysE3WEnZd9vO;J$l59!r;PurUJW1C3(UruiSXI zx6kq?ecCs{c|(zSO`lwLklrO;qzqgVe;EymL^MQRFILf^pf0QW=$C%`OLD>f&BV0C zPiAKn4MID8oZ2%feF<8q>PEl|v6u8UHcw;HG(M`tRlGeBRuiAwG*pbYD{)o_Zjddj zofO`Uv7B4d1G*S89ns{>((Gqzi#dD$_E$G8osFzdC9PA{GOBDd(QL=?mkU2|koWBS z6Vt<#pL}=GlrMJ}pf@FO?~3l?vNpeB={uE(;;OwrxScsF zH^};#`rm3sgjjSJDd;*iyrx~nn`BQ|c3wzBiB31%-j?3mQ4aUOk7K8Z-r}UTxSxv5VUSRo@VENiN zMbQk*U#63^qtN%}RW}>Y0I8)TAB!(DP^GTRQNKm6;9KaoXKU-T_)|PsdOal5&-^CkoQCPCHxijWW-C4t2e8{ixB&dmQ``2m{(9 zyRp0X-4~R(B31*gw%2t}oexEQWXe?OeN@?UoL)6w9vJm?Z2i9T+d`DfH9ilWbw~Y5 z!H;+ARo34fkCha2_tj&dI|~~?SEj=rmlOv8v&d7W6w~t%w*vGYzpC%wuN6qC?*!+q zr2wyM!0g>h)mduvLWSE)Dkd>Sp$Uy&UGwW_dL%v@0=_J*=9} z{=93}h;6I$I9h;O;ashlUa6c`yR9~i zZ_bC-W{%h7ia2YwHNLE}P8naz_D=FY!|h$=P-2(8{y>mA=-KC?)SI4HSi%%V;r?0R zM{@mnWd^i@6B?X#a^`sdL?83C&CEM`Rq;__L0R=G{%4hWh{B4uMi_G@@4e%he<}{B zj8sKIqVN-T1z;ZE%ySD6=(+gr+- z2N+g4qDPs2{i9?X$`fd+!)Lk{cS#1Dtvm}@?CcYo7^1hW$q=-yUscr)Gc>=PAJ_Z| zG%(t#z*+5-i3!URM96<)y%Zw;fC`etE@{h>7aFg^qptG$@nw)vqm$`<{&VS1Ep#@s zSTI!?!+qS|6#k7@ZOeTVX*o3O{#)Gs>dc?6V{ z&+IDrh)z}O)23Y!+l!@nf4yXOam3kKxV>2Y&ZOrB^m5&YKI5J9qpJ^#Rzw(PT?l8j z9=hd4q6M#=QJRsjj}-)DXNwzdDVT?Le7)CuLB&O)`0NG$Fq-rZPBF!^k9{88o2blR z0<^sNyesJNLc!4KhLPBOvgJAEm*=A*ka+bAMtXgNfm4^daFu+1*Rbp0qHQ6e25kMC zol0jF`TY6W-78$R6xxl0@kRGX0~s!gzKK z5~BLq2+0kh8)n@$J7-`BYs|B_wbiU9YXI~Z0{R*%59@F<;Vxp#oymu*|a#`uC11t-U?>NjquVh|fdZ9uPIV$yXqB z{>zx&hu)6YvtjX0D%&F4mO4F`(qbQJWk5KR6tNLsb`>^fDVmrIgjyba4Z~)pDFCg+ zeHnO;y-^|393M|S$ts?_$jy@GzA$n_0)N>?A-tqL;$a$UBx&5HFT9{AAC~_(^-1Bh ze_6mnzt3qE@XacQyk?vE!lB#x=kC@i^cSBW(uvh(9e$btpqXcW{*_w(2Amkzck`WV z(9sFB0_@7&rQAq9v~E>+hJQ$MySeTBmCIF{X)ZdGLx~y^pBq@yZ#4qmp^F3nqvTId zx0t?uAD>JF-k2O(@|zv)n|9%hb!yEDlGp2(i@9hm;lDn_awQ+RULl}=*1nM2@sf~F zJdfIo7CkX2WmQ)wekA*ON&c27AepXtNSG#q{tDH40RX0TI@b%08m4n{_lT}WeXDWf zb8N1WlanZR_q;eit{wQVV4t$f-GP5x?W)Vp}ws39zY4wH_$1_&V zZ&V3!l7jx>-WB02y54UMF2%iO*XORK0iWo#jmuUzN}S4g%6yE%z;X|cUtg-QidKCY z-x?y9S(0yLkn(*DZ?Rvk|x8R%vQ@xTgxUk)#AG z0?C`6vE(~%(aZ17xv**^=-J4A)-+(^v;FvxPZPnp(d78Ik^nhlWy;81!yMOmc z@;1B6TZxt#?Q`d)?Z;SI286=JT>S{ilCLW+Lnx-siL#js%{Rv|U3^4$Tve%daGTa( z$A_D@O$Oai-zu(% z6|MnRmR5;*N#;~}T+X*#J?Qv+>}qd0JV+2IIyFzPgbr+F7Ew)cXE(loKEP#dWhmyX z(+L)k*rVbmCc&y>$Db|c=XLlE)?U|7mx=GuUA*^*y7^Hpf2AS4QuDiPGygLBz zTzxK8k7+wCV#~yOH{Q3>8<2fdH4VX}p>p5GCYu6lk2}VnuFQHK{mKv+%rrYo!Aa`8 zVa~IOxP|G32Z7;&DJ$03bd|WJJ0)yrd_G+rofnZ9 zIr-`y#z6$MMJgi4%WNQ5LufDZfl~c|_g#a@3`$uBGvJlljUv~fmU4s2Z$T+rjsbGi z{Zesb2yj^|al-vJ?%JC3ZM=($^dLZLG1l6I^FrK-YC1mU7O9NQ4SAKq*cw5DH^Ozo z)MY5Y`zDnxyA=~L0F%n4(?mfZQ6~7VXH~zs^A3B@?)}=ni^Dl)&DY)SUus>BT-eCr ze4mJqFiX5YF<0`^;FuNtaHFHL+KdkF2_nZ+aCJQY_~Vlhg1Rt`w1Htz-cygPv>Wj^ zuLU!_(&x(zOoUxU%ssnB$l=># zV832)`nt+3Lk{J0#meok%3rPQawuthq;U{5DNAgRJH6c9X|b`K9a1b8zw*`~_~xeY z0xP`=*x1GEZQc#VX5P}3qQ=Y$^X_Z*A4d4A65)Z9XsHy0Kz~K74bAbmfyDR zYJWp&Hq`)ug&=g7^@1i-@y+8Qc_s_Ob$|2<(cYzky4SEfWg0(ej6_&A$W5DS7Rw1$ zkJOEdk6-Qna5u+b(rWXYo{{Ob>_mj{=)0xUb$NrFzR8#3f?Q%G_7Z#;>>V5(v=^?v zrk-Z*jM0z}ajLaI*|)EMVGPfYDOSu2(RLb_sUC}ZzAYEXgjap?+2<~AaIZ^7we97~ zhs!Y!ZpMm+Fy<(ysqx<%Q~*kT?CJCA(U+PqS`(xiY?X zg>Qr=UAfh%H>RUq^-X!ej`7?VrRu!@Z zy~u~Jd^McR*yv60*R>?nwkv)N;Zjb{E;_mQ0emrXVnU6lS zZT%xmk6p?8E?|P9C}v=zD7v05`)px_BxOgq$kEKp0Z%E#0d%}m{-WT7JothZL*#I&pIq*s!80>9As?gb61Ycq_R z8BZEP;$UAq5AI)U;*@0DzRw5K>sFpk4^$UFZGe^NyFcF;7-JJol@{Ygi*2 zIQD35Bk8$Mdg7-~0Wg)bap}V`PLnf!Cwf^s#&xi@%%Y3SXM8?DIo z^s$$$L6L7HVlLW#uFdC5>TU$K&@>xOG|NV=QXB`PZARkVx!#O53_GfrZUuaB221*v z%dZDg;yX3_>{R;3&8!WY1A0oMZdRTh7_sfU7CA-HlY~+rCdEV&K1C!7nMF3OMZ|hM%Ny8h1omz3$*}^1ryiqRswXExgQ44?iciNWiskFfZr!K@+}K zgr5!SOmkQ@?VXyZSADxcF)s_C>X^Gm0;}mOn+{ZtXnR#vr+{iG*PXq|ONKD`_@$$c z<4VelKMdMp-LI>vTRrI@4)=7dX`G*2Ka;b_ZoZ}Px$mh_W6VW0t}AZzfj93rFrBwiZVJePP60Ktnt=RdN}x#>fC5n*N1puq;P=V_3NBeX+V8?C8N+7 z;uoj(LaX*r_nx?{{ev5k9aif_Qb{_)2RwBIR{&dO)*F6y$FddmjA$-*Sw&k8<1v*4 z2C(lX?oQEAE3e7%SPo-t@@Yxwg;zNI*h+ZdT=EH}q_Zl(<{qI0xNqrNiZ5I9_Em+3&)axlX#< z?wPv0sb^YRrv2KI*aSIL7#2qDk9zl8CYm)42<<_OunkdLu6*KIS2a{$?RVWudquwi zXS3%)tktSij>wh<%rxv>_et($dt-4kdJU4YlES+9^;YY=Oo2vYR5a9-_MCr*GG0&S zDGwPI?!P0N(35$p>m^FC*CYz{VyvK4H$;Zb$%fIgEb$moePqt}}NxgY+a%yGWlKDyg ziT#E)RY_@$fr~2NMrfk)?^xtLSsSQt$$c-b|5Rg3qiXk+8@TroYryF9haZaQ>p2Bx_W%SR`Wy# zz4Uyk1`{KOp8t%Q9-{z_-M7b`-?qsUQpDDhMmQYZ#U5k|IK(80^Ttwzc(`mv7+$U? z$T!u+;w{p=3YNV#`&oi2}iDj(q){ zVWt2^o;>WXeApCRs3N*>P)+YS*=gsXgrJ4bYEN`S8?3Njdg$MTQ89WL)sOhuB5?^*N>b#ilDNEi-+yF)I>r-md zSsOO$;AGN+)Zxq#kph?AIQ4g z!bv~Ee4pPr40Dd+r!c=Oa`Yq2_hFpFFgrho`CS2`A7Q?a*c^tbq5NrOepedkN0{%U zGlya9sD29byD~mM!hD~FISeC2{Zkn9Z_4ic2=jfu#Vcz{5 z=63}&euVix%y1Ycis`47`CYk-A7Q=^Cme=R0sIu^ccm$Qg!w*oa2SUA_)lSeS32T* sn1i6h_Zzo^pu>NTOrP3AFuxpo&{Cr!^&VtoCrMxXqzOq0Kltze1Fwb)