From f457e9bf74420d8eb8897fcb2ec14ecdac97b449 Mon Sep 17 00:00:00 2001 From: Justin Smith Date: Fri, 4 Oct 2024 11:21:03 -0400 Subject: [PATCH 1/4] aws-lc-sys: log clang version used w/ bindgen --- aws-lc-sys/Cargo.toml | 4 +- aws-lc-sys/builder/main.rs | 162 +++++++++--------- .../builder/{bindgen.rs => sys_bindgen.rs} | 0 3 files changed, 83 insertions(+), 83 deletions(-) rename aws-lc-sys/builder/{bindgen.rs => sys_bindgen.rs} (100%) diff --git a/aws-lc-sys/Cargo.toml b/aws-lc-sys/Cargo.toml index f04a2f4d031..a88f1dc6202 100644 --- a/aws-lc-sys/Cargo.toml +++ b/aws-lc-sys/Cargo.toml @@ -60,10 +60,10 @@ fs_extra = "1.3" cc = { version = "1.0.100", features = ["parallel"] } [target.'cfg(any(all(any(target_arch="x86_64",target_arch="aarch64"),any(target_os="linux",target_os="macos",target_os="windows"),any(target_env="gnu",target_env="musl",target_env="msvc",target_env="")),all(target_arch="x86",target_os="windows",target_env="msvc"),all(target_arch="x86",target_os="linux",target_env="gnu")))'.build-dependencies] -bindgen = { version = "0.69.2", optional = true } +bindgen = { version = "0.69.5", optional = true } [target.'cfg(not(any(all(any(target_arch="x86_64",target_arch="aarch64"),any(target_os="linux",target_os="macos",target_os="windows"),any(target_env="gnu",target_env="musl",target_env="msvc",target_env="")),all(target_arch="x86",target_os="windows",target_env="msvc"),all(target_arch="x86",target_os="linux",target_env="gnu"))))'.build-dependencies] -bindgen = { version = "0.69.2" } +bindgen = { version = "0.69.5" } [dependencies] libc = "0.2.121" diff --git a/aws-lc-sys/builder/main.rs b/aws-lc-sys/builder/main.rs index 214dac5feb9..5679878ab08 100644 --- a/aws-lc-sys/builder/main.rs +++ b/aws-lc-sys/builder/main.rs @@ -15,24 +15,36 @@ use std::{env, fmt, fmt::Debug}; use cc_builder::CcBuilder; use cmake_builder::CmakeBuilder; -#[cfg(any( - feature = "bindgen", - not(any( - all( - any(target_arch = "x86_64", target_arch = "aarch64"), - any(target_os = "linux", target_os = "macos", target_os = "windows"), - any( - target_env = "gnu", - target_env = "musl", - target_env = "msvc", - target_env = "" - ) - ), - all(target_arch = "x86", target_os = "windows", target_env = "msvc"), - all(target_arch = "x86", target_os = "linux", target_env = "gnu") - )) -))] -mod bindgen; +macro_rules! bindgen_available { + ($top:ident, $item:item) => { + #[allow(clippy::non_minimal_cfg)] + #[cfg($top(any( + feature = "bindgen", + not(any( + all( + any(target_arch = "x86_64", target_arch = "aarch64"), + any(target_os = "linux", target_os = "macos", target_os = "windows"), + any( + target_env = "gnu", + target_env = "musl", + target_env = "msvc", + target_env = "" + ) + ), + all(target_arch = "x86", target_os = "windows", target_env = "msvc"), + all(target_arch = "x86", target_os = "linux", target_env = "gnu") + )) + )))] + $item + }; + ($item:item) => { + bindgen_available!(any, $item); + }; +} + +bindgen_available!( + mod sys_bindgen; +); mod cc_builder; mod cmake_builder; @@ -186,40 +198,25 @@ fn execute_command(executable: &OsStr, args: &[&OsStr]) -> TestCommandResult { } } -#[cfg(any( - feature = "bindgen", - not(any( - all( - any(target_arch = "x86_64", target_arch = "aarch64"), - any(target_os = "linux", target_os = "macos", target_os = "windows"), - any( - target_env = "gnu", - target_env = "musl", - target_env = "msvc", - target_env = "" - ) - ), - all(target_arch = "x86", target_os = "windows", target_env = "msvc"), - all(target_arch = "x86", target_os = "linux", target_env = "gnu") - )) -))] -fn generate_bindings(manifest_dir: &Path, prefix: &Option, bindings_path: &PathBuf) { - let options = BindingOptions { - build_prefix: prefix.clone(), - include_ssl: cfg!(feature = "ssl"), - disable_prelude: true, - }; +bindgen_available!( + fn generate_bindings(manifest_dir: &Path, prefix: &Option, bindings_path: &PathBuf) { + let options = BindingOptions { + build_prefix: prefix.clone(), + include_ssl: cfg!(feature = "ssl"), + disable_prelude: true, + }; - let bindings = bindgen::generate_bindings(manifest_dir, &options); + let bindings = sys_bindgen::generate_bindings(manifest_dir, &options); - bindings - .write(Box::new(std::fs::File::create(bindings_path).unwrap())) - .expect("written bindings"); -} + bindings + .write(Box::new(std::fs::File::create(bindings_path).unwrap())) + .expect("written bindings"); + } +); #[cfg(feature = "bindgen")] fn generate_src_bindings(manifest_dir: &Path, prefix: &Option, src_bindings_path: &Path) { - bindgen::generate_bindings( + sys_bindgen::generate_bindings( manifest_dir, &BindingOptions { build_prefix: prefix.clone(), @@ -407,12 +404,13 @@ fn is_bindgen_required() -> bool { || !has_pregenerated() } -#[allow(dead_code)] -fn internal_bindgen_supported() -> bool { - // TODO: internal bindgen creates invalid bindings on FreeBSD - // See: https://github.com/aws/aws-lc-rs/issues/476 - target_os() != "freebsd" -} +bindgen_available!( + fn internal_bindgen_supported() -> bool { + let cv = bindgen::clang_version(); + emit_warning(&format!("Clang version: {}", cv.full)); + true + } +); fn is_no_prefix() -> bool { unsafe { AWS_LC_SYS_NO_PREFIX } @@ -486,6 +484,30 @@ fn is_crt_static() -> bool { features.contains("crt-static") } +bindgen_available!( + fn handle_bindgen(manifest_dir: &PathBuf, prefix: &Option) -> bool { + if internal_bindgen_supported() && !is_external_bindgen() { + emit_warning(&format!( + "Generating bindings - internal bindgen. Platform: {}", + target() + )); + let gen_bindings_path = out_dir().join("bindings.rs"); + generate_bindings(manifest_dir, prefix, &gen_bindings_path); + emit_rustc_cfg("use_bindgen_generated"); + true + } else { + false + } + } +); + +bindgen_available!( + not, + fn handle_bindgen(_manifest_dir: &PathBuf, _prefix: &Option) -> bool { + false + } +); + fn main() { initialize(); prepare_cargo_cfg(); @@ -510,39 +532,16 @@ fn main() { if is_pregenerating_bindings() { #[cfg(feature = "bindgen")] { - emit_warning(&format!("Generating src bindings. Platform: {}", target())); - let src_bindings_path = Path::new(&manifest_dir).join("src"); - generate_src_bindings(&manifest_dir, &prefix, &src_bindings_path); - bindings_available = true; - } - } else if is_bindgen_required() { - #[cfg(any( - feature = "bindgen", - not(any( - all( - any(target_arch = "x86_64", target_arch = "aarch64"), - any(target_os = "linux", target_os = "macos", target_os = "windows"), - any( - target_env = "gnu", - target_env = "musl", - target_env = "msvc", - target_env = "" - ) - ), - all(target_arch = "x86", target_os = "windows", target_env = "msvc"), - all(target_arch = "x86", target_os = "linux", target_env = "gnu") - )) - ))] - if internal_bindgen_supported() && !is_external_bindgen() { emit_warning(&format!( - "Generating bindings - internal bindgen. Platform: {}", + "Generating src bindings. Platform: '{}' Prefix: '{prefix:?}'", target() )); - let gen_bindings_path = out_dir().join("bindings.rs"); - generate_bindings(&manifest_dir, &prefix, &gen_bindings_path); - emit_rustc_cfg("use_bindgen_generated"); + let src_bindings_path = Path::new(&manifest_dir).join("src"); + generate_src_bindings(&manifest_dir, &prefix, &src_bindings_path); bindings_available = true; } + } else if is_bindgen_required() { + bindings_available = handle_bindgen(&manifest_dir, &prefix); } else { bindings_available = true; } @@ -688,6 +687,7 @@ fn invoke_external_bindgen( verify_bindgen()?; let options = BindingOptions { + // We collect the symbols w/o the prefix added build_prefix: None, include_ssl: false, disable_prelude: true, diff --git a/aws-lc-sys/builder/bindgen.rs b/aws-lc-sys/builder/sys_bindgen.rs similarity index 100% rename from aws-lc-sys/builder/bindgen.rs rename to aws-lc-sys/builder/sys_bindgen.rs From 06ceb840fed1255fd91345620dff7688ddfab095 Mon Sep 17 00:00:00 2001 From: Justin Smith Date: Fri, 4 Oct 2024 13:22:18 -0400 Subject: [PATCH 2/4] aws-lc-fips-sys: log clang version used w/ bindgen --- aws-lc-fips-sys/Cargo.toml | 4 +- aws-lc-fips-sys/builder/cmake_builder.rs | 53 +++++--- aws-lc-fips-sys/builder/main.rs | 122 ++++++++++-------- .../builder/{bindgen.rs => sys_bindgen.rs} | 0 4 files changed, 105 insertions(+), 74 deletions(-) rename aws-lc-fips-sys/builder/{bindgen.rs => sys_bindgen.rs} (100%) diff --git a/aws-lc-fips-sys/Cargo.toml b/aws-lc-fips-sys/Cargo.toml index b08f4a6d04a..abf1827ec8a 100644 --- a/aws-lc-fips-sys/Cargo.toml +++ b/aws-lc-fips-sys/Cargo.toml @@ -66,10 +66,10 @@ fs_extra = "1.3" cc = "1.0.100" [target.'cfg(all(any(target_arch = "x86_64", target_arch = "aarch64"), any(target_os = "linux", target_os = "macos"), any(target_env = "gnu", target_env = "musl", target_env = "")))'.build-dependencies] -bindgen = { version = "0.69.2", optional = true } +bindgen = { version = "0.69.5", optional = true } [target.'cfg(not(all(any(target_arch = "x86_64", target_arch = "aarch64"), any(target_os = "linux", target_os = "macos"), any(target_env = "gnu", target_env = "musl", target_env = ""))))'.build-dependencies] -bindgen = { version = "0.69.2" } +bindgen = { version = "0.69.5" } [dependencies] libc = "0.2.121" diff --git a/aws-lc-fips-sys/builder/cmake_builder.rs b/aws-lc-fips-sys/builder/cmake_builder.rs index 0b4d0f7cdef..e4cda6dbb55 100644 --- a/aws-lc-fips-sys/builder/cmake_builder.rs +++ b/aws-lc-fips-sys/builder/cmake_builder.rs @@ -8,7 +8,7 @@ use crate::{ }; use std::collections::HashMap; use std::env; -use std::ffi::OsStr; +use std::ffi::OsString; use std::path::PathBuf; pub(crate) struct CmakeBuilder { @@ -39,11 +39,21 @@ fn test_nasm_command() -> bool { execute_command("nasm".as_ref(), &["-version".as_ref()]).status } -fn find_cmake_command() -> Option<&'static OsStr> { - if execute_command("cmake3".as_ref(), &["--version".as_ref()]).status { - Some("cmake3".as_ref()) +fn find_cmake_command() -> Option { + if let Some(cmake) = option_env("CMAKE") { + emit_warning(&format!( + "CMAKE environment variable set: {}", + cmake.clone() + )); + if execute_command(cmake.as_ref(), &["--version".as_ref()]).status { + Some(cmake.into()) + } else { + None + } + } else if execute_command("cmake3".as_ref(), &["--version".as_ref()]).status { + Some("cmake3".into()) } else if execute_command("cmake".as_ref(), &["--version".as_ref()]).status { - Some("cmake".as_ref()) + Some("cmake".into()) } else { None } @@ -87,18 +97,10 @@ impl CmakeBuilder { } else { cmake_cfg.define("BUILD_SHARED_LIBS", "0"); } - let opt_level = cargo_env("OPT_LEVEL"); - - if is_no_asm() { - if opt_level == "0" { - cmake_cfg.define("OPENSSL_NO_ASM", "1"); - } else { - panic!("AWS_LC_FIPS_SYS_NO_ASM only allowed for debug builds!") - } - } - if opt_level != "0" { - if opt_level == "1" || opt_level == "2" { + let opt_level = cargo_env("OPT_LEVEL"); + if opt_level.ne("0") { + if opt_level.eq("1") || opt_level.eq("2") { cmake_cfg.define("CMAKE_BUILD_TYPE", "relwithdebinfo"); } else { cmake_cfg.define("CMAKE_BUILD_TYPE", "release"); @@ -153,6 +155,15 @@ impl CmakeBuilder { } cmake_cfg.define("FIPS", "1"); + if is_no_asm() { + let opt_level = cargo_env("OPT_LEVEL"); + if opt_level == "0" { + cmake_cfg.define("OPENSSL_NO_ASM", "1"); + } else { + panic!("AWS_LC_SYS_NO_ASM only allowed for debug builds!") + } + } + if cfg!(feature = "asan") { env::set_var("CC", "clang"); env::set_var("CXX", "clang++"); @@ -162,9 +173,13 @@ impl CmakeBuilder { } // Allow environment to specify CMake toolchain. - if option_env("CMAKE_TOOLCHAIN_FILE").is_some() - || option_env(format!("CMAKE_TOOLCHAIN_FILE_{}", target_underscored())).is_some() - { + if let Some(toolchain) = option_env("CMAKE_TOOLCHAIN_FILE").or(option_env(format!( + "CMAKE_TOOLCHAIN_FILE_{}", + target_underscored() + ))) { + emit_warning(&format!( + "CMAKE_TOOLCHAIN_FILE environment variable set: {toolchain}" + )); return cmake_cfg; } diff --git a/aws-lc-fips-sys/builder/main.rs b/aws-lc-fips-sys/builder/main.rs index 319b5ad663a..07d5174338f 100644 --- a/aws-lc-fips-sys/builder/main.rs +++ b/aws-lc-fips-sys/builder/main.rs @@ -16,15 +16,27 @@ use std::process::Command; use cmake_builder::CmakeBuilder; -#[cfg(any( - feature = "bindgen", - not(all( - any(target_arch = "x86_64", target_arch = "aarch64"), - any(target_os = "linux", target_os = "macos"), - any(target_env = "gnu", target_env = "musl", target_env = "") - )) -))] -mod bindgen; +macro_rules! bindgen_available { + ($top:ident, $item:item) => { + #[allow(clippy::non_minimal_cfg)] + #[cfg($top(any( + feature = "bindgen", + not(all( + any(target_arch = "x86_64", target_arch = "aarch64"), + any(target_os = "linux", target_os = "macos"), + any(target_env = "gnu", target_env = "musl", target_env = "") + )) + )))] + $item + }; + ($item:item) => { + bindgen_available!(any, $item); + }; +} + +bindgen_available!( + mod sys_bindgen; +); mod cmake_builder; pub(crate) fn get_aws_lc_include_path(manifest_dir: &Path) -> PathBuf { @@ -184,31 +196,25 @@ fn execute_command(executable: &OsStr, args: &[&OsStr]) -> TestCommandResult { } } -#[cfg(any( - feature = "bindgen", - not(all( - any(target_arch = "x86_64", target_arch = "aarch64"), - any(target_os = "linux", target_os = "macos"), - any(target_env = "gnu", target_env = "musl", target_env = "") - )) -))] -fn generate_bindings(manifest_dir: &Path, prefix: &Option, bindings_path: &PathBuf) { - let options = BindingOptions { - build_prefix: prefix.clone(), - include_ssl: cfg!(feature = "ssl"), - disable_prelude: true, - }; +bindgen_available!( + fn generate_bindings(manifest_dir: &Path, prefix: &Option, bindings_path: &PathBuf) { + let options = BindingOptions { + build_prefix: prefix.clone(), + include_ssl: cfg!(feature = "ssl"), + disable_prelude: true, + }; - let bindings = bindgen::generate_bindings(manifest_dir, &options); + let bindings = sys_bindgen::generate_bindings(manifest_dir, &options); - bindings - .write(Box::new(std::fs::File::create(bindings_path).unwrap())) - .expect("written bindings"); -} + bindings + .write(Box::new(std::fs::File::create(bindings_path).unwrap())) + .expect("written bindings"); + } +); #[cfg(feature = "bindgen")] fn generate_src_bindings(manifest_dir: &Path, prefix: &Option, src_bindings_path: &Path) { - bindgen::generate_bindings( + sys_bindgen::generate_bindings( manifest_dir, &BindingOptions { build_prefix: prefix.clone(), @@ -327,12 +333,13 @@ fn is_bindgen_required() -> bool { || !has_pregenerated() } -#[allow(dead_code)] -fn internal_bindgen_supported() -> bool { - // TODO: internal bindgen creates invalid bindings on FreeBSD - // See: https://github.com/aws/aws-lc-rs/issues/476 - target_os() != "freebsd" -} +bindgen_available!( + fn internal_bindgen_supported() -> bool { + let cv = bindgen::clang_version(); + emit_warning(&format!("Clang version: {}", cv.full)); + true + } +); fn is_no_prefix() -> bool { unsafe { AWS_LC_FIPS_SYS_NO_PREFIX } @@ -372,6 +379,30 @@ fn prepare_cargo_cfg() { } } +bindgen_available!( + fn handle_bindgen(manifest_dir: &PathBuf, prefix: &Option) -> bool { + if internal_bindgen_supported() && !is_external_bindgen() { + emit_warning(&format!( + "Generating bindings - internal bindgen. Platform: {}", + target() + )); + let gen_bindings_path = out_dir().join("bindings.rs"); + generate_bindings(manifest_dir, prefix, &gen_bindings_path); + emit_rustc_cfg("use_bindgen_generated"); + true + } else { + false + } + } +); + +bindgen_available!( + not, + fn handle_bindgen(_manifest_dir: &PathBuf, _prefix: &Option) -> bool { + false + } +); + fn main() { initialize(); prepare_cargo_cfg(); @@ -386,6 +417,8 @@ fn main() { }; let builder = get_builder(&prefix, &manifest_dir, &out_dir()); + emit_warning("Building with: CMake"); + emit_warning(&format!("Symbol Prefix: {:?}", &prefix)); builder.check_dependencies().unwrap(); @@ -403,24 +436,7 @@ fn main() { bindings_available = true; } } else if is_bindgen_required() { - #[cfg(any( - feature = "bindgen", - not(all( - any(target_arch = "x86_64", target_arch = "aarch64"), - any(target_os = "linux", target_os = "macos"), - any(target_env = "gnu", target_env = "musl", target_env = "") - )) - ))] - if internal_bindgen_supported() && !is_external_bindgen() { - emit_warning(&format!( - "Generating bindings - internal bindgen. Platform: {}", - target() - )); - let gen_bindings_path = out_dir().join("bindings.rs"); - generate_bindings(&manifest_dir, &prefix, &gen_bindings_path); - emit_rustc_cfg("use_bindgen_generated"); - bindings_available = true; - } + bindings_available = handle_bindgen(&manifest_dir, &prefix); } else { bindings_available = true; } diff --git a/aws-lc-fips-sys/builder/bindgen.rs b/aws-lc-fips-sys/builder/sys_bindgen.rs similarity index 100% rename from aws-lc-fips-sys/builder/bindgen.rs rename to aws-lc-fips-sys/builder/sys_bindgen.rs From 16981620ec5ad97b034adff64f211e69792aebdc Mon Sep 17 00:00:00 2001 From: Justin Smith Date: Fri, 4 Oct 2024 14:00:19 -0400 Subject: [PATCH 3/4] Add testing --- .github/workflows/tests.yml | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 6ae77d643b9..27e226842ac 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -602,3 +602,33 @@ jobs: working-directory: ./aws-lc-rs # Rust doc tests dislike dynamic linking run: cargo careful test --tests --features fips,bindgen + + clang-19-bindgen: + if: github.repository_owner == 'aws' + name: Clang 19.1 + bindgen tests + runs-on: macos-14-xlarge + env: + LIBCLANG_PATH: /opt/homebrew/opt/llvm/lib + LLVM_CONFIG_PATH: /opt/homebrew/opt/llvm/llvm-config + steps: + - uses: actions/checkout@v4 + with: + submodules: 'recursive' + - uses: actions/setup-go@v4 + with: + go-version: '>=1.18' + - uses: dtolnay/rust-toolchain@master + id: toolchain + with: + toolchain: stable + - run: | + brew update + brew uninstall --force llvm + brew install llvm@19 + echo 'export PATH="/opt/homebrew/opt/llvm/bin:$PATH"' >> "$GITHUB_ENV" + - name: aws-lc-sys bindgen build + working-directory: ./aws-lc-sys + run: cargo test --features bindgen + - name: aws-lc-fips-sys bindgen build + working-directory: ./aws-lc-fips-sys + run: cargo test --features bindgen From 099f3acdc9a9920b3a04bd2c46a7d7abcb302917 Mon Sep 17 00:00:00 2001 From: Justin Smith Date: Tue, 8 Oct 2024 14:13:21 -0400 Subject: [PATCH 4/4] bindgen-cli not needed on FreeBSD --- .github/workflows/cross.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cross.yml b/.github/workflows/cross.yml index 3638048390c..c6cd7d112b1 100644 --- a/.github/workflows/cross.yml +++ b/.github/workflows/cross.yml @@ -264,7 +264,7 @@ jobs: usesh: true copyback: false prepare: | - pkg install -y git gmake bash sudo cmake-core llvm-devel-lite curl rust-bindgen-cli go + pkg install -y git gmake bash sudo cmake-core llvm-devel-lite curl go run: | curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y . "$HOME/.cargo/env"