diff --git a/3rdparty/python/requirements.txt b/3rdparty/python/requirements.txt index ea21b3f58e5..cc894087013 100644 --- a/3rdparty/python/requirements.txt +++ b/3rdparty/python/requirements.txt @@ -35,3 +35,5 @@ six>=1.9.0,<2 subprocess32==3.2.7 ; python_version<'3' wheel==0.31.1 www-authenticate==0.9.2 +toml==0.10.0 + diff --git a/contrib/rust/examples/src/rust/binary/BUILD b/contrib/rust/examples/src/rust/binary/BUILD new file mode 100644 index 00000000000..ddd82901586 --- /dev/null +++ b/contrib/rust/examples/src/rust/binary/BUILD @@ -0,0 +1,8 @@ +# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + + +cargo_binary( + name='binary', + sources=rglobs('*.rs', 'Cargo.toml', exclude=[rglobs('**/target/*.rs')]), +) diff --git a/contrib/rust/examples/src/rust/binary/Cargo.toml b/contrib/rust/examples/src/rust/binary/Cargo.toml new file mode 100644 index 00000000000..0acf7d1aad1 --- /dev/null +++ b/contrib/rust/examples/src/rust/binary/Cargo.toml @@ -0,0 +1,7 @@ +[package] +name = "binary" +version = "0.1.0" +authors = ["Pants Build "] +edition = "2018" + +[dependencies] diff --git a/contrib/rust/examples/src/rust/binary/src/main.rs b/contrib/rust/examples/src/rust/binary/src/main.rs new file mode 100644 index 00000000000..e7a11a969c0 --- /dev/null +++ b/contrib/rust/examples/src/rust/binary/src/main.rs @@ -0,0 +1,3 @@ +fn main() { + println!("Hello, world!"); +} diff --git a/contrib/rust/examples/src/rust/dependencies/bin_c/BUILD b/contrib/rust/examples/src/rust/dependencies/bin_c/BUILD new file mode 100644 index 00000000000..0d579521b6c --- /dev/null +++ b/contrib/rust/examples/src/rust/dependencies/bin_c/BUILD @@ -0,0 +1,11 @@ +# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + + +cargo_binary( + name='bin_c', + sources=rglobs('*.rs', 'Cargo.toml', exclude=[rglobs('**/target/*.rs')]), + dependencies=[ + "contrib/rust/examples/src/rust/dependencies/lib_d:lib_d" + ] +) diff --git a/contrib/rust/examples/src/rust/dependencies/bin_c/Cargo.toml b/contrib/rust/examples/src/rust/dependencies/bin_c/Cargo.toml new file mode 100644 index 00000000000..4e860de7b44 --- /dev/null +++ b/contrib/rust/examples/src/rust/dependencies/bin_c/Cargo.toml @@ -0,0 +1,7 @@ +[package] +name = "bin_c" +version = "0.1.0" +authors = ["Pants Build "] +edition = "2018" + +[dependencies] diff --git a/contrib/rust/examples/src/rust/dependencies/bin_c/src/main.rs b/contrib/rust/examples/src/rust/dependencies/bin_c/src/main.rs new file mode 100644 index 00000000000..e7a11a969c0 --- /dev/null +++ b/contrib/rust/examples/src/rust/dependencies/bin_c/src/main.rs @@ -0,0 +1,3 @@ +fn main() { + println!("Hello, world!"); +} diff --git a/contrib/rust/examples/src/rust/dependencies/lib_d/BUILD b/contrib/rust/examples/src/rust/dependencies/lib_d/BUILD new file mode 100644 index 00000000000..8f9dedcbb90 --- /dev/null +++ b/contrib/rust/examples/src/rust/dependencies/lib_d/BUILD @@ -0,0 +1,12 @@ +# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + + +cargo_library( + name='lib_d', + sources=rglobs('*.rs', 'Cargo.toml', exclude=[rglobs('**/target/*.rs')]), + dependencies=[ + "contrib/rust/examples/src/rust/dependencies/workspace_a:workspace_a", + "contrib/rust/examples/src/rust/dependencies/workspace_b:workspace_b" + ] +) diff --git a/contrib/rust/examples/src/rust/dependencies/lib_d/Cargo.toml b/contrib/rust/examples/src/rust/dependencies/lib_d/Cargo.toml new file mode 100644 index 00000000000..07cc9374712 --- /dev/null +++ b/contrib/rust/examples/src/rust/dependencies/lib_d/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "lib_d" +version = "0.1.0" +authors = ["Pants Build "] +edition = "2018" + +[dependencies] +sha2 = "0.8" diff --git a/contrib/rust/examples/src/rust/dependencies/lib_d/src/lib.rs b/contrib/rust/examples/src/rust/dependencies/lib_d/src/lib.rs new file mode 100644 index 00000000000..31e1bb209f9 --- /dev/null +++ b/contrib/rust/examples/src/rust/dependencies/lib_d/src/lib.rs @@ -0,0 +1,7 @@ +#[cfg(test)] +mod tests { + #[test] + fn it_works() { + assert_eq!(2 + 2, 4); + } +} diff --git a/contrib/rust/examples/src/rust/dependencies/workspace_a/BUILD b/contrib/rust/examples/src/rust/dependencies/workspace_a/BUILD new file mode 100644 index 00000000000..be161938b6e --- /dev/null +++ b/contrib/rust/examples/src/rust/dependencies/workspace_a/BUILD @@ -0,0 +1,8 @@ +# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + + +cargo_workspace( + name='workspace_a', + include=['*.rs', 'Cargo.toml'] +) \ No newline at end of file diff --git a/contrib/rust/examples/src/rust/dependencies/workspace_a/Cargo.toml b/contrib/rust/examples/src/rust/dependencies/workspace_a/Cargo.toml new file mode 100644 index 00000000000..0a82893f10c --- /dev/null +++ b/contrib/rust/examples/src/rust/dependencies/workspace_a/Cargo.toml @@ -0,0 +1,6 @@ +[workspace] +members = [ + "bin_a", + "lib_a", + "lib_b" +] diff --git a/contrib/rust/examples/src/rust/dependencies/workspace_a/bin_a/Cargo.toml b/contrib/rust/examples/src/rust/dependencies/workspace_a/bin_a/Cargo.toml new file mode 100644 index 00000000000..bd0e6c10ace --- /dev/null +++ b/contrib/rust/examples/src/rust/dependencies/workspace_a/bin_a/Cargo.toml @@ -0,0 +1,7 @@ +[package] +name = "bin_a" +version = "0.1.0" +authors = ["Pants Build "] +edition = "2018" + +[dependencies] diff --git a/contrib/rust/examples/src/rust/dependencies/workspace_a/bin_a/src/main.rs b/contrib/rust/examples/src/rust/dependencies/workspace_a/bin_a/src/main.rs new file mode 100644 index 00000000000..e7a11a969c0 --- /dev/null +++ b/contrib/rust/examples/src/rust/dependencies/workspace_a/bin_a/src/main.rs @@ -0,0 +1,3 @@ +fn main() { + println!("Hello, world!"); +} diff --git a/contrib/rust/examples/src/rust/dependencies/workspace_a/lib_a/Cargo.toml b/contrib/rust/examples/src/rust/dependencies/workspace_a/lib_a/Cargo.toml new file mode 100644 index 00000000000..bc84115bf9c --- /dev/null +++ b/contrib/rust/examples/src/rust/dependencies/workspace_a/lib_a/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "lib_a" +version = "0.1.0" +authors = ["Pants Build "] +edition = "2018" + +[dependencies] +sha2 = "0.8" diff --git a/contrib/rust/examples/src/rust/dependencies/workspace_a/lib_a/src/lib.rs b/contrib/rust/examples/src/rust/dependencies/workspace_a/lib_a/src/lib.rs new file mode 100644 index 00000000000..31e1bb209f9 --- /dev/null +++ b/contrib/rust/examples/src/rust/dependencies/workspace_a/lib_a/src/lib.rs @@ -0,0 +1,7 @@ +#[cfg(test)] +mod tests { + #[test] + fn it_works() { + assert_eq!(2 + 2, 4); + } +} diff --git a/contrib/rust/examples/src/rust/dependencies/workspace_a/lib_b/Cargo.toml b/contrib/rust/examples/src/rust/dependencies/workspace_a/lib_b/Cargo.toml new file mode 100644 index 00000000000..af05319432d --- /dev/null +++ b/contrib/rust/examples/src/rust/dependencies/workspace_a/lib_b/Cargo.toml @@ -0,0 +1,7 @@ +[package] +name = "lib_b" +version = "0.1.0" +authors = ["Pants Build "] +edition = "2018" + +[dependencies] diff --git a/contrib/rust/examples/src/rust/dependencies/workspace_a/lib_b/src/lib.rs b/contrib/rust/examples/src/rust/dependencies/workspace_a/lib_b/src/lib.rs new file mode 100644 index 00000000000..31e1bb209f9 --- /dev/null +++ b/contrib/rust/examples/src/rust/dependencies/workspace_a/lib_b/src/lib.rs @@ -0,0 +1,7 @@ +#[cfg(test)] +mod tests { + #[test] + fn it_works() { + assert_eq!(2 + 2, 4); + } +} diff --git a/contrib/rust/examples/src/rust/dependencies/workspace_b/BUILD b/contrib/rust/examples/src/rust/dependencies/workspace_b/BUILD new file mode 100644 index 00000000000..26ee1dc48cf --- /dev/null +++ b/contrib/rust/examples/src/rust/dependencies/workspace_b/BUILD @@ -0,0 +1,8 @@ +# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + + +cargo_workspace( + name='workspace_b', + include=['*.rs', 'Cargo.toml'] +) \ No newline at end of file diff --git a/contrib/rust/examples/src/rust/dependencies/workspace_b/Cargo.toml b/contrib/rust/examples/src/rust/dependencies/workspace_b/Cargo.toml new file mode 100644 index 00000000000..0d2c40e2c21 --- /dev/null +++ b/contrib/rust/examples/src/rust/dependencies/workspace_b/Cargo.toml @@ -0,0 +1,5 @@ +[workspace] +members = [ + "bin_b", + "lib_c" +] diff --git a/contrib/rust/examples/src/rust/dependencies/workspace_b/bin_b/Cargo.toml b/contrib/rust/examples/src/rust/dependencies/workspace_b/bin_b/Cargo.toml new file mode 100644 index 00000000000..e39e5db5e60 --- /dev/null +++ b/contrib/rust/examples/src/rust/dependencies/workspace_b/bin_b/Cargo.toml @@ -0,0 +1,7 @@ +[package] +name = "bin_b" +version = "0.1.0" +authors = ["Pants Build "] +edition = "2018" + +[dependencies] diff --git a/contrib/rust/examples/src/rust/dependencies/workspace_b/bin_b/src/main.rs b/contrib/rust/examples/src/rust/dependencies/workspace_b/bin_b/src/main.rs new file mode 100644 index 00000000000..e7a11a969c0 --- /dev/null +++ b/contrib/rust/examples/src/rust/dependencies/workspace_b/bin_b/src/main.rs @@ -0,0 +1,3 @@ +fn main() { + println!("Hello, world!"); +} diff --git a/contrib/rust/examples/src/rust/dependencies/workspace_b/lib_c/Cargo.toml b/contrib/rust/examples/src/rust/dependencies/workspace_b/lib_c/Cargo.toml new file mode 100644 index 00000000000..45e39fa7b7f --- /dev/null +++ b/contrib/rust/examples/src/rust/dependencies/workspace_b/lib_c/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "lib_c" +version = "0.1.0" +authors = ["Pants Build "] +edition = "2018" + +[dependencies] +sha2 = "0.8" diff --git a/contrib/rust/examples/src/rust/dependencies/workspace_b/lib_c/src/lib.rs b/contrib/rust/examples/src/rust/dependencies/workspace_b/lib_c/src/lib.rs new file mode 100644 index 00000000000..31e1bb209f9 --- /dev/null +++ b/contrib/rust/examples/src/rust/dependencies/workspace_b/lib_c/src/lib.rs @@ -0,0 +1,7 @@ +#[cfg(test)] +mod tests { + #[test] + fn it_works() { + assert_eq!(2 + 2, 4); + } +} diff --git a/contrib/rust/examples/src/rust/lib_kinds/cdylib/BUILD b/contrib/rust/examples/src/rust/lib_kinds/cdylib/BUILD new file mode 100644 index 00000000000..8dadcdf8ba6 --- /dev/null +++ b/contrib/rust/examples/src/rust/lib_kinds/cdylib/BUILD @@ -0,0 +1,8 @@ +# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + + +cargo_library( + name='cdylib', + sources=rglobs('*.rs', 'Cargo.toml', exclude=[rglobs('**/target/*.rs')]), +) diff --git a/contrib/rust/examples/src/rust/lib_kinds/cdylib/Cargo.toml b/contrib/rust/examples/src/rust/lib_kinds/cdylib/Cargo.toml new file mode 100644 index 00000000000..5e40e6b177f --- /dev/null +++ b/contrib/rust/examples/src/rust/lib_kinds/cdylib/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "cdylib" +version = "0.1.0" +authors = ["Pants Build "] +edition = "2018" + +[lib] +crate-type = ["cdylib"] \ No newline at end of file diff --git a/contrib/rust/examples/src/rust/lib_kinds/cdylib/src/lib.rs b/contrib/rust/examples/src/rust/lib_kinds/cdylib/src/lib.rs new file mode 100644 index 00000000000..31e1bb209f9 --- /dev/null +++ b/contrib/rust/examples/src/rust/lib_kinds/cdylib/src/lib.rs @@ -0,0 +1,7 @@ +#[cfg(test)] +mod tests { + #[test] + fn it_works() { + assert_eq!(2 + 2, 4); + } +} diff --git a/contrib/rust/examples/src/rust/lib_kinds/dylib/BUILD b/contrib/rust/examples/src/rust/lib_kinds/dylib/BUILD new file mode 100644 index 00000000000..6a7d1853fe2 --- /dev/null +++ b/contrib/rust/examples/src/rust/lib_kinds/dylib/BUILD @@ -0,0 +1,8 @@ +# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + + +cargo_library( + name='dylib', + sources=rglobs('*.rs', 'Cargo.toml', exclude=[rglobs('**/target/*.rs')]), +) diff --git a/contrib/rust/examples/src/rust/lib_kinds/dylib/Cargo.toml b/contrib/rust/examples/src/rust/lib_kinds/dylib/Cargo.toml new file mode 100644 index 00000000000..44cd05cd6b1 --- /dev/null +++ b/contrib/rust/examples/src/rust/lib_kinds/dylib/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "dylib" +version = "0.1.0" +authors = ["Pants Build "] +edition = "2018" + +[lib] +crate-type = ["dylib"] + +[dependencies] +sha2 = "0.8" \ No newline at end of file diff --git a/contrib/rust/examples/src/rust/lib_kinds/dylib/src/lib.rs b/contrib/rust/examples/src/rust/lib_kinds/dylib/src/lib.rs new file mode 100644 index 00000000000..31e1bb209f9 --- /dev/null +++ b/contrib/rust/examples/src/rust/lib_kinds/dylib/src/lib.rs @@ -0,0 +1,7 @@ +#[cfg(test)] +mod tests { + #[test] + fn it_works() { + assert_eq!(2 + 2, 4); + } +} diff --git a/contrib/rust/examples/src/rust/lib_kinds/proc-macro/BUILD b/contrib/rust/examples/src/rust/lib_kinds/proc-macro/BUILD new file mode 100644 index 00000000000..7adc340e132 --- /dev/null +++ b/contrib/rust/examples/src/rust/lib_kinds/proc-macro/BUILD @@ -0,0 +1,8 @@ +# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + + +cargo_library( + name='proc-macro', + sources=rglobs('*.rs', 'Cargo.toml', exclude=[rglobs('**/target/*.rs')]), +) diff --git a/contrib/rust/examples/src/rust/lib_kinds/proc-macro/Cargo.toml b/contrib/rust/examples/src/rust/lib_kinds/proc-macro/Cargo.toml new file mode 100644 index 00000000000..141e5c082f4 --- /dev/null +++ b/contrib/rust/examples/src/rust/lib_kinds/proc-macro/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "proc-macro" +version = "0.1.0" +authors = ["Pants Build "] +edition = "2018" + +[lib] +proc-macro = true diff --git a/contrib/rust/examples/src/rust/lib_kinds/proc-macro/src/lib.rs b/contrib/rust/examples/src/rust/lib_kinds/proc-macro/src/lib.rs new file mode 100644 index 00000000000..31e1bb209f9 --- /dev/null +++ b/contrib/rust/examples/src/rust/lib_kinds/proc-macro/src/lib.rs @@ -0,0 +1,7 @@ +#[cfg(test)] +mod tests { + #[test] + fn it_works() { + assert_eq!(2 + 2, 4); + } +} diff --git a/contrib/rust/examples/src/rust/lib_kinds/rlib/BUILD b/contrib/rust/examples/src/rust/lib_kinds/rlib/BUILD new file mode 100644 index 00000000000..9d805ca6d9b --- /dev/null +++ b/contrib/rust/examples/src/rust/lib_kinds/rlib/BUILD @@ -0,0 +1,8 @@ +# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + + +cargo_library( + name='rlib', + sources=rglobs('*.rs', 'Cargo.toml', exclude=[rglobs('**/target/*.rs')]), +) diff --git a/contrib/rust/examples/src/rust/lib_kinds/rlib/Cargo.toml b/contrib/rust/examples/src/rust/lib_kinds/rlib/Cargo.toml new file mode 100644 index 00000000000..154fdd42a0a --- /dev/null +++ b/contrib/rust/examples/src/rust/lib_kinds/rlib/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "rlib" +version = "0.1.0" +authors = ["Pants Build "] +edition = "2018" + +[lib] +crate-type = ["rlib"] diff --git a/contrib/rust/examples/src/rust/lib_kinds/rlib/src/lib.rs b/contrib/rust/examples/src/rust/lib_kinds/rlib/src/lib.rs new file mode 100644 index 00000000000..31e1bb209f9 --- /dev/null +++ b/contrib/rust/examples/src/rust/lib_kinds/rlib/src/lib.rs @@ -0,0 +1,7 @@ +#[cfg(test)] +mod tests { + #[test] + fn it_works() { + assert_eq!(2 + 2, 4); + } +} diff --git a/contrib/rust/examples/src/rust/lib_kinds/staticlib/BUILD b/contrib/rust/examples/src/rust/lib_kinds/staticlib/BUILD new file mode 100644 index 00000000000..a5196e3416b --- /dev/null +++ b/contrib/rust/examples/src/rust/lib_kinds/staticlib/BUILD @@ -0,0 +1,8 @@ +# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + + +cargo_library( + name='staticlib', + sources=rglobs('*.rs', 'Cargo.toml', exclude=[rglobs('**/target/*.rs')]), +) diff --git a/contrib/rust/examples/src/rust/lib_kinds/staticlib/Cargo.toml b/contrib/rust/examples/src/rust/lib_kinds/staticlib/Cargo.toml new file mode 100644 index 00000000000..209348bcc63 --- /dev/null +++ b/contrib/rust/examples/src/rust/lib_kinds/staticlib/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "staticlib" +version = "0.1.0" +authors = ["Pants Build "] +edition = "2018" + +[lib] +crate-type = ["staticlib"] \ No newline at end of file diff --git a/contrib/rust/examples/src/rust/lib_kinds/staticlib/src/lib.rs b/contrib/rust/examples/src/rust/lib_kinds/staticlib/src/lib.rs new file mode 100644 index 00000000000..31e1bb209f9 --- /dev/null +++ b/contrib/rust/examples/src/rust/lib_kinds/staticlib/src/lib.rs @@ -0,0 +1,7 @@ +#[cfg(test)] +mod tests { + #[test] + fn it_works() { + assert_eq!(2 + 2, 4); + } +} diff --git a/contrib/rust/examples/src/rust/library/BUILD b/contrib/rust/examples/src/rust/library/BUILD new file mode 100644 index 00000000000..453ae3473eb --- /dev/null +++ b/contrib/rust/examples/src/rust/library/BUILD @@ -0,0 +1,8 @@ +# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + + +cargo_library( + name='library', + sources=rglobs('*.rs', 'Cargo.toml', exclude=[rglobs('**/target/*.rs')]) +) diff --git a/contrib/rust/examples/src/rust/library/Cargo.toml b/contrib/rust/examples/src/rust/library/Cargo.toml new file mode 100644 index 00000000000..11a633a54d8 --- /dev/null +++ b/contrib/rust/examples/src/rust/library/Cargo.toml @@ -0,0 +1,7 @@ +[package] +name = "library" +version = "0.1.0" +authors = ["Pants Build "] +edition = "2018" + +[dependencies] diff --git a/contrib/rust/examples/src/rust/library/src/lib.rs b/contrib/rust/examples/src/rust/library/src/lib.rs new file mode 100644 index 00000000000..31e1bb209f9 --- /dev/null +++ b/contrib/rust/examples/src/rust/library/src/lib.rs @@ -0,0 +1,7 @@ +#[cfg(test)] +mod tests { + #[test] + fn it_works() { + assert_eq!(2 + 2, 4); + } +} diff --git a/contrib/rust/examples/src/rust/workspace/BUILD b/contrib/rust/examples/src/rust/workspace/BUILD new file mode 100644 index 00000000000..9b765e9628c --- /dev/null +++ b/contrib/rust/examples/src/rust/workspace/BUILD @@ -0,0 +1,7 @@ +# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +cargo_workspace( + name='workspace', + include=['*.rs', 'Cargo.toml'] +) diff --git a/contrib/rust/examples/src/rust/workspace/Cargo.toml b/contrib/rust/examples/src/rust/workspace/Cargo.toml new file mode 100644 index 00000000000..a2695df4e21 --- /dev/null +++ b/contrib/rust/examples/src/rust/workspace/Cargo.toml @@ -0,0 +1,6 @@ +[workspace] +members = [ + "bin_1", + "bin_2", + "lib_1" +] diff --git a/contrib/rust/examples/src/rust/workspace/bin_1/Cargo.toml b/contrib/rust/examples/src/rust/workspace/bin_1/Cargo.toml new file mode 100644 index 00000000000..6332a8bdd9e --- /dev/null +++ b/contrib/rust/examples/src/rust/workspace/bin_1/Cargo.toml @@ -0,0 +1,7 @@ +[package] +name = "bin_1" +version = "0.1.0" +authors = ["Pants Build "] +edition = "2018" + +[dependencies] diff --git a/contrib/rust/examples/src/rust/workspace/bin_1/src/main.rs b/contrib/rust/examples/src/rust/workspace/bin_1/src/main.rs new file mode 100644 index 00000000000..e7a11a969c0 --- /dev/null +++ b/contrib/rust/examples/src/rust/workspace/bin_1/src/main.rs @@ -0,0 +1,3 @@ +fn main() { + println!("Hello, world!"); +} diff --git a/contrib/rust/examples/src/rust/workspace/bin_2/Cargo.toml b/contrib/rust/examples/src/rust/workspace/bin_2/Cargo.toml new file mode 100644 index 00000000000..ebda5ac8ca5 --- /dev/null +++ b/contrib/rust/examples/src/rust/workspace/bin_2/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "bin_2" +version = "0.1.0" +authors = ["Pants Build "] +edition = "2018" + +[dependencies] +bin_1 = { path = "../bin_1" } \ No newline at end of file diff --git a/contrib/rust/examples/src/rust/workspace/bin_2/src/main.rs b/contrib/rust/examples/src/rust/workspace/bin_2/src/main.rs new file mode 100644 index 00000000000..e7a11a969c0 --- /dev/null +++ b/contrib/rust/examples/src/rust/workspace/bin_2/src/main.rs @@ -0,0 +1,3 @@ +fn main() { + println!("Hello, world!"); +} diff --git a/contrib/rust/examples/src/rust/workspace/lib_1/Cargo.toml b/contrib/rust/examples/src/rust/workspace/lib_1/Cargo.toml new file mode 100644 index 00000000000..59b200c286c --- /dev/null +++ b/contrib/rust/examples/src/rust/workspace/lib_1/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "lib_1" +version = "0.1.0" +authors = ["Pants Build "] +edition = "2018" + +[dependencies] +bin_2 = { path = "../bin_2" } \ No newline at end of file diff --git a/contrib/rust/examples/src/rust/workspace/lib_1/src/lib.rs b/contrib/rust/examples/src/rust/workspace/lib_1/src/lib.rs new file mode 100644 index 00000000000..31e1bb209f9 --- /dev/null +++ b/contrib/rust/examples/src/rust/workspace/lib_1/src/lib.rs @@ -0,0 +1,7 @@ +#[cfg(test)] +mod tests { + #[test] + fn it_works() { + assert_eq!(2 + 2, 4); + } +} diff --git a/contrib/rust/src/python/pants/__init__.py b/contrib/rust/src/python/pants/__init__.py new file mode 100644 index 00000000000..de40ea7ca05 --- /dev/null +++ b/contrib/rust/src/python/pants/__init__.py @@ -0,0 +1 @@ +__import__('pkg_resources').declare_namespace(__name__) diff --git a/contrib/rust/src/python/pants/contrib/__init__.py b/contrib/rust/src/python/pants/contrib/__init__.py new file mode 100644 index 00000000000..de40ea7ca05 --- /dev/null +++ b/contrib/rust/src/python/pants/contrib/__init__.py @@ -0,0 +1 @@ +__import__('pkg_resources').declare_namespace(__name__) diff --git a/contrib/rust/src/python/pants/contrib/rust/BUILD b/contrib/rust/src/python/pants/contrib/rust/BUILD new file mode 100644 index 00000000000..7dd17526ca3 --- /dev/null +++ b/contrib/rust/src/python/pants/contrib/rust/BUILD @@ -0,0 +1,16 @@ +# Copyright 2015 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +contrib_plugin( + name='plugin', + dependencies=[ + 'contrib/rust/src/python/pants/contrib/rust/targets/original', + 'contrib/rust/src/python/pants/contrib/rust/tasks', + 'src/python/pants/build_graph', + 'src/python/pants/goal:task_registrar' + ], + distribution_name='pantsbuild.pants.contrib.rust', + description='Rust support for pants.', + build_file_aliases=True, + register_goals=True +) diff --git a/contrib/rust/src/python/pants/contrib/rust/__init__.py b/contrib/rust/src/python/pants/contrib/rust/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/contrib/rust/src/python/pants/contrib/rust/register.py b/contrib/rust/src/python/pants/contrib/rust/register.py new file mode 100644 index 00000000000..f86e2e1ff15 --- /dev/null +++ b/contrib/rust/src/python/pants/contrib/rust/register.py @@ -0,0 +1,33 @@ +# coding=utf-8 +# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import, division, print_function, unicode_literals + +from pants.build_graph.build_file_aliases import BuildFileAliases +from pants.goal.task_registrar import TaskRegistrar as task + +from pants.contrib.rust.targets.original.cargo_binary import CargoBinary +from pants.contrib.rust.targets.original.cargo_library import CargoLibrary +from pants.contrib.rust.targets.original.cargo_workspace import CargoWorkspace +from pants.contrib.rust.tasks.cargo_binary import Binary +from pants.contrib.rust.tasks.cargo_build import Build +from pants.contrib.rust.tasks.cargo_fetch import Fetch +from pants.contrib.rust.tasks.cargo_test import Test +from pants.contrib.rust.tasks.cargo_toolchain import Toolchain + + +def build_file_aliases(): + return BuildFileAliases(targets={ + 'cargo_binary': CargoBinary, + 'cargo_library': CargoLibrary, + 'cargo_workspace': CargoWorkspace + }) + + +def register_goals(): + task(name='cargo', action=Toolchain).install('bootstrap') + task(name='cargo', action=Fetch).install('resolve') + task(name='cargo', action=Build).install('compile') + task(name='cargo', action=Binary).install('binary') + task(name='cargo', action=Test).install('test') diff --git a/contrib/rust/src/python/pants/contrib/rust/targets/__init__.py b/contrib/rust/src/python/pants/contrib/rust/targets/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/contrib/rust/src/python/pants/contrib/rust/targets/original/BUILD b/contrib/rust/src/python/pants/contrib/rust/targets/original/BUILD new file mode 100644 index 00000000000..91ad88f5fba --- /dev/null +++ b/contrib/rust/src/python/pants/contrib/rust/targets/original/BUILD @@ -0,0 +1,54 @@ +target( + dependencies=[ + ':cargo_base', + ':cargo_binary', + ':cargo_library', + ':cargo_workspace', + ] +) + +python_library( + name='cargo_base', + sources=['cargo_base.py'], + dependencies=[ + '3rdparty/python:future', + 'src/python/pants/build_graph', + 'src/python/pants/base:payload', + 'src/python/pants/base:payload_field', + 'src/python/pants/base:build_environment', + ] +) + +python_library( + name='cargo_binary', + sources=['cargo_binary.py'], + dependencies=[ + '3rdparty/python:future', + ':cargo_base', + ] +) + +python_library( + name='cargo_library', + sources=['cargo_library.py'], + dependencies=[ + '3rdparty/python:future', + ':cargo_base', + ] +) + +python_library( + name='cargo_workspace', + sources=['cargo_workspace.py'], + dependencies=[ + '3rdparty/python:future', + '3rdparty/python:toml', + 'src/python/pants/base:payload', + 'src/python/pants/base:payload_field', + 'src/python/pants/base:build_environment', + ':cargo_base', + ] +) + + + diff --git a/contrib/rust/src/python/pants/contrib/rust/targets/original/__init__.py b/contrib/rust/src/python/pants/contrib/rust/targets/original/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/contrib/rust/src/python/pants/contrib/rust/targets/original/cargo_base.py b/contrib/rust/src/python/pants/contrib/rust/targets/original/cargo_base.py new file mode 100644 index 00000000000..b678b8e3c1e --- /dev/null +++ b/contrib/rust/src/python/pants/contrib/rust/targets/original/cargo_base.py @@ -0,0 +1,41 @@ +# coding=utf-8 +# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import, division, print_function, unicode_literals + +import os + +from pants.base.build_environment import get_buildroot +from pants.base.payload import Payload +from pants.base.payload_field import PrimitiveField +from pants.build_graph.target import Target + + +class CargoBase(Target): + """A base class for all original cargo targets.""" + + def __init__(self, address=None, sources=None, manifest=None, payload=None, **kwargs): + """ + :param sources: Source code files to build. Paths are relative to the BUILD file's directory. + :type sources: :class:`pants.source.wrapped_globs.FilesetWithSpec` (from globs or rglobs) or + list of strings + :param manifest: The path of the Cargo.toml file (relative to the BUILD file directory). Default is the path of the BUILD file directory. + :type manifest: String + """ + payload = payload or Payload() + + payload.add_field( + 'sources', + self.create_sources_field( + sources=sources, sources_rel_path=address.spec_path, key_arg='sources')) + + manifest_default = os.path.join(get_buildroot(), address.spec_path) + + payload.add_field('manifest', PrimitiveField(manifest or manifest_default)) + + super(CargoBase, self).__init__(address=address, payload=payload, **kwargs) + + @property + def manifest(self): + return self.payload.get_field_value('manifest') diff --git a/contrib/rust/src/python/pants/contrib/rust/targets/original/cargo_binary.py b/contrib/rust/src/python/pants/contrib/rust/targets/original/cargo_binary.py new file mode 100644 index 00000000000..7a2710f0e91 --- /dev/null +++ b/contrib/rust/src/python/pants/contrib/rust/targets/original/cargo_binary.py @@ -0,0 +1,11 @@ +# coding=utf-8 +# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import, division, print_function, unicode_literals + +from pants.contrib.rust.targets.original.cargo_base import CargoBase + + +class CargoBinary(CargoBase): + """A base class for all original cargo binary targets.""" diff --git a/contrib/rust/src/python/pants/contrib/rust/targets/original/cargo_library.py b/contrib/rust/src/python/pants/contrib/rust/targets/original/cargo_library.py new file mode 100644 index 00000000000..769e451d8f0 --- /dev/null +++ b/contrib/rust/src/python/pants/contrib/rust/targets/original/cargo_library.py @@ -0,0 +1,11 @@ +# coding=utf-8 +# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import, division, print_function, unicode_literals + +from pants.contrib.rust.targets.original.cargo_base import CargoBase + + +class CargoLibrary(CargoBase): + """A base class for all original cargo library targets.""" diff --git a/contrib/rust/src/python/pants/contrib/rust/targets/original/cargo_workspace.py b/contrib/rust/src/python/pants/contrib/rust/targets/original/cargo_workspace.py new file mode 100644 index 00000000000..c0194f79ad3 --- /dev/null +++ b/contrib/rust/src/python/pants/contrib/rust/targets/original/cargo_workspace.py @@ -0,0 +1,67 @@ +# coding=utf-8 +# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import, division, print_function, unicode_literals + +import os + +import toml +from pants.base.build_environment import get_buildroot +from pants.base.payload import Payload +from pants.base.payload_field import PrimitiveField + +from pants.contrib.rust.targets.original.cargo_base import CargoBase + + +class CargoWorkspace(CargoBase): + """A base class for all original cargo workspace targets.""" + + def __init__(self, address=None, manifest=None, include=None, payload=None, **kwargs): + + if manifest is not None: + manifest = os.path.join(get_buildroot(), address.spec_path, manifest) + else: + manifest = os.path.join(get_buildroot(), address.spec_path) + + payload = payload or Payload() + + member_toml_paths = self.get_member_paths(manifest) + member_names = self.get_member_names(manifest, member_toml_paths) + member_src_paths = list( + map(lambda path: os.path.join(address.spec_path, path), member_toml_paths)) + + payload.add_field('member_names', PrimitiveField(member_names)) + payload.add_field('member_paths', PrimitiveField(member_src_paths)) + + payload.add_field('include_sources', PrimitiveField(include)) + + super(CargoWorkspace, self).__init__( + address=address, manifest=manifest, payload=payload, **kwargs) + + @property + def member_paths(self): + return self.payload.get_field_value('member_paths') + + @property + def member_names(self): + return self.payload.get_field_value('member_names') + + @property + def include_sources(self): + return self.payload.get_field_value('include_sources') + + def get_member_paths(self, workspace_manifest): + workspace_toml_path = os.path.join(workspace_manifest, 'Cargo.toml') + workspace_toml = toml.load(workspace_toml_path) + member_paths = workspace_toml['workspace']['members'] + + return member_paths + + def get_member_names(self, workspace_manifest, member_paths): + member_names = [] + for member_path in member_paths: + member_toml_path = os.path.join(workspace_manifest, member_path, 'Cargo.toml') + member_toml = toml.load(member_toml_path) + member_names.append(member_toml['package']['name']) + return member_names diff --git a/contrib/rust/src/python/pants/contrib/rust/targets/synthetic/BUILD b/contrib/rust/src/python/pants/contrib/rust/targets/synthetic/BUILD new file mode 100644 index 00000000000..e9e14aff4b1 --- /dev/null +++ b/contrib/rust/src/python/pants/contrib/rust/targets/synthetic/BUILD @@ -0,0 +1,79 @@ +target( + dependencies=[ + ':cargo_project_binary', + ':cargo_project_library', + ':cargo_project_test', + ':cargo_synthetic_base', + ':cargo_synthetic_binary', + ':cargo_synthetic_library', + ':cargo_synthetic_custom_build', + ] +) + +python_library( + name='cargo_project_binary', + sources=['cargo_project_binary.py'], + dependencies=[ + '3rdparty/python:future', + 'src/python/pants/base:payload', + ':cargo_synthetic_binary', + ] +) + +python_library( + name='cargo_project_library', + sources=['cargo_project_library.py'], + dependencies=[ + '3rdparty/python:future', + 'src/python/pants/base:payload', + ':cargo_synthetic_library', + ] +) + +python_library( + name='cargo_project_test', + sources=['cargo_project_test.py'], + dependencies=[ + '3rdparty/python:future', + 'src/python/pants/base:payload', + ':cargo_synthetic_base', + ] +) + +python_library( + name='cargo_synthetic_base', + sources=['cargo_synthetic_base.py'], + dependencies=[ + '3rdparty/python:future', + 'src/python/pants/build_graph', + 'src/python/pants/base:payload', + 'src/python/pants/base:payload_field', + ] +) + +python_library( + name='cargo_synthetic_binary', + sources=['cargo_synthetic_binary.py'], + dependencies=[ + '3rdparty/python:future', + ':cargo_synthetic_base', + ] +) + +python_library( + name='cargo_synthetic_library', + sources=['cargo_synthetic_library.py'], + dependencies=[ + '3rdparty/python:future', + ':cargo_synthetic_base', + ] +) + +python_library( + name='cargo_synthetic_custom_build', + sources=['cargo_synthetic_custom_build.py'], + dependencies=[ + '3rdparty/python:future', + ':cargo_synthetic_base', + ] +) diff --git a/contrib/rust/src/python/pants/contrib/rust/targets/synthetic/__init__.py b/contrib/rust/src/python/pants/contrib/rust/targets/synthetic/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/contrib/rust/src/python/pants/contrib/rust/targets/synthetic/cargo_project_binary.py b/contrib/rust/src/python/pants/contrib/rust/targets/synthetic/cargo_project_binary.py new file mode 100644 index 00000000000..c1665c3753f --- /dev/null +++ b/contrib/rust/src/python/pants/contrib/rust/targets/synthetic/cargo_project_binary.py @@ -0,0 +1,24 @@ +# coding=utf-8 +# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import, division, print_function, unicode_literals + +from pants.base.payload import Payload + +from pants.contrib.rust.targets.synthetic.cargo_synthetic_binary import CargoSyntheticBinary + + +class CargoProjectBinary(CargoSyntheticBinary): + """A base class for all synthetic project related cargo binary targets.""" + + def __init__(self, address=None, payload=None, sources=None, **kwargs): + payload = payload or Payload() + + if sources: + payload.add_field( + 'sources', + self.create_sources_field( + sources=sources, sources_rel_path=address.spec_path, key_arg='sources')) + + super(CargoProjectBinary, self).__init__(address=address, payload=payload, **kwargs) diff --git a/contrib/rust/src/python/pants/contrib/rust/targets/synthetic/cargo_project_library.py b/contrib/rust/src/python/pants/contrib/rust/targets/synthetic/cargo_project_library.py new file mode 100644 index 00000000000..96f345a45dd --- /dev/null +++ b/contrib/rust/src/python/pants/contrib/rust/targets/synthetic/cargo_project_library.py @@ -0,0 +1,24 @@ +# coding=utf-8 +# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import, division, print_function, unicode_literals + +from pants.base.payload import Payload + +from pants.contrib.rust.targets.synthetic.cargo_synthetic_library import CargoSyntheticLibrary + + +class CargoProjectLibrary(CargoSyntheticLibrary): + """A base class for all synthetic project related cargo library targets.""" + + def __init__(self, address=None, payload=None, sources=None, **kwargs): + payload = payload or Payload() + + if sources: + payload.add_field( + 'sources', + self.create_sources_field( + sources=sources, sources_rel_path=address.spec_path, key_arg='sources')) + + super(CargoProjectLibrary, self).__init__(address=address, payload=payload, **kwargs) diff --git a/contrib/rust/src/python/pants/contrib/rust/targets/synthetic/cargo_project_test.py b/contrib/rust/src/python/pants/contrib/rust/targets/synthetic/cargo_project_test.py new file mode 100644 index 00000000000..ecc4bc81769 --- /dev/null +++ b/contrib/rust/src/python/pants/contrib/rust/targets/synthetic/cargo_project_test.py @@ -0,0 +1,24 @@ +# coding=utf-8 +# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import, division, print_function, unicode_literals + +from pants.base.payload import Payload + +from pants.contrib.rust.targets.synthetic.cargo_synthetic_base import CargoSyntheticBase + + +class CargoProjectTest(CargoSyntheticBase): + """A base class for all synthetic project related cargo test targets.""" + + def __init__(self, address=None, payload=None, sources=None, **kwargs): + payload = payload or Payload() + + if sources: + payload.add_field( + 'sources', + self.create_sources_field( + sources=sources, sources_rel_path=address.spec_path, key_arg='sources')) + + super(CargoProjectTest, self).__init__(address=address, payload=payload, **kwargs) diff --git a/contrib/rust/src/python/pants/contrib/rust/targets/synthetic/cargo_synthetic_base.py b/contrib/rust/src/python/pants/contrib/rust/targets/synthetic/cargo_synthetic_base.py new file mode 100644 index 00000000000..7eb0ec83dfa --- /dev/null +++ b/contrib/rust/src/python/pants/contrib/rust/targets/synthetic/cargo_synthetic_base.py @@ -0,0 +1,23 @@ +# coding=utf-8 +# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import, division, print_function, unicode_literals + +from pants.base.payload import Payload +from pants.base.payload_field import PrimitiveField +from pants.build_graph.target import Target + + +class CargoSyntheticBase(Target): + """A base class for all synthetic cargo targets.""" + + def __init__(self, address=None, payload=None, cargo_invocation=None, **kwargs): + payload = payload or Payload() + payload.add_field('cargo_invocation', PrimitiveField(cargo_invocation)) + + super(CargoSyntheticBase, self).__init__(address=address, payload=payload, **kwargs) + + @property + def cargo_invocation(self): + return self.payload.get_field_value('cargo_invocation') diff --git a/contrib/rust/src/python/pants/contrib/rust/targets/synthetic/cargo_synthetic_binary.py b/contrib/rust/src/python/pants/contrib/rust/targets/synthetic/cargo_synthetic_binary.py new file mode 100644 index 00000000000..ed92a22e335 --- /dev/null +++ b/contrib/rust/src/python/pants/contrib/rust/targets/synthetic/cargo_synthetic_binary.py @@ -0,0 +1,11 @@ +# coding=utf-8 +# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import, division, print_function, unicode_literals + +from pants.contrib.rust.targets.synthetic.cargo_synthetic_base import CargoSyntheticBase + + +class CargoSyntheticBinary(CargoSyntheticBase): + """A base class for all synthetic cargo binary targets.""" diff --git a/contrib/rust/src/python/pants/contrib/rust/targets/synthetic/cargo_synthetic_custom_build.py b/contrib/rust/src/python/pants/contrib/rust/targets/synthetic/cargo_synthetic_custom_build.py new file mode 100644 index 00000000000..638ddb335b1 --- /dev/null +++ b/contrib/rust/src/python/pants/contrib/rust/targets/synthetic/cargo_synthetic_custom_build.py @@ -0,0 +1,11 @@ +# coding=utf-8 +# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import, division, print_function, unicode_literals + +from pants.contrib.rust.targets.synthetic.cargo_synthetic_base import CargoSyntheticBase + + +class CargoSyntheticCustomBuild(CargoSyntheticBase): + """A base class for all synthetic cargo custom build targets.""" diff --git a/contrib/rust/src/python/pants/contrib/rust/targets/synthetic/cargo_synthetic_library.py b/contrib/rust/src/python/pants/contrib/rust/targets/synthetic/cargo_synthetic_library.py new file mode 100644 index 00000000000..c4ea98e25ee --- /dev/null +++ b/contrib/rust/src/python/pants/contrib/rust/targets/synthetic/cargo_synthetic_library.py @@ -0,0 +1,11 @@ +# coding=utf-8 +# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import, division, print_function, unicode_literals + +from pants.contrib.rust.targets.synthetic.cargo_synthetic_base import CargoSyntheticBase + + +class CargoSyntheticLibrary(CargoSyntheticBase): + """A base class for all synthetic cargo library targets.""" diff --git a/contrib/rust/src/python/pants/contrib/rust/tasks/BUILD b/contrib/rust/src/python/pants/contrib/rust/tasks/BUILD new file mode 100644 index 00000000000..41505d90503 --- /dev/null +++ b/contrib/rust/src/python/pants/contrib/rust/tasks/BUILD @@ -0,0 +1,131 @@ +target( + dependencies=[ + ':cargo_binary', + ':cargo_build', + ':cargo_fetch', + ':cargo_task', + ':cargo_test', + ':cargo_toolchain', + ':cargo_fingerprint_strategy', + ] +) + +python_library( + name='cargo_binary', + sources=['cargo_binary.py'], + dependencies=[ + '3rdparty/python:future', + 'src/python/pants/base:build_environment', + 'src/python/pants/util:dirutil', + ':cargo_task', + ] +) + +python_library( + name='cargo_build', + sources=['cargo_build.py'], + dependencies=[ + '3rdparty/python:future', + 'src/python/pants/base:exceptions', + 'src/python/pants/base:workunit', + 'src/python/pants/build_graph', + 'src/python/pants/base:build_environment', + 'src/python/pants/util:dirutil', + 'src/python/pants/invalidation', + 'contrib/rust/src/python/pants/contrib/rust/utils:basic_invocation_conversion', + 'contrib/rust/src/python/pants/contrib/rust/utils:custom_build_invocation_conversion', + 'contrib/rust/src/python/pants/contrib/rust/utils:custom_build_output_parsing', + 'contrib/rust/src/python/pants/contrib/rust/utils/basic_invocation:args_rules', + 'contrib/rust/src/python/pants/contrib/rust/utils/basic_invocation:env_rules', + ':cargo_workspace', + ':cargo_fingerprint_strategy', + ] +) + +python_library( + name='cargo_fetch', + sources=['cargo_fetch.py'], + dependencies=[ + '3rdparty/python:future', + 'src/python/pants/base:workunit', + 'src/python/pants/util:dirutil', + 'src/python/pants/base:exceptions', + ':cargo_task', + ] +) + +python_library( + name='cargo_task', + sources=['cargo_task.py'], + dependencies=[ + '3rdparty/python:future', + 'src/python/pants/task:task', + 'src/python/pants/util:process_handler', + 'src/python/pants/base:workunit', + 'src/python/pants/base:build_environment', + 'contrib/rust/src/python/pants/contrib/rust/targets/original:cargo_base', + 'contrib/rust/src/python/pants/contrib/rust/targets/original:cargo_binary', + 'contrib/rust/src/python/pants/contrib/rust/targets/original:cargo_library', + 'contrib/rust/src/python/pants/contrib/rust/targets/original:cargo_workspace', + 'contrib/rust/src/python/pants/contrib/rust/targets/synthetic:cargo_project_binary', + 'contrib/rust/src/python/pants/contrib/rust/targets/synthetic:cargo_project_library', + 'contrib/rust/src/python/pants/contrib/rust/targets/synthetic:cargo_project_test', + 'contrib/rust/src/python/pants/contrib/rust/targets/synthetic:cargo_synthetic_base', + 'contrib/rust/src/python/pants/contrib/rust/targets/synthetic:cargo_synthetic_library', + 'contrib/rust/src/python/pants/contrib/rust/targets/synthetic:cargo_synthetic_binary', + 'contrib/rust/src/python/pants/contrib/rust/targets/synthetic:cargo_synthetic_custom_build', + ] +) + +python_library( + name='cargo_test', + sources=['cargo_test.py'], + dependencies=[ + '3rdparty/python:future', + 'src/python/pants/base:build_environment', + 'src/python/pants/base:workunit', + ':cargo_task', + ] +) + +python_library( + name='cargo_toolchain', + sources=['cargo_toolchain.py'], + dependencies=[ + '3rdparty/python:future', + 'src/python/pants/base:build_environment', + 'src/python/pants/base:exceptions', + 'src/python/pants/base:workunit', + 'src/python/pants/util:dirutil', + ':cargo_task', + ] +) + +python_library( + name='cargo_workspace', + sources=['cargo_workspace.py'], + dependencies=[ + '3rdparty/python:future', + 'src/python/pants/base:build_environment', + 'src/python/pants/source', + 'src/python/pants/engine:fs', + 'contrib/rust/src/python/pants/contrib/rust/targets/synthetic:cargo_project_binary', + 'contrib/rust/src/python/pants/contrib/rust/targets/synthetic:cargo_project_library', + 'contrib/rust/src/python/pants/contrib/rust/targets/synthetic:cargo_project_test', + 'contrib/rust/src/python/pants/contrib/rust/targets/synthetic:cargo_synthetic_library', + 'contrib/rust/src/python/pants/contrib/rust/targets/synthetic:cargo_synthetic_binary', + 'contrib/rust/src/python/pants/contrib/rust/targets/synthetic:cargo_synthetic_custom_build', + ':cargo_task', + ] +) + +python_library( + name='cargo_fingerprint_strategy', + sources=['cargo_fingerprint_strategy.py'], + dependencies=[ + '3rdparty/python:future', + 'src/python/pants/base:fingerprint_strategy', + 'contrib/rust/src/python/pants/contrib/rust/targets/synthetic:cargo_synthetic_base', + ] +) + diff --git a/contrib/rust/src/python/pants/contrib/rust/tasks/__init__.py b/contrib/rust/src/python/pants/contrib/rust/tasks/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/contrib/rust/src/python/pants/contrib/rust/tasks/cargo_binary.py b/contrib/rust/src/python/pants/contrib/rust/tasks/cargo_binary.py new file mode 100644 index 00000000000..42d96672be9 --- /dev/null +++ b/contrib/rust/src/python/pants/contrib/rust/tasks/cargo_binary.py @@ -0,0 +1,59 @@ +# coding=utf-8 +# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import, division, print_function, unicode_literals + +import os +import shutil + +from pants.base.build_environment import get_buildroot +from pants.util.dirutil import safe_mkdir + +from pants.contrib.rust.tasks.cargo_task import CargoTask + + +class Binary(CargoTask): + + @classmethod + def prepare(cls, options, round_manager): + super(Binary, cls).prepare(options, round_manager) + round_manager.require_data('rust_libs') + round_manager.require_data('rust_bins') + + @classmethod + def implementation_version(cls): + return super(Binary, cls).implementation_version() + [('Cargo_Binary', 1)] + + def copy_libraries_into_dist(self): + files = self.context.products.get_data('rust_libs') + dist_path = self.get_options().pants_distdir + path_libs = os.path.join(dist_path, 'rust', 'lib') + safe_mkdir(path_libs, clean=True) + self.copy_files_into_dist(path_libs, files) + + def copy_binaries_into_dist(self): + files = self.context.products.get_data('rust_bins') + dist_path = self.get_options().pants_distdir + path_bins = os.path.join(dist_path, 'rust', 'bin') + safe_mkdir(path_bins, clean=True) + self.copy_files_into_dist(path_bins, files) + + def copy_files_into_dist(self, libs_bins_path, files): + build_root = get_buildroot() + for name, paths in files.items(): + path_project = os.path.join(libs_bins_path, name) + safe_mkdir(path_project, clean=True) + for path in paths: + self.context.log.info('Copy: {0}\n\tto: {1}'.format( + os.path.relpath(path, build_root), os.path.relpath(path_project, build_root))) + if os.path.isfile(path): + shutil.copy(path, path_project) + else: + dest_path = os.path.join(path_project, os.path.basename(path)) + shutil.rmtree(dest_path, ignore_errors=True) + shutil.copytree(path, dest_path) + + def execute(self): + self.copy_libraries_into_dist() + self.copy_binaries_into_dist() diff --git a/contrib/rust/src/python/pants/contrib/rust/tasks/cargo_build.py b/contrib/rust/src/python/pants/contrib/rust/tasks/cargo_build.py new file mode 100644 index 00000000000..378a68ec71e --- /dev/null +++ b/contrib/rust/src/python/pants/contrib/rust/tasks/cargo_build.py @@ -0,0 +1,550 @@ +# coding=utf-8 +# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import, division, print_function, unicode_literals + +import collections +import copy +import functools +import hashlib +import json +import os +import shutil + +from future.utils import PY3 +from pants.base.build_environment import get_buildroot +from pants.base.exceptions import TaskError +from pants.base.workunit import WorkUnitLabel +from pants.build_graph.address import Address +from pants.invalidation.build_invalidator import CacheKeyGenerator +from pants.util.dirutil import absolute_symlink, read_file, safe_file_dump, safe_mkdir + +from pants.contrib.rust.tasks.cargo_fingerprint_strategy import CargoFingerprintStrategy +from pants.contrib.rust.tasks.cargo_workspace import Workspace +from pants.contrib.rust.utils.basic_invocation.args_rules import (args_rules, c_flag_rules, + l_flag_rules) +from pants.contrib.rust.utils.basic_invocation.env_rules import env_rules +from pants.contrib.rust.utils.basic_invocation_conversion import \ + convert_into_pants_invocation as convert_basic_into_pants_invocation +from pants.contrib.rust.utils.custom_build_invocation_conversion import \ + convert_into_pants_invocation as convert_custom_build_into_pants_invocation +from pants.contrib.rust.utils.custom_build_output_parsing import (filter_cargo_statements, + parse_multiple_cargo_statements) + + +class Build(Workspace): + _build_script_output_cache = {} + _build_index = {} + _package_out_dirs = {} + _libraries_dir_path = None + _build_index_dir_path = None + + @classmethod + def implementation_version(cls): + return super(Build, cls).implementation_version() + [('Cargo_Build', 1)] + + @classmethod + def register_options(cls, register): + super(Build, cls).register_options(register) + register( + '--cargo-opt', + type=list, + default=[], + help='Additional cargo build options e.g. `--release`.') + register( + '--ignore-rerun', + type=bool, + default=False, + help='Ignore rebuilding build scripts which include a cargo `rerun-if-changed` statement.') + + @classmethod + def prepare(cls, options, round_manager): + super(Build, cls).prepare(options, round_manager) + round_manager.require_data('cargo_env') + round_manager.require_data('cargo_toolchain') + + @property + def cache_target_dirs(self): + return True + + @classmethod + def product_types(cls): + return ['rust_libs', 'rust_bins', 'rust_tests'] + + def execute(self): + self.prepare_task() + self.prepare_cargo_targets() + + cargo_targets = self.context.build_graph.targets(self.is_cargo_synthetic) + self.build_targets(cargo_targets) + self.write_build_index() + + def prepare_task(self): + self.context.products.safe_create_data('rust_libs', lambda: {}) + self.context.products.safe_create_data('rust_bins', lambda: {}) + self.context.products.safe_create_data('rust_tests', lambda: {}) + self._libraries_dir_path = self.create_libraries_dir() + self._build_index_dir_path = self.create_build_index_dir() + self.read_build_index() + + def create_libraries_dir(self): + libraries_dir_path = os.path.join(self.versioned_workdir, 'deps') + safe_mkdir(libraries_dir_path) + return libraries_dir_path + + def create_build_index_dir(self): + build_index_dir_path = os.path.join(self.versioned_workdir, 'build_index') + safe_mkdir(build_index_dir_path) + return build_index_dir_path + + def prepare_cargo_targets(self): + cargo_targets = self.get_targets(self.is_cargo_original) + for target in cargo_targets: + cargo_build_plan = self.get_cargo_build_plan(target) + target_definitions = self.get_target_definitions_out_of_cargo_build_plan(cargo_build_plan) + self.generate_targets(target_definitions, target) + + if self.get_options().ignore_rerun is False: + self.check_if_build_scripts_are_invalid() + + def get_cargo_build_plan(self, target): + abs_manifest_path = os.path.join(target.manifest, self.manifest_name()) + + self.context.log.info( + 'Getting cargo build plan for manifest: {0}\nAdditional cargo options: {1}'.format( + abs_manifest_path, + self.get_options().cargo_opt)) + + toolchain = "+{}".format(self.context.products.get_data('cargo_toolchain')) + + cmd = [ + 'cargo', toolchain, 'build', '--manifest-path', abs_manifest_path, '--build-plan', '-Z', + 'unstable-options' + ] + + if self.include_compiling_tests(): + cmd.extend(['--tests']) + + cmd.extend(self.get_options().cargo_opt) + + env = { + 'CARGO_HOME': (self.context.products.get_data('cargo_env')['CARGO_HOME'], False), + 'PATH': (self.context.products.get_data('cargo_env')['PATH'], True) + } + + returncode, std_output, std_error = self.execute_command_and_get_output( + cmd, + 'cargo-build-plan', [WorkUnitLabel.COMPILER], + env_vars=env, + current_working_dir=target.manifest) + + if returncode != 0: + self.print_std_out_and_std_err(std_output, std_error) + raise TaskError('Cannot create build plan for: {}'.format(abs_manifest_path)) + + cargo_build_plan = json.loads(std_output) + + return cargo_build_plan + + def include_compiling_tests(self): + return self.context.products.is_required_data('rust_tests') + + def get_target_definitions_out_of_cargo_build_plan(self, cargo_build_plan): + cargo_invocations = cargo_build_plan['invocations'] + return list( + map(lambda invocation: self.create_target_definition(invocation), cargo_invocations)) + + def create_target_definition(self, cargo_invocation): + TargetDefinition = collections.namedtuple( + 'TargetDefinition', 'name, id, address, ' + 'dependencies, kind, ' + 'invocation, compile_mode') + name = cargo_invocation['package_name'] + fingerprint = self.calculate_target_definition_fingerprint(cargo_invocation) + id = "{}_{}".format(name, fingerprint) + rel_path = os.path.relpath(cargo_invocation['cwd'], get_buildroot()) + dependencies = cargo_invocation.pop('deps') + assert (len(cargo_invocation['target_kind']) == 1) + kind = cargo_invocation['target_kind'][0] + compile_mode = cargo_invocation['compile_mode'] + return TargetDefinition(name, id, Address(rel_path, id), dependencies, kind, cargo_invocation, + compile_mode) + + def calculate_target_definition_fingerprint(self, cargo_invocation): + invocation = copy.deepcopy(cargo_invocation) + + invocation.pop('deps') + invocation.pop('outputs') + invocation.pop('links') + + c_flag_rule = { + 'incremental': lambda _: "", + } + + l_flag_rule = { + 'dependency': lambda _: "", + } + + args_rules_set = { + '--out-dir': lambda _: "", + '-C': functools.partial(c_flag_rules, rules=c_flag_rule), + '-L': functools.partial(l_flag_rules, rules=l_flag_rule), + '--extern': lambda _: "" + } + + args_rules(invocation['args'], rules=args_rules_set) + + env_rules_set = { + 'OUT_DIR': lambda _: "", + # nightly-2018-12-31 macOS + 'DYLD_LIBRARY_PATH': lambda _: "", + # nightly macOS + 'DYLD_FALLBACK_LIBRARY_PATH': lambda _: "", + # nightly-2018-12-31 linux + 'LD_LIBRARY_PATH': lambda _: "" + } + + env_rules(invocation['env'], rules=env_rules_set) + + if invocation['compile_mode'] == 'run-custom-build': + invocation.pop('program') + + hasher = hashlib.md5(json.dumps(invocation, sort_keys=True).encode('utf-8')) + return hasher.hexdigest() if PY3 else hasher.hexdigest().decode('utf-8') + + def generate_targets(self, target_definitions, cargo_target): + for target in target_definitions: + target_exist = self.context.build_graph.get_target(target.address) + if not target_exist: + if self.is_workspace_member(target, cargo_target): + self.context.log.debug('Add project member target: {0}\ttarget kind: {1}'.format( + target.id, target.kind)) + self.inject_member_target(target, cargo_target) + elif self.is_lib_or_bin(target, cargo_target): + self.context.log.debug('Add project target: {0}\ttarget kind: {1}'.format( + target.id, target.kind)) + self.inject_lib_or_bin_target(target, cargo_target) + else: + self.context.log.debug('Add synthetic target: {0}\ttarget kind: {1}'.format( + target.id, target.kind)) + self.context.build_graph.inject_synthetic_target( + address=target.address, + target_type=self._synthetic_target_kind[target.kind], + cargo_invocation=target.invocation) + for dependency in target.dependencies: + dependency = target_definitions[dependency] + self.context.log.debug('\tInject dependency: {0}\tfor: {1}\ttarget kind: {2}'.format( + dependency.id, target.id, dependency.kind)) + self.context.build_graph.inject_dependency(target.address, dependency.address) + + def is_lib_or_bin(self, target_definition, original_target): + if not self.is_cargo_original_library(original_target) and not self.is_cargo_original_binary( + original_target): + return False + else: + is_original_target = target_definition.name == original_target.name + + if is_original_target and (self.is_lib_or_bin_target(target_definition) or + self.is_test_target(target_definition)): + return True + else: + return False + + def inject_lib_or_bin_target(self, target_definition, original_target): + if self.is_test_target(target_definition): + synthetic_target_type = self._project_target_kind['test'] + else: + synthetic_target_type = self._project_target_kind[target_definition.kind] + + synthetic_of_original_target = self.context.add_new_target( + address=target_definition.address, + target_type=synthetic_target_type, + cargo_invocation=target_definition.invocation, + dependencies=original_target.dependencies, + derived_from=original_target, + sources=original_target.sources_relative_to_target_base()) + + self.inject_synthetic_of_original_target_into_build_graph(synthetic_of_original_target, + original_target) + + def get_build_flags(self): + get_build_flags_func = copy.copy(self.get_options().cargo_opt) + get_build_flags_func.sort() + get_build_flags_func.append(self.context.products.get_data('cargo_toolchain')) + if self.include_compiling_tests(): + get_build_flags_func.append('--tests') + return ' '.join(get_build_flags_func) + + def build_targets(self, targets): + build_flags = self.get_build_flags() + fingerprint_strategy = CargoFingerprintStrategy(build_flags) + with self.invalidated( + targets, + invalidate_dependents=True, + fingerprint_strategy=fingerprint_strategy, + topological_order=True) as invalidation_check: + for vt in invalidation_check.all_vts: + pants_invocation = self.convert_cargo_invocation_into_pants_invocation(vt) + if not vt.valid: + self.build_target(vt.target, pants_invocation) + else: + self.context.log.info('{0} v{1} is up to date.'.format( + pants_invocation['package_name'], pants_invocation['package_version'])) + self.add_rust_products(vt.target, pants_invocation) + + def convert_cargo_invocation_into_pants_invocation(self, vt): + if self.is_cargo_synthetic_library(vt.target): + self.context.log.debug('Convert library invocation for: {0}'.format( + vt.target.address.target_name)) + pants_invocation = convert_basic_into_pants_invocation( + vt.target, vt.results_dir, self._package_out_dirs, self._libraries_dir_path) + elif self.is_cargo_synthetic_binary(vt.target): + self.context.log.debug('Convert binary invocation for: {0}'.format( + vt.target.address.target_name)) + pants_invocation = convert_basic_into_pants_invocation( + vt.target, vt.results_dir, self._package_out_dirs, self._libraries_dir_path) + elif self.is_cargo_project_test(vt.target): + self.context.log.debug('Convert test invocation for: {0}'.format( + vt.target.address.target_name)) + pants_invocation = convert_basic_into_pants_invocation( + vt.target, vt.results_dir, self._package_out_dirs, self._libraries_dir_path) + elif self.is_cargo_synthetic_custom_build(vt.target): + self.context.log.debug('Convert custom build invocation for: {0}'.format( + vt.target.address.target_name)) + pants_invocation = convert_custom_build_into_pants_invocation( + vt.target, vt.results_dir, self._package_out_dirs, self._libraries_dir_path) + else: + raise TaskError('Unsupported target kind for target: {0}'.format( + vt.target.address.target_name)) + return pants_invocation + + def build_target(self, target, pants_invocation): + self.context.log.info('{0} v{1}'.format(pants_invocation['package_name'], + pants_invocation['package_version'])) + + self.create_directories(pants_invocation['pants_make_dirs']) + + cmd = self.create_command(pants_invocation['program'], pants_invocation['args'], target) + + env = self.create_env(pants_invocation['env'], target) + + if pants_invocation['program'] == 'rustc': + self.execute_rustc( + pants_invocation['package_name'], cmd, '{}-{}'.format(pants_invocation['compile_mode'], + pants_invocation['target_kind'][0]), + env, pants_invocation['cwd']) + else: + self.execute_custom_build(cmd, pants_invocation['compile_mode'], env, pants_invocation['cwd'], + target) + + self.create_copies(pants_invocation['links']) + + if 'pants_make_sym_links' in pants_invocation: + self.create_library_symlink(pants_invocation['pants_make_sym_links'], + self._libraries_dir_path) + + def save_and_parse_build_script_output(self, std_output, out_dir, target): + cargo_statements = self.parse_build_script_output(std_output) + output_path = self.create_build_script_output_path(out_dir) + safe_file_dump(output_path, '\n'.join(cargo_statements), mode='w') + self._build_index.update({target.address.spec: output_path}) + return parse_multiple_cargo_statements(cargo_statements) + + def parse_build_script_output(self, output): + lines = output.split('\n') + return filter_cargo_statements(lines) + + def create_build_script_output_path(self, out_dir): + head, _ = os.path.split(out_dir) + output_path = os.path.join(head, 'output') + return output_path + + def create_build_script_stderr_path(self, out_dir): + head, _ = os.path.split(out_dir) + stderr_path = os.path.join(head, 'stderr') + return stderr_path + + def create_directories(self, make_dirs): + build_root = get_buildroot() + for dir in make_dirs: + self.context.log.debug('Create directory: {0}'.format(os.path.relpath(dir, build_root))) + safe_mkdir(dir) + + def create_library_symlink(self, make_symlinks, libraries_dir): + build_root = get_buildroot() + for file in make_symlinks: + self.context.log.debug('Create sym link: {0}\n\tto: {1}'.format( + os.path.relpath(file, build_root), os.path.relpath(libraries_dir, build_root))) + file_name = os.path.basename(file) + destination = os.path.join(libraries_dir, file_name) + absolute_symlink(file, destination) + + def create_copies(self, links): + build_root = get_buildroot() + for destination, source in links.items(): + self.context.log.debug('Copy: {0}\n\tto: {1}'.format( + os.path.relpath(source, build_root), os.path.relpath(destination, build_root))) + if not os.path.exists(source): + ## --release flag doesn't create dSYM folder + self.context.log.warn('{0} doesn\'t exist.'.format(os.path.relpath(source, build_root))) + else: + if os.path.isfile(source): + shutil.copy(source, destination) + else: + shutil.copytree(source, destination) + + def create_command(self, program, args, target): + cmd = [program] + cmd.extend(args) + return self.extend_args_with_cargo_statement(cmd, target) + + def extend_args_with_cargo_statement(self, cmd, target): + + def extend_cmd(cmd, build_script_output): + for output_cmd in build_script_output: + cmd.extend(output_cmd) + return cmd + + for dependency in target.dependencies: + build_script_output = self._build_script_output_cache.get(dependency.address.target_name, + None) + if build_script_output: + self.context.log.debug('Custom build outputs:\n{0}'.format( + self.pretty(build_script_output))) + cmd = extend_cmd(cmd, build_script_output['rustc-link-lib']) + cmd = extend_cmd(cmd, build_script_output['rustc-link-search']) + cmd = extend_cmd(cmd, build_script_output['rustc-flags']) + cmd = extend_cmd(cmd, build_script_output['rustc-cfg']) + return cmd + + def create_env(self, invocation_env, target): + env = { + 'PATH': (self.context.products.get_data('cargo_env')['PATH'], True), + 'RUSTUP_TOOLCHAIN': (self.context.products.get_data('cargo_toolchain'), False) + } + + env = self._add_env_vars(env, invocation_env) + + return self.extend_env_with_cargo_statement(env, target) + + def extend_env_with_cargo_statement(self, env, target): + for dependency in target.dependencies: + build_script_output = self._build_script_output_cache.get(dependency.address.target_name, + None) + if build_script_output: + self.context.log.debug('Custom build output:\n{0}'.format( + self.pretty(build_script_output['rustc-env']))) + for rustc_env in build_script_output['rustc-env']: + # is PATH also possible? + name, value = rustc_env + env = self._add_env_var(env, name, value) + return env + + def execute_custom_build(self, cmd, workunit_name, env, cwd, target): + returncode, std_output, std_error = self.execute_command_and_get_output( + cmd, workunit_name, [WorkUnitLabel.COMPILER], env_vars=env, current_working_dir=cwd) + + if returncode != 0: + self.print_std_out_and_std_err(std_output, std_error) + raise TaskError('Cannot execute build script for: {}'.format(cwd)) + + build_script_std_out_dir = self._package_out_dirs[target.address.target_name][1] + self._build_script_output_cache[target.address.target_name] = \ + self.save_and_parse_build_script_output(std_output, build_script_std_out_dir, target) + + if len(std_error) > 0: + stderr_path = self.create_build_script_stderr_path(build_script_std_out_dir) + safe_file_dump(stderr_path, std_error, mode='w') + self.context.log.warn('STD_ERR of {0} saved in: {1}'.format(target.address.target_name, + stderr_path)) + + for warning in self._build_script_output_cache[target.address.target_name]['warning']: + self.context.log.warn('Warning: {0}'.format(warning)) + + def execute_rustc(self, package_name, cmd, workunit_name, env, cwd): + returncode = self.execute_command( + cmd, workunit_name, [WorkUnitLabel.COMPILER], env_vars=env, current_working_dir=cwd) + + if returncode != 0: + raise TaskError('Cannot build target: {}'.format(package_name)) + + def add_rust_products(self, target, pants_invocation): + name = pants_invocation['package_name'] + links = pants_invocation['links'] + if self.is_cargo_project_library(target): + rust_libs = self.context.products.get_data('rust_libs') + current = rust_libs.get(name, []) + current.extend(list(filter(lambda path: os.path.exists(path), links.keys()))) + rust_libs.update({name: current}) + elif self.is_cargo_project_binary(target): + rust_bins = self.context.products.get_data('rust_bins') + current = rust_bins.get(name, []) + current.extend(list(filter(lambda path: os.path.exists(path), links.keys()))) + rust_bins.update({name: current}) + elif self.is_cargo_project_test(target): + env = pants_invocation['env'] + cwd_test = pants_invocation['cwd_test'] + rust_tests = self.context.products.get_data('rust_tests') + current = rust_tests.get(target.address.target_name, []) + current.extend( + list( + map(lambda path: (path, cwd_test, env), + filter(lambda path: os.path.exists(path) and os.path.isfile(path), + links.keys())))) + rust_tests.update({target.address.target_name: current}) + + def mark_target_invalid(self, address): + target = self.context.build_graph.get_target(address) + self._build_invalidator.force_invalidate((CacheKeyGenerator().key_for_target(target))) + + def mark_dependee_invalid(dependee): + self._build_invalidator.force_invalidate((CacheKeyGenerator().key_for_target(dependee))) + + self.context.build_graph.walk_transitive_dependee_graph( + [address], + work=lambda dependee: mark_dependee_invalid(dependee), + ) + + def check_if_build_scripts_are_invalid(self): + for target_addr_spec, build_script_output_path in self._build_index.items(): + target_address = Address.parse(target_addr_spec) + if self.context.build_graph.get_target(target_address) and os.path.isfile( + build_script_output_path): + build_scripts_output = read_file(build_script_output_path, binary_mode=False) + cargo_statements = self.parse_build_script_output(build_scripts_output) + parsed_statements = parse_multiple_cargo_statements(cargo_statements) + if len(parsed_statements['rerun-if-changed']) != 0 or len( + parsed_statements['rerun-if-env-changed']) != 0: + self.context.log.info('Rebuild target: {0}'.format(target_address.target_name)) + self.mark_target_invalid(target_address) + + def get_build_index_file_path(self): + return os.path.join(self._build_index_dir_path, 'index.json') + + def read_build_index(self): + build_index_path = self.get_build_index_file_path() + if not os.path.isfile(build_index_path): + self.context.log.debug('No build index was found.') + self._build_index = {} + self.invalidate() + else: + self.context.log.debug('Read build index from: {0}'.format(build_index_path)) + with open(build_index_path, 'r') as build_index_json_file: + try: + self._build_index = json.load(build_index_json_file) + except ValueError as ve: + self.context.log.debug(ve) + self._build_index = {} + self.invalidate() + + def write_build_index(self): + build_index_path = self.get_build_index_file_path() + self.context.log.debug('Write build index to: {0}'.format(build_index_path)) + mode = 'w' if PY3 else 'wb' + with open(build_index_path, mode) as build_index_json_file: + json.dump(self._build_index, build_index_json_file, indent=2, separators=(',', ': ')) + + def print_std_out_and_std_err(self, std_output, std_error): + self.context.log.error('==== stdout ====\n{0}'.format(std_output)) + self.context.log.error('==== stderr ====\n{0}'.format(std_error)) diff --git a/contrib/rust/src/python/pants/contrib/rust/tasks/cargo_fetch.py b/contrib/rust/src/python/pants/contrib/rust/tasks/cargo_fetch.py new file mode 100644 index 00000000000..6cb0dbf6d67 --- /dev/null +++ b/contrib/rust/src/python/pants/contrib/rust/tasks/cargo_fetch.py @@ -0,0 +1,67 @@ +# coding=utf-8 +# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import, division, print_function, unicode_literals + +import os + +from pants.base.exceptions import TaskError +from pants.base.workunit import WorkUnitLabel +from pants.util.dirutil import safe_mkdir + +from pants.contrib.rust.tasks.cargo_task import CargoTask + + +class Fetch(CargoTask): + + @classmethod + def prepare(cls, options, round_manager): + super(Fetch, cls).prepare(options, round_manager) + round_manager.require_data('cargo_env') + round_manager.require_data('cargo_toolchain') + + @classmethod + def implementation_version(cls): + return super(Fetch, cls).implementation_version() + [('Cargo_Fetch', 1)] + + def create_cargo_home(self): + cargo_home_path = os.path.join(self.versioned_workdir, 'cargo_home') + self.context.log.debug('Creating Cargo home in: {0}'.format(cargo_home_path)) + safe_mkdir(cargo_home_path) + return cargo_home_path + + def fetch(self, target): + abs_manifest_path = os.path.join(target.manifest, self.manifest_name()) + + self.context.log.debug('Fetching dependencies for: {0}'.format(abs_manifest_path)) + + toolchain = "+{}".format(self.context.products.get_data('cargo_toolchain')) + + cmd = ['cargo', toolchain, 'fetch', '--manifest-path', abs_manifest_path] + + env = { + 'CARGO_HOME': (self.context.products.get_data('cargo_env')['CARGO_HOME'], False), + 'PATH': (self.context.products.get_data('cargo_env')['PATH'], True) + } + + returncode = self.execute_command( + cmd, 'fetch', [WorkUnitLabel.TOOL], env_vars=env, current_working_dir=target.manifest) + + if returncode != 0: + raise TaskError('Cannot fetch dependencies for: {}'.format(abs_manifest_path)) + + def set_cargo_home(self, cargo_home): + cargo_env = self.context.products.get_data('cargo_env') + cargo_env['CARGO_HOME'] = cargo_home + + def create_and_set_cargo_home(self): + cargo_home = self.create_cargo_home() + self.set_cargo_home(cargo_home) + + def execute(self): + self.create_and_set_cargo_home() + + targets = self.get_targets(self.is_cargo_original) + for target in targets: + self.fetch(target) diff --git a/contrib/rust/src/python/pants/contrib/rust/tasks/cargo_fingerprint_strategy.py b/contrib/rust/src/python/pants/contrib/rust/tasks/cargo_fingerprint_strategy.py new file mode 100644 index 00000000000..5080a3100ab --- /dev/null +++ b/contrib/rust/src/python/pants/contrib/rust/tasks/cargo_fingerprint_strategy.py @@ -0,0 +1,35 @@ +# coding=utf-8 +# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import, division, print_function, unicode_literals + +import hashlib + +from future.utils import PY3 +from pants.base.fingerprint_strategy import FingerprintStrategy + +from pants.contrib.rust.targets.synthetic.cargo_synthetic_base import CargoSyntheticBase + + +class CargoFingerprintStrategy(FingerprintStrategy): + + def __init__(self, build_flags): + self._build_flags = build_flags + + def compute_fingerprint(self, target): + fp = target.payload.fingerprint() + if not isinstance(target, CargoSyntheticBase): + return fp + + hasher = hashlib.sha1() + hasher.update(fp.encode('utf-8')) + hasher.update(self._build_flags.encode('utf-8')) + return hasher.hexdigest() if PY3 else hasher.hexdigest().decode('utf-8') + + def __hash__(self): + return hash((type(self), self._build_flags.encode('utf-8'))) + + def __eq__(self, other): + return type(self) == type(other) and \ + self._build_flags == other._build_flags diff --git a/contrib/rust/src/python/pants/contrib/rust/tasks/cargo_task.py b/contrib/rust/src/python/pants/contrib/rust/tasks/cargo_task.py new file mode 100644 index 00000000000..e0f4f857f31 --- /dev/null +++ b/contrib/rust/src/python/pants/contrib/rust/tasks/cargo_task.py @@ -0,0 +1,158 @@ +# coding=utf-8 +# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import, division, print_function, unicode_literals + +import os +import pprint + +from pants.base.build_environment import get_buildroot +from pants.base.workunit import WorkUnit +from pants.task.task import Task +from pants.util.process_handler import subprocess + +from pants.contrib.rust.targets.original.cargo_base import CargoBase +from pants.contrib.rust.targets.original.cargo_binary import CargoBinary +from pants.contrib.rust.targets.original.cargo_library import CargoLibrary +from pants.contrib.rust.targets.original.cargo_workspace import CargoWorkspace +from pants.contrib.rust.targets.synthetic.cargo_project_binary import CargoProjectBinary +from pants.contrib.rust.targets.synthetic.cargo_project_library import CargoProjectLibrary +from pants.contrib.rust.targets.synthetic.cargo_project_test import CargoProjectTest +from pants.contrib.rust.targets.synthetic.cargo_synthetic_base import CargoSyntheticBase +from pants.contrib.rust.targets.synthetic.cargo_synthetic_binary import CargoSyntheticBinary +from pants.contrib.rust.targets.synthetic.cargo_synthetic_custom_build import \ + CargoSyntheticCustomBuild +from pants.contrib.rust.targets.synthetic.cargo_synthetic_library import CargoSyntheticLibrary + + +class CargoTask(Task): + pp = pprint.PrettyPrinter(indent=2) + + @staticmethod + def manifest_name(): + return 'Cargo.toml' + + @staticmethod + def is_cargo_original(target): + return isinstance(target, CargoBase) + + @staticmethod + def is_cargo_original_binary(target): + return isinstance(target, CargoBinary) + + @staticmethod + def is_cargo_original_library(target): + return isinstance(target, CargoLibrary) + + @staticmethod + def is_cargo_original_workspace(target): + return isinstance(target, CargoWorkspace) + + @staticmethod + def is_cargo_synthetic(target): + return isinstance(target, CargoSyntheticBase) + + @staticmethod + def is_cargo_synthetic_library(target): + return isinstance(target, CargoSyntheticLibrary) + + @staticmethod + def is_cargo_synthetic_binary(target): + return isinstance(target, CargoSyntheticBinary) + + @staticmethod + def is_cargo_synthetic_custom_build(target): + return isinstance(target, CargoSyntheticCustomBuild) + + @staticmethod + def is_cargo_project_binary(target): + return isinstance(target, CargoProjectBinary) + + @staticmethod + def is_cargo_project_library(target): + return isinstance(target, CargoProjectLibrary) + + @staticmethod + def is_cargo_project_test(target): + return isinstance(target, CargoProjectTest) + + def _add_env_var(self, dict, name, value, extend=False): + dict.update({name: (value, extend)}) + return dict + + def _add_env_vars(self, dict, other_dict, extend=False): + for name, value in other_dict.items(): + self._add_env_var(dict, name, value, extend) + return dict + + def _set_env_vars(self, env_vars): + current_env = os.environ.copy() + for name, value in env_vars.items(): + env_value, extend = value + if extend: + current_env[name] = "{}:{}".format(current_env[name], env_value).encode('utf-8') + else: + current_env[name] = env_value.encode('utf-8') + return current_env + + def execute_command(self, + command, + workunit_name, + workunit_labels, + current_working_dir=None, + env_vars=None): + + current_working_dir = current_working_dir or get_buildroot() + env_vars = env_vars or {} + + with self.context.new_workunit( + name=workunit_name, labels=workunit_labels, cmd=str(command)) as workunit: + proc_env = self._set_env_vars(env_vars) + + self.context.log.debug('Run\n\tCMD: {0}\n\tENV: {1}\n\tCWD: {2}'.format( + self.pretty(command), self.pretty(proc_env), current_working_dir)) + returncode = subprocess.call( + command, + env=proc_env, + cwd=current_working_dir, + stdout=workunit.output('stdout'), + stderr=workunit.output('stderr')) + + workunit.set_outcome(WorkUnit.SUCCESS if returncode == 0 else WorkUnit.FAILURE) + + return returncode + + def execute_command_and_get_output(self, + command, + workunit_name, + workunit_labels, + env_vars=None, + current_working_dir=None): + + current_working_dir = current_working_dir or get_buildroot() + env_vars = env_vars or {} + + with self.context.new_workunit( + name=workunit_name, labels=workunit_labels, cmd=str(command)) as workunit: + proc_env = self._set_env_vars(env_vars) + + self.context.log.debug('Run\n\tCMD: {0}\n\tENV: {1}\n\tCWD: {2}'.format( + self.pretty(command), self.pretty(proc_env), current_working_dir)) + + proc = subprocess.Popen( + command, + env=proc_env, + cwd=current_working_dir, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + + std_out, std_err = proc.communicate() + + returncode = proc.returncode + workunit.set_outcome(WorkUnit.SUCCESS if returncode == 0 else WorkUnit.FAILURE) + + return returncode, std_out.decode('utf-8'), std_err.decode('utf-8') + + def pretty(self, obj): + return self.pp.pformat(obj) diff --git a/contrib/rust/src/python/pants/contrib/rust/tasks/cargo_test.py b/contrib/rust/src/python/pants/contrib/rust/tasks/cargo_test.py new file mode 100644 index 00000000000..994983631f6 --- /dev/null +++ b/contrib/rust/src/python/pants/contrib/rust/tasks/cargo_test.py @@ -0,0 +1,60 @@ +# coding=utf-8 +# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import, division, print_function, unicode_literals + +import os +import shutil + +from pants.base.build_environment import get_buildroot +from pants.base.workunit import WorkUnitLabel + +from pants.contrib.rust.tasks.cargo_task import CargoTask + + +class Test(CargoTask): + + @classmethod + def prepare(cls, options, round_manager): + super(Test, cls).prepare(options, round_manager) + round_manager.require_data('rust_tests') + + @classmethod + def implementation_version(cls): + return super(Test, cls).implementation_version() + [('Cargo_Test', 1)] + + def maybe_copy_tests(self, test_definitions): + result_dir = self.versioned_workdir + + for definition in test_definitions: + test_path, _, _ = definition + self.context.log.info('Copy: {0}\n\tto: {1}'.format( + os.path.relpath(test_path, get_buildroot()), os.path.relpath(result_dir, + get_buildroot()))) + shutil.copy(test_path, result_dir) + + def run_test(self, test_path, test_cwd, test_env): + + test_env = self._add_env_vars({}, test_env) + + self.execute_command( + test_path, + 'run-test', [WorkUnitLabel.TEST], + env_vars=test_env, + current_working_dir=test_cwd) + + def run_tests(self, test_definitions): + result_dir = self.versioned_workdir + + for definition in test_definitions: + test_path, test_cwd, test_env = definition + test_path = os.path.join(result_dir, os.path.basename(test_path)) + if os.path.isfile(test_path): + self.run_test(test_path, test_cwd, test_env) + + def execute(self): + test_targets = self.context.products.get_data('rust_tests') + for test_definitions in test_targets.values(): + self.maybe_copy_tests(test_definitions) + self.run_tests(test_definitions) diff --git a/contrib/rust/src/python/pants/contrib/rust/tasks/cargo_toolchain.py b/contrib/rust/src/python/pants/contrib/rust/tasks/cargo_toolchain.py new file mode 100644 index 00000000000..0fc1f25701b --- /dev/null +++ b/contrib/rust/src/python/pants/contrib/rust/tasks/cargo_toolchain.py @@ -0,0 +1,138 @@ +# coding=utf-8 +# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import, division, print_function, unicode_literals + +import distutils.spawn +import hashlib +import os + +import requests +from future.utils import PY3 +from pants.base.build_environment import get_buildroot +from pants.base.exceptions import TaskError +from pants.base.workunit import WorkUnitLabel +from pants.util.dirutil import chmod_plus_x, read_file, safe_file_dump, safe_mkdir + +from pants.contrib.rust.tasks.cargo_task import CargoTask + + +class Toolchain(CargoTask): + + @classmethod + def register_options(cls, register): + super(Toolchain, cls).register_options(register) + register('--toolchain', type=str, default='nightly-2018-12-31', help='Toolchain') + register( + '--script_fingerprint', + type=str, + default='a741519217f27635fe49004764969cfac20fbce744d20b7114600012e6e80796', + help='The sha256 hash of the rustup install script (https://sh.rustup.rs).') + + @classmethod + def product_types(cls): + return ['cargo_env', 'cargo_toolchain'] + + @classmethod + def implementation_version(cls): + return super(Toolchain, cls).implementation_version() + [('Cargo_Toolchain', 1)] + + def setup_rustup(self): + self.context.log.info('Installing rustup...') + install_script = self.download_rustup_install_script() + self.check_integrity_of_rustup_install_script(install_script.text) + install_script_dir_path, install_script_file_path = self.save_rustup_install_script( + install_script.text) + + cmd = [install_script_file_path, '-y'] # -y Disable confirmation prompt. + + returncode = self.execute_command( + cmd, 'setup-rustup', [WorkUnitLabel.BOOTSTRAP], current_working_dir=install_script_dir_path) + + if returncode != 0: + raise TaskError('Cannot install rustup.') + + def download_rustup_install_script(self): + return requests.get('https://sh.rustup.rs') + + def check_integrity_of_rustup_install_script(self, install_script): + hasher = hashlib.sha256(install_script.encode('utf-8')) + current_fingerprint = hasher.hexdigest() if PY3 else hasher.hexdigest().decode('utf-8') + if current_fingerprint != self.get_options().script_fingerprint: + raise TaskError( + 'The fingerprint of the rustup script has changed!\nLast known: {0}\ncurrent: {1}'.format( + self.get_options().script_fingerprint, current_fingerprint)) + + def save_rustup_install_script(self, install_script): + save_dir_path = os.path.join(self.versioned_workdir, 'rustup_install_script') + safe_mkdir(save_dir_path, clean=True) + save_file_path = os.path.join(save_dir_path, 'rustup.sh') + self.context.log.debug('Save rustup.sh in {}'.format(save_file_path)) + safe_file_dump(save_file_path, install_script, mode='w') + chmod_plus_x(save_file_path) + return save_dir_path, save_file_path + + def install_rust_toolchain(self, toolchain): + self.context.log.debug('Installing toolchain: {0}'.format(toolchain)) + + cmd = ['rustup', 'install', toolchain] + + env = {'PATH': (self.context.products.get_data('cargo_env')['PATH'], True)} + + returncode = self.execute_command( + cmd, 'install-rustup-toolchain', [WorkUnitLabel.BOOTSTRAP], env_vars=env) + + if returncode != 0: + raise TaskError('Cannot install toolchain: {}'.format(toolchain)) + + def check_if_rustup_exist(self): + # If the rustup executable can't be find via the path variable, + # try to find it in the default location. + + return self.try_to_find_rustup_executable( + ) or self.try_to_find_rustup_executable_in_default_location() + + def try_to_find_rustup_executable(self): + return distutils.spawn.find_executable('rustup') is not None + + def try_to_find_rustup_executable_in_default_location(self): + env = os.environ.copy() + default_rustup_location = os.path.join(env['HOME'], '.cargo/bin', 'rustup') + if os.path.isfile(default_rustup_location): + self.context.log.debug('Found rustup in default location') + return True + else: + return False + + def setup_toolchain(self): + toolchain = self.get_toolchain() + self.install_rust_toolchain(toolchain) + self.context.products.safe_create_data('cargo_toolchain', lambda: toolchain) + + def set_cargo_path(self): + env = os.environ.copy() + self.context.products.safe_create_data('cargo_env', lambda: {}) + cargo_env = self.context.products.get_data('cargo_env') + cargo_env['PATH'] = os.path.join(env['HOME'], '.cargo/bin') + + def get_toolchain(self): + toolchain_opt = self.get_options().toolchain + toolchain_path = os.path.join(get_buildroot(), toolchain_opt, 'rust-toolchain') + if os.path.isfile(toolchain_path): + self.context.log.debug('Found rust-toolchain file.') + toolchain = read_file(toolchain_path, binary_mode=False) + return toolchain.strip() + else: + return toolchain_opt + + def execute(self): + self.context.log.debug('Check if rustup exist.') + if self.check_if_rustup_exist(): + self.context.log.debug('Rustup is already installed.') + else: + self.context.log.info('Rustup is missing.') + self.setup_rustup() + + self.set_cargo_path() + self.setup_toolchain() diff --git a/contrib/rust/src/python/pants/contrib/rust/tasks/cargo_workspace.py b/contrib/rust/src/python/pants/contrib/rust/tasks/cargo_workspace.py new file mode 100644 index 00000000000..552159c76ee --- /dev/null +++ b/contrib/rust/src/python/pants/contrib/rust/tasks/cargo_workspace.py @@ -0,0 +1,144 @@ +# coding=utf-8 +# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import, division, print_function, unicode_literals + +from future.utils import text_type +from pants.base.build_environment import get_buildroot +from pants.engine.fs import PathGlobs, PathGlobsAndRoot +from pants.source.wrapped_globs import EagerFilesetWithSpec, RGlobs + +from pants.contrib.rust.targets.synthetic.cargo_project_binary import CargoProjectBinary +from pants.contrib.rust.targets.synthetic.cargo_project_library import CargoProjectLibrary +from pants.contrib.rust.targets.synthetic.cargo_project_test import CargoProjectTest +from pants.contrib.rust.targets.synthetic.cargo_synthetic_binary import CargoSyntheticBinary +from pants.contrib.rust.targets.synthetic.cargo_synthetic_custom_build import \ + CargoSyntheticCustomBuild +from pants.contrib.rust.targets.synthetic.cargo_synthetic_library import CargoSyntheticLibrary +from pants.contrib.rust.tasks.cargo_task import CargoTask + + +class Workspace(CargoTask): + # https://docs.rs/cargo/0.20.0/cargo/core/manifest/enum.TargetKind.html + _synthetic_target_kind = { + 'bin': CargoSyntheticBinary, + 'lib': CargoSyntheticLibrary, + 'cdylib': CargoSyntheticLibrary, + 'rlib': CargoSyntheticLibrary, + 'proc-macro': CargoSyntheticLibrary, + 'dylib': CargoSyntheticLibrary, + 'staticlib': CargoSyntheticLibrary, + 'custom-build': CargoSyntheticCustomBuild, + } + + _project_target_kind = { + 'bin': CargoProjectBinary, + 'lib': CargoProjectLibrary, + 'cdylib': CargoProjectLibrary, + 'rlib': CargoProjectLibrary, + 'proc-macro': CargoProjectLibrary, + 'dylib': CargoProjectLibrary, + 'staticlib': CargoProjectLibrary, + 'test': CargoProjectTest, + } + + @classmethod + def implementation_version(cls): + return super(Workspace, cls).implementation_version() + [('Cargo_Workspace', 1)] + + @staticmethod + def is_target_a_member(target_name, member_names): + return target_name in member_names + + def is_workspace_member(self, target_definition, workspace_target): + if not self.is_cargo_original_workspace(workspace_target): + return False + else: + target_is_a_member = self.is_target_a_member(target_definition.name, + workspace_target.member_names) + if target_is_a_member and (self.is_lib_or_bin_target(target_definition) or + self.is_test_target(target_definition)): + return True + else: + return False + + @staticmethod + def is_lib_or_bin_target(target_definition): + if target_definition.kind == 'lib' or \ + target_definition.kind == 'bin' or \ + target_definition.kind == 'cdylib' or \ + target_definition.kind == 'rlib' or \ + target_definition.kind == 'dylib' or \ + target_definition.kind == 'staticlib' or \ + target_definition.kind == 'proc-macro': + return True + else: + return False + + @staticmethod + def is_test_target(target_definition): + if target_definition.kind == 'test' or target_definition.compile_mode == 'test': + return True + else: + return False + + def inject_member_target(self, target_definition, workspace_target): + + def find_member(target_name, members): + for member in members: + name, _, _ = member + if target_name == name: + return member + + member_definitions = tuple( + (name, path, workspace_target.include_sources) + for (name, path) in zip(workspace_target.member_names, workspace_target.member_paths)) + + member_definition = find_member(target_definition.name, member_definitions) + target_sources = self.get_member_sources_files(member_definition) + + if self.is_test_target(target_definition): + synthetic_target_type = self._project_target_kind['test'] + else: + synthetic_target_type = self._project_target_kind[target_definition.kind] + + synthetic_of_original_target = self.context.add_new_target( + address=target_definition.address, + target_type=synthetic_target_type, + derived_from=workspace_target, + dependencies=workspace_target.dependencies, + cargo_invocation=target_definition.invocation, + sources=target_sources) + + self.inject_synthetic_of_original_target_into_build_graph(synthetic_of_original_target, + workspace_target) + + def inject_synthetic_of_original_target_into_build_graph(self, synthetic_target, original_target): + # https://github.com/pantsbuild/pants/blob/fedc91cb2e1455b7a8dca9c843bbd8b553a04241/src/python/pants/task/simple_codegen_task.py#L359 + build_graph = self.context.build_graph + for dependent_address in build_graph.dependents_of(original_target.address): + build_graph.inject_dependency( + dependent=dependent_address, + dependency=synthetic_target.address, + ) + + for concrete_dependency_address in build_graph.dependencies_of(original_target.address): + build_graph.inject_dependency( + dependent=synthetic_target.address, + dependency=concrete_dependency_address, + ) + + if original_target in self.context.target_roots: + self.context.target_roots.append(synthetic_target) + + def get_member_sources_files(self, member_definition): + _, path, include_sources = member_definition + rglobs = RGlobs.to_filespec(include_sources, root=path) + path_globs = [PathGlobsAndRoot( + PathGlobs(tuple(rglobs['globs'])), + text_type(get_buildroot()), + )] + snapshot = self.context._scheduler.capture_snapshots(tuple(path_globs)) + fileset = EagerFilesetWithSpec(path, rglobs, snapshot[0]) + return fileset diff --git a/contrib/rust/src/python/pants/contrib/rust/utils/BUILD b/contrib/rust/src/python/pants/contrib/rust/utils/BUILD new file mode 100644 index 00000000000..72d3b4e03f4 --- /dev/null +++ b/contrib/rust/src/python/pants/contrib/rust/utils/BUILD @@ -0,0 +1,64 @@ +target( + dependencies=[ + ':basic_invocation_conversion', + ':basic_invocation_conversion_utils', + ':collector', + ':custom_build_invocation_conversion', + ':custom_build_output_parsing', + ] +) + +python_library( + name='basic_invocation_conversion', + sources=['basic_invocation_conversion.py'], + dependencies=[ + '3rdparty/python:future', + 'contrib/rust/src/python/pants/contrib/rust/tasks:cargo_task', + 'contrib/rust/src/python/pants/contrib/rust/utils/basic_invocation:args_rules', + 'contrib/rust/src/python/pants/contrib/rust/utils/basic_invocation:env_rules', + 'contrib/rust/src/python/pants/contrib/rust/utils/basic_invocation:links_rule', + 'contrib/rust/src/python/pants/contrib/rust/utils/basic_invocation:outputs_rule', + ':basic_invocation_conversion_utils', + ':collector', + ] +) + +python_library( + name='basic_invocation_conversion_utils', + sources=['basic_invocation_conversion_utils.py'], + dependencies=[ + '3rdparty/python:future', + ] +) + +python_library( + name='collector', + sources=['collector.py'], + dependencies=[ + '3rdparty/python:future', + ] +) + +python_library( + name='custom_build_invocation_conversion', + sources=['custom_build_invocation_conversion.py'], + dependencies=[ + '3rdparty/python:future', + 'src/python/pants/base:exceptions', + ':basic_invocation_conversion_utils', + 'contrib/rust/src/python/pants/contrib/rust/utils/custom_build_invocation:args_rules', + 'contrib/rust/src/python/pants/contrib/rust/utils/custom_build_invocation:env_rules', + 'contrib/rust/src/python/pants/contrib/rust/utils/custom_build_invocation:links_rule', + 'contrib/rust/src/python/pants/contrib/rust/utils/custom_build_invocation:outputs_rule', + 'contrib/rust/src/python/pants/contrib/rust/utils/custom_build_invocation:program_rule', + ':collector', + ] +) + +python_library( + name='custom_build_output_parsing', + sources=['custom_build_output_parsing.py'], + dependencies=[ + '3rdparty/python:future', + ] +) diff --git a/contrib/rust/src/python/pants/contrib/rust/utils/__init__.py b/contrib/rust/src/python/pants/contrib/rust/utils/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/contrib/rust/src/python/pants/contrib/rust/utils/basic_invocation/BUILD b/contrib/rust/src/python/pants/contrib/rust/utils/basic_invocation/BUILD new file mode 100644 index 00000000000..93594cb4fa8 --- /dev/null +++ b/contrib/rust/src/python/pants/contrib/rust/utils/basic_invocation/BUILD @@ -0,0 +1,61 @@ +target( + dependencies=[ + ':args_rules', + ':c_flag_rules', + ':l_flag_rules', + ':env_rules', + ':links_rule', + ':outputs_rule', + ] +) + +python_library( + name='args_rules', + sources=['args_rules.py'], + dependencies=[ + '3rdparty/python:future', + ':c_flag_rules', + ':l_flag_rules', + ] +) + +python_library( + name='c_flag_rules', + sources=['c_flag_rules.py'], + dependencies=[ + '3rdparty/python:future', + ] +) + +python_library( + name='l_flag_rules', + sources=['l_flag_rules.py'], + dependencies=[ + '3rdparty/python:future', + ] +) + +python_library( + name='env_rules', + sources=['env_rules.py'], + dependencies=[ + '3rdparty/python:future', + 'contrib/rust/src/python/pants/contrib/rust/targets/synthetic:cargo_synthetic_custom_build', + ] +) + +python_library( + name='links_rule', + sources=['links_rule.py'], + dependencies=[ + '3rdparty/python:future', + ] +) + +python_library( + name='outputs_rule', + sources=['outputs_rule.py'], + dependencies=[ + '3rdparty/python:future', + ] +) \ No newline at end of file diff --git a/contrib/rust/src/python/pants/contrib/rust/utils/basic_invocation/__init__.py b/contrib/rust/src/python/pants/contrib/rust/utils/basic_invocation/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/contrib/rust/src/python/pants/contrib/rust/utils/basic_invocation/args_rules.py b/contrib/rust/src/python/pants/contrib/rust/utils/basic_invocation/args_rules.py new file mode 100644 index 00000000000..dd2d8b7a4e6 --- /dev/null +++ b/contrib/rust/src/python/pants/contrib/rust/utils/basic_invocation/args_rules.py @@ -0,0 +1,79 @@ +# coding=utf-8 +# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import, division, print_function, unicode_literals + +import os + +from pants.contrib.rust.utils.basic_invocation.c_flag_rules import incremental +from pants.contrib.rust.utils.basic_invocation.l_flag_rules import dependency as dependency_rule + + +def c_flag_rules(key_value, rules=None, **kwargs): + if rules is None: + rules = { + 'incremental': incremental, + } + + is_key_value = key_value.split('=') + + if len(is_key_value) == 2: + key, value = is_key_value + apply_rule = rules.get(key) + return ("{}={}".format(key, apply_rule(value, **kwargs))) if apply_rule else key_value + else: + return key_value + + +def l_flag_rules(key_value, rules=None, **kwargs): + if rules is None: + rules = { + 'dependency': dependency_rule, + } + + is_key_value = key_value.split('=') + + if len(is_key_value) == 2: + key, value = is_key_value + apply_rule = rules.get(key) + return ("{}={}".format(key, apply_rule(value, **kwargs))) if apply_rule else key_value + else: + return key_value + + +def out_dir_flag_rule(_, result_dir, make_dirs, **kwargs): + new_path = os.path.join(result_dir, 'deps') + make_dirs.add(new_path) + return new_path + + +def extern_flag_rule(key_value, target, crate_out_dirs, **kwargs): + key, _ = key_value.split('=') + + extern = None + for dependency in target.dependencies: + if crate_out_dirs.get(dependency.address.target_name): + package_name, out_dir = crate_out_dirs[dependency.address.target_name] + if key == package_name: + assert (len(out_dir) == 1) + extern = "{}={}".format(key, out_dir[0]) + + assert (extern is not None) + + return extern + + +def args_rules(invocation_key, rules=None, **kwargs): + if rules is None: + rules = { + '--out-dir': out_dir_flag_rule, + '-C': c_flag_rules, + '-L': l_flag_rules, + '--extern': extern_flag_rule + } + + for index, arg in enumerate(invocation_key): + apply_rule = rules.get(arg) + if apply_rule: + invocation_key[index + 1] = apply_rule(invocation_key[index + 1], **kwargs) diff --git a/contrib/rust/src/python/pants/contrib/rust/utils/basic_invocation/c_flag_rules.py b/contrib/rust/src/python/pants/contrib/rust/utils/basic_invocation/c_flag_rules.py new file mode 100644 index 00000000000..270daa8fdcb --- /dev/null +++ b/contrib/rust/src/python/pants/contrib/rust/utils/basic_invocation/c_flag_rules.py @@ -0,0 +1,13 @@ +# coding=utf-8 +# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import, division, print_function, unicode_literals + +import os + + +def incremental(_, result_dir, make_dirs, **kwargs): + new_path = os.path.join(result_dir, 'incremental') + make_dirs.add(new_path) + return new_path diff --git a/contrib/rust/src/python/pants/contrib/rust/utils/basic_invocation/env_rules.py b/contrib/rust/src/python/pants/contrib/rust/utils/basic_invocation/env_rules.py new file mode 100644 index 00000000000..dde8043c2c1 --- /dev/null +++ b/contrib/rust/src/python/pants/contrib/rust/utils/basic_invocation/env_rules.py @@ -0,0 +1,45 @@ +# coding=utf-8 +# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import, division, print_function, unicode_literals + +import os + +from pants.contrib.rust.targets.synthetic.cargo_synthetic_custom_build import \ + CargoSyntheticCustomBuild + + +def out_dir_rule(old_path, target, crate_out_dirs, result_dir, **kwargs): + if len(target.dependencies) != 0: + for dependency in target.dependencies: + if isinstance(dependency, CargoSyntheticCustomBuild): + return crate_out_dirs[dependency.address.target_name][1] + else: + head, out = os.path.split(old_path) + head, package_name = os.path.split(head) + return os.path.join(result_dir, 'build', package_name, out) + + +def dyld_lib_path_rule(old_path, libraries_dir, **kwargs): + _, system_paths = old_path.split(':', 1) + new_path = "{}:{}".format(libraries_dir, system_paths) + return new_path + + +def env_rules(invocation_key, rules=None, **kargs): + if rules is None: + rules = { + 'OUT_DIR': out_dir_rule, + # nightly-2018-12-31 macOS + 'DYLD_LIBRARY_PATH': dyld_lib_path_rule, + # nightly macOS + 'DYLD_FALLBACK_LIBRARY_PATH': dyld_lib_path_rule, + # nightly-2018-12-31 linux + 'LD_LIBRARY_PATH': dyld_lib_path_rule + } + + for key, value in invocation_key.items(): + apply_rule = rules.get(key) + if apply_rule: + invocation_key[key] = apply_rule(value, **kargs) diff --git a/contrib/rust/src/python/pants/contrib/rust/utils/basic_invocation/l_flag_rules.py b/contrib/rust/src/python/pants/contrib/rust/utils/basic_invocation/l_flag_rules.py new file mode 100644 index 00000000000..a9a04cb5bbf --- /dev/null +++ b/contrib/rust/src/python/pants/contrib/rust/utils/basic_invocation/l_flag_rules.py @@ -0,0 +1,9 @@ +# coding=utf-8 +# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import, division, print_function, unicode_literals + + +def dependency(_, libraries_dir, **kwargs): + return libraries_dir diff --git a/contrib/rust/src/python/pants/contrib/rust/utils/basic_invocation/links_rule.py b/contrib/rust/src/python/pants/contrib/rust/utils/basic_invocation/links_rule.py new file mode 100644 index 00000000000..42c377ea1c4 --- /dev/null +++ b/contrib/rust/src/python/pants/contrib/rust/utils/basic_invocation/links_rule.py @@ -0,0 +1,25 @@ +# coding=utf-8 +# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import, division, print_function, unicode_literals + +import copy +import os + + +def change_key(path, result_dir, **kargs): + file_name = os.path.basename(path) + return os.path.join(result_dir, file_name) + + +def change_value(path, result_dir, **kargs): + file_name = os.path.basename(path) + return os.path.join(result_dir, 'deps', file_name) + + +def links_rule(invocation_key, **kargs): + cache = copy.deepcopy(invocation_key) + for key, value in cache.items(): + invocation_key.pop(key) + invocation_key[change_key(key, **kargs)] = change_value(value, **kargs) diff --git a/contrib/rust/src/python/pants/contrib/rust/utils/basic_invocation/outputs_rule.py b/contrib/rust/src/python/pants/contrib/rust/utils/basic_invocation/outputs_rule.py new file mode 100644 index 00000000000..8ae213e0ab8 --- /dev/null +++ b/contrib/rust/src/python/pants/contrib/rust/utils/basic_invocation/outputs_rule.py @@ -0,0 +1,21 @@ +# coding=utf-8 +# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import, division, print_function, unicode_literals + +import os + + +def change_path(path, result_dir, make_dirs, make_sym_links, **kargs): + file_name = os.path.basename(path) + new_dir = os.path.join(result_dir, 'deps') + make_dirs.add(new_dir) + new_file = os.path.join(new_dir, file_name) + make_sym_links.add(new_file) + return new_file + + +def outputs_rule(invocation_key, **kargs): + for index, path in enumerate(invocation_key): + invocation_key[index] = change_path(path, **kargs) diff --git a/contrib/rust/src/python/pants/contrib/rust/utils/basic_invocation_conversion.py b/contrib/rust/src/python/pants/contrib/rust/utils/basic_invocation_conversion.py new file mode 100644 index 00000000000..b58b0eb6e5d --- /dev/null +++ b/contrib/rust/src/python/pants/contrib/rust/utils/basic_invocation_conversion.py @@ -0,0 +1,56 @@ +# coding=utf-8 +# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import, division, print_function, unicode_literals + +import copy + +from pants.contrib.rust.tasks.cargo_task import CargoTask +from pants.contrib.rust.utils.basic_invocation.args_rules import args_rules +from pants.contrib.rust.utils.basic_invocation.env_rules import env_rules +from pants.contrib.rust.utils.basic_invocation.links_rule import links_rule +from pants.contrib.rust.utils.basic_invocation.outputs_rule import outputs_rule +from pants.contrib.rust.utils.basic_invocation_conversion_utils import (reduce_invocation, + sanitize_crate_name) +from pants.contrib.rust.utils.collector import (collect_information, get_default_information, + get_test_target_information) + + +def get_default_conversion_rules(): + return {'args': args_rules, 'outputs': outputs_rule, 'links': links_rule, 'env': env_rules} + + +def convert_into_pants_invocation(target, result_dir, crate_out_dirs, libraries_dir): + pants_invocation = copy.deepcopy(target.cargo_invocation) + make_dirs = set() + make_sym_links = set() + + transformation_rules = get_default_conversion_rules() + + for key in pants_invocation: + apply_rule = transformation_rules.get(key) + if apply_rule: + apply_rule( + invocation_key=pants_invocation[key], + target=target, + result_dir=result_dir, + crate_out_dirs=crate_out_dirs, + libraries_dir=libraries_dir, + make_dirs=make_dirs, + make_sym_links=make_sym_links, + ) + + # package_name and crate_name can be different and create_name is used in the rustc flag '--extern =' + information = collect_information(pants_invocation, get_default_information()) + crate_name = sanitize_crate_name(information['crate_name']) + crate_out_dirs[target.address.target_name] = (crate_name, pants_invocation['outputs']) + + if CargoTask.is_cargo_project_test(target): + test_cwd = collect_information(pants_invocation, get_test_target_information()) + pants_invocation['cwd_test'] = test_cwd['CARGO_MANIFEST_DIR'] + + pants_invocation['pants_make_dirs'] = make_dirs + pants_invocation['pants_make_sym_links'] = make_sym_links + + return reduce_invocation(pants_invocation) diff --git a/contrib/rust/src/python/pants/contrib/rust/utils/basic_invocation_conversion_utils.py b/contrib/rust/src/python/pants/contrib/rust/utils/basic_invocation_conversion_utils.py new file mode 100644 index 00000000000..61ecd16c6f2 --- /dev/null +++ b/contrib/rust/src/python/pants/contrib/rust/utils/basic_invocation_conversion_utils.py @@ -0,0 +1,14 @@ +# coding=utf-8 +# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import, division, print_function, unicode_literals + + +def sanitize_crate_name(create_name): + return create_name.replace('-', '_') + + +def reduce_invocation(invocation): + invocation.pop('kind', None) + return invocation diff --git a/contrib/rust/src/python/pants/contrib/rust/utils/collector.py b/contrib/rust/src/python/pants/contrib/rust/utils/collector.py new file mode 100644 index 00000000000..822a7ac4808 --- /dev/null +++ b/contrib/rust/src/python/pants/contrib/rust/utils/collector.py @@ -0,0 +1,63 @@ +# coding=utf-8 +# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import, division, print_function, unicode_literals + + +def args(args, collector): + + def _c_flag_filter(key_value, collector): + needs = { + 'extra-filename': True, + } + + is_key_value = key_value.split('=') + + if len(is_key_value) == 2: + key, value = is_key_value + has = needs.get(key, None) + + if has: + collector.update({key: value}) + + needs = { + '--crate-name': lambda name, collector: collector.update({'crate_name': name}), + '-C': _c_flag_filter, + } + + for index, arg in enumerate(args): + filter = needs.get(arg, None) + if filter: + filter(args[index + 1], collector) + + +def get_default_information(): + return { + 'args': args, + 'package_name': lambda name, collector: collector.update({'package_name': name}) + } + + +def get_test_target_information(): + return {'env': env} + + +def env(invocation, collector): + needs = {'CARGO_MANIFEST_DIR': lambda key, vaule, collector: collector.update({key: vaule})} + + for key, value in invocation.items(): + filter = needs.get(key, None) + if filter: + filter(key, value, collector) + + +def collect_information(invocation, requirements): + collector = dict() + + for key in invocation: + found = requirements.get(key, None) + if found: + found(invocation[key], collector) + + return collector diff --git a/contrib/rust/src/python/pants/contrib/rust/utils/custom_build_invocation/BUILD b/contrib/rust/src/python/pants/contrib/rust/utils/custom_build_invocation/BUILD new file mode 100644 index 00000000000..fb58327b134 --- /dev/null +++ b/contrib/rust/src/python/pants/contrib/rust/utils/custom_build_invocation/BUILD @@ -0,0 +1,51 @@ +target( + dependencies=[ + ':args_rules', + ':env_rules', + ':links_rule', + ':outputs_rule', + ':program_rule', + ] +) + +python_library( + name='args_rules', + sources=['args_rules.py'], + dependencies=[ + '3rdparty/python:future', + 'contrib/rust/src/python/pants/contrib/rust/utils/basic_invocation:args_rules', + ] +) + +python_library( + name='env_rules', + sources=['env_rules.py'], + dependencies=[ + '3rdparty/python:future', + 'contrib/rust/src/python/pants/contrib/rust/targets/synthetic:cargo_synthetic_custom_build', + ] +) + +python_library( + name='links_rule', + sources=['links_rule.py'], + dependencies=[ + '3rdparty/python:future', + ] +) + +python_library( + name='outputs_rule', + sources=['outputs_rule.py'], + dependencies=[ + '3rdparty/python:future', + ] +) + +python_library( + name='program_rule', + sources=['program_rule.py'], + dependencies=[ + '3rdparty/python:future', + ] +) \ No newline at end of file diff --git a/contrib/rust/src/python/pants/contrib/rust/utils/custom_build_invocation/__init__.py b/contrib/rust/src/python/pants/contrib/rust/utils/custom_build_invocation/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/contrib/rust/src/python/pants/contrib/rust/utils/custom_build_invocation/args_rules.py b/contrib/rust/src/python/pants/contrib/rust/utils/custom_build_invocation/args_rules.py new file mode 100644 index 00000000000..aa5ce58b596 --- /dev/null +++ b/contrib/rust/src/python/pants/contrib/rust/utils/custom_build_invocation/args_rules.py @@ -0,0 +1,32 @@ +# coding=utf-8 +# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import, division, print_function, unicode_literals + +import os + +from pants.contrib.rust.utils.basic_invocation.args_rules import (c_flag_rules, extern_flag_rule, + l_flag_rules) + + +def out_dir_flag_rule(_, information, result_dir, make_dirs, **kwargs): + package_name = "{}{}".format(information['package_name'], information['extra-filename']) + new_path = os.path.join(result_dir, 'build', package_name) + make_dirs.add(new_path) + return new_path + + +def args_rules(invocation_key, rules=None, **kwargs): + if rules is None: + rules = { + '--out-dir': out_dir_flag_rule, + '-C': c_flag_rules, + '-L': l_flag_rules, + '--extern': extern_flag_rule + } + + for index, arg in enumerate(invocation_key): + apply_rule = rules.get(arg) + if apply_rule: + invocation_key[index + 1] = apply_rule(invocation_key[index + 1], **kwargs) diff --git a/contrib/rust/src/python/pants/contrib/rust/utils/custom_build_invocation/env_rules.py b/contrib/rust/src/python/pants/contrib/rust/utils/custom_build_invocation/env_rules.py new file mode 100644 index 00000000000..e84b4564add --- /dev/null +++ b/contrib/rust/src/python/pants/contrib/rust/utils/custom_build_invocation/env_rules.py @@ -0,0 +1,35 @@ +# coding=utf-8 +# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import, division, print_function, unicode_literals + +import os + +from pants.contrib.rust.utils.basic_invocation.env_rules import dyld_lib_path_rule + + +def out_dir_rule(old_path, result_dir, make_dirs, **kargs): + head, out = os.path.split(old_path) + head, package_name = os.path.split(head) + new_path = os.path.join(result_dir, 'build', package_name, out) + make_dirs.add(new_path) + return new_path + + +def env_rules(invocation_key, rules=None, **kargs): + if rules is None: + rules = { + 'OUT_DIR': out_dir_rule, + # nightly-2018-12-31 macOS + 'DYLD_LIBRARY_PATH': dyld_lib_path_rule, + # nightly macOS + 'DYLD_FALLBACK_LIBRARY_PATH': dyld_lib_path_rule, + # nightly-2018-12-31 linux + 'LD_LIBRARY_PATH': dyld_lib_path_rule + } + + for key, value in invocation_key.items(): + apply_rule = rules.get(key) + if apply_rule: + invocation_key[key] = apply_rule(value, **kargs) diff --git a/contrib/rust/src/python/pants/contrib/rust/utils/custom_build_invocation/links_rule.py b/contrib/rust/src/python/pants/contrib/rust/utils/custom_build_invocation/links_rule.py new file mode 100644 index 00000000000..4400c1c547a --- /dev/null +++ b/contrib/rust/src/python/pants/contrib/rust/utils/custom_build_invocation/links_rule.py @@ -0,0 +1,21 @@ +# coding=utf-8 +# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import, division, print_function, unicode_literals + +import copy +import os + + +def change_key_value(path, result_dir, information, **kargs): + file_name = os.path.basename(path) + package_name = "{}{}".format(information['package_name'], information['extra-filename']) + return os.path.join(result_dir, 'build', package_name, file_name) + + +def links_rule(invocation_key, **kargs): + cache = copy.deepcopy(invocation_key) + for key, value in cache.items(): + invocation_key.pop(key) + invocation_key[change_key_value(key, **kargs)] = change_key_value(value, **kargs) diff --git a/contrib/rust/src/python/pants/contrib/rust/utils/custom_build_invocation/outputs_rule.py b/contrib/rust/src/python/pants/contrib/rust/utils/custom_build_invocation/outputs_rule.py new file mode 100644 index 00000000000..3d88d9daec1 --- /dev/null +++ b/contrib/rust/src/python/pants/contrib/rust/utils/custom_build_invocation/outputs_rule.py @@ -0,0 +1,20 @@ +# coding=utf-8 +# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import, division, print_function, unicode_literals + +import os + + +def change_path(path, result_dir, information, make_dirs, **kargs): + file_name = os.path.basename(path) + package_name = "{}{}".format(information['package_name'], information['extra-filename']) + new_dir = os.path.join(result_dir, 'build', package_name) + make_dirs.add(new_dir) + return os.path.join(new_dir, file_name) + + +def outputs_rule(invocation_key, **kargs): + for index, path in enumerate(invocation_key): + invocation_key[index] = change_path(path, **kargs) diff --git a/contrib/rust/src/python/pants/contrib/rust/utils/custom_build_invocation/program_rule.py b/contrib/rust/src/python/pants/contrib/rust/utils/custom_build_invocation/program_rule.py new file mode 100644 index 00000000000..0eef056f605 --- /dev/null +++ b/contrib/rust/src/python/pants/contrib/rust/utils/custom_build_invocation/program_rule.py @@ -0,0 +1,18 @@ +# coding=utf-8 +# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import, division, print_function, unicode_literals + +import os + + +def program_rule(target, crate_out_dirs, invocation, **kwargs): + for dependency in target.dependencies: + _, links = crate_out_dirs[dependency.address.target_name] + assert len(list(links.keys())) == 1, ('Cannot find build script of {target_name}'.format( + target_name=target.address.target_name)) + out_dir = list(links.keys())[0] + # https://github.com/rust-lang/cargo/blob/245818076052dd7178f5bb7585f5aec5b6c1e03e/src/cargo/util/toml/targets.rs#L107 + if os.path.basename(out_dir).startswith('build-script-', 0, 13): + invocation['program'] = out_dir diff --git a/contrib/rust/src/python/pants/contrib/rust/utils/custom_build_invocation_conversion.py b/contrib/rust/src/python/pants/contrib/rust/utils/custom_build_invocation_conversion.py new file mode 100644 index 00000000000..47624bf4b56 --- /dev/null +++ b/contrib/rust/src/python/pants/contrib/rust/utils/custom_build_invocation_conversion.py @@ -0,0 +1,62 @@ +# coding=utf-8 +# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import, division, print_function, unicode_literals + +import copy + +from pants.base.exceptions import TaskError + +from pants.contrib.rust.utils.basic_invocation_conversion_utils import reduce_invocation +from pants.contrib.rust.utils.collector import collect_information, get_default_information +from pants.contrib.rust.utils.custom_build_invocation.args_rules import args_rules +from pants.contrib.rust.utils.custom_build_invocation.env_rules import env_rules +from pants.contrib.rust.utils.custom_build_invocation.links_rule import links_rule +from pants.contrib.rust.utils.custom_build_invocation.outputs_rule import outputs_rule +from pants.contrib.rust.utils.custom_build_invocation.program_rule import program_rule + + +def get_default_build_conversion_rules(): + return {'args': args_rules, 'outputs': outputs_rule, 'links': links_rule, 'env': env_rules} + + +def get_default_run_conversion_rules(): + return {'program': program_rule, 'env': env_rules} + + +def convert_into_pants_invocation(target, result_dir, crate_out_dirs, libraries_dir): + pants_invocation = copy.deepcopy(target.cargo_invocation) + information = collect_information(pants_invocation, get_default_information()) + make_dirs = set() + + compile_mode = pants_invocation["compile_mode"] + + if compile_mode == "build": + conversion_rules_set = get_default_build_conversion_rules() + elif compile_mode == "run-custom-build": + conversion_rules_set = get_default_run_conversion_rules() + else: + raise TaskError('Unsupported compile mode! {0}'.format(compile_mode)) + + for key in pants_invocation: + apply_rule = conversion_rules_set.get(key) + if apply_rule: + apply_rule( + invocation_key=pants_invocation[key], + target=target, + result_dir=result_dir, + crate_out_dirs=crate_out_dirs, + libraries_dir=libraries_dir, + information=information, + invocation=pants_invocation, + make_dirs=make_dirs) + + if compile_mode == "build": + crate_out_dirs[target.address.target_name] = ('None', pants_invocation['links']) + else: + crate_out_dirs[target.address.target_name] = ('None', pants_invocation['env']['OUT_DIR']) + + pants_invocation['pants_make_dirs'] = make_dirs + + return reduce_invocation(pants_invocation) diff --git a/contrib/rust/src/python/pants/contrib/rust/utils/custom_build_output_parsing.py b/contrib/rust/src/python/pants/contrib/rust/utils/custom_build_output_parsing.py new file mode 100644 index 00000000000..84a76ce3e8c --- /dev/null +++ b/contrib/rust/src/python/pants/contrib/rust/utils/custom_build_output_parsing.py @@ -0,0 +1,86 @@ +# coding=utf-8 +# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import, division, print_function, unicode_literals + +from itertools import chain + + +def get_default_converter(): + # https://github.com/rust-lang/cargo/blob/dc83ead224d8622f748f507574e1448a28d8dcc7/src/cargo/core/compiler/custom_build.rs#L482 + return { + 'rustc-link-lib': lambda kind: ['-l', kind], + 'rustc-link-search': lambda path: ['-L', path], + 'rustc-flags': lambda flags: spilt_flags(flags), + 'rustc-cfg': lambda cfg: ['--cfg', cfg], + 'rustc-env': lambda var_value: spilt_into_key_value(var_value), + 'warning': lambda warning: warning, + 'rerun-if-changed': lambda file: file, + 'rerun-if-env-changed': lambda env: env, + } + + +def spilt_flags(flags): + array_flags = flags.split(' ') + filter_whitespaces = list(filter(lambda flag: flag.strip() != '', array_flags)) + iter_flags = iter(filter_whitespaces) + return list(chain.from_iterable(map(list, zip(iter_flags, iter_flags)))) + + +def spilt_into_key_value(key_value_str): + key_value = key_value_str.split('=', 1) + return key_value + + +def convert(key, value, converter): + convert_fn = converter.get(key, None) + if convert_fn: + return [key, convert_fn(value)] + else: + return [key, value] + + +def parse_cargo_statement(cargo_statement): + converter = get_default_converter() + key_value = spilt_into_key_value(cargo_statement) + if len(key_value) == 2: + key, value = key_value + return convert(key, value, converter) + else: + return key_value + + +def parse_multiple_cargo_statements(cargo_statements): + result = { + 'rustc-link-lib': [], + 'rustc-link-search': [], + 'rustc-flags': [], + 'rustc-cfg': [], + 'rustc-env': [], + 'warning': [], + 'rerun-if-changed': [], + 'rerun-if-env-changed': [], + } + + cargo_statements_without_prefix = list( + map(lambda cargo: cargo.split('cargo:', 1)[1], cargo_statements)) + + for cargo_statement in cargo_statements_without_prefix: + key_value = parse_cargo_statement(cargo_statement) + if len(key_value) == 2: + key = key_value[0] + value = key_value[1] + in_result = result.get(key, None) + if in_result is not None: + result[key].append(value) + else: + result['warning'].append('(Pants) Unsupported cargo statement: {0} - {1}'.format( + key, value)) + else: + result['warning'].append('(Pants) Unsupported cargo statement: {0}'.format(key_value)) + return result + + +def filter_cargo_statements(build_output): + return list(filter(lambda line: line.startswith('cargo:', 0, 6), build_output)) diff --git a/contrib/rust/tests/python/pants_test/__init__.py b/contrib/rust/tests/python/pants_test/__init__.py new file mode 100644 index 00000000000..de40ea7ca05 --- /dev/null +++ b/contrib/rust/tests/python/pants_test/__init__.py @@ -0,0 +1 @@ +__import__('pkg_resources').declare_namespace(__name__) diff --git a/contrib/rust/tests/python/pants_test/contrib/__init__.py b/contrib/rust/tests/python/pants_test/contrib/__init__.py new file mode 100644 index 00000000000..de40ea7ca05 --- /dev/null +++ b/contrib/rust/tests/python/pants_test/contrib/__init__.py @@ -0,0 +1 @@ +__import__('pkg_resources').declare_namespace(__name__) diff --git a/contrib/rust/tests/python/pants_test/contrib/rust/__init__.py b/contrib/rust/tests/python/pants_test/contrib/rust/__init__.py new file mode 100644 index 00000000000..de40ea7ca05 --- /dev/null +++ b/contrib/rust/tests/python/pants_test/contrib/rust/__init__.py @@ -0,0 +1 @@ +__import__('pkg_resources').declare_namespace(__name__) diff --git a/contrib/rust/tests/python/pants_test/contrib/rust/tasks/BUILD b/contrib/rust/tests/python/pants_test/contrib/rust/tasks/BUILD new file mode 100644 index 00000000000..bfce373dd62 --- /dev/null +++ b/contrib/rust/tests/python/pants_test/contrib/rust/tasks/BUILD @@ -0,0 +1,104 @@ +# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +python_tests( + name='test_cargo_task', + sources=['test_cargo_task.py'], + dependencies=[ + '3rdparty/python:future', + 'src/python/pants/build_graph', + 'src/python/pants/util:contextutil', + 'src/python/pants/util:dirutil', + 'src/python/pants/base:workunit', + 'tests/python/pants_test:task_test_base', + 'contrib/rust/src/python/pants/contrib/rust/targets/original:cargo_base', + 'contrib/rust/src/python/pants/contrib/rust/targets/original:cargo_binary', + 'contrib/rust/src/python/pants/contrib/rust/targets/original:cargo_library', + 'contrib/rust/src/python/pants/contrib/rust/targets/original:cargo_workspace', + 'contrib/rust/src/python/pants/contrib/rust/targets/synthetic:cargo_project_binary', + 'contrib/rust/src/python/pants/contrib/rust/targets/synthetic:cargo_project_library', + 'contrib/rust/src/python/pants/contrib/rust/targets/synthetic:cargo_project_test', + 'contrib/rust/src/python/pants/contrib/rust/targets/synthetic:cargo_synthetic_base', + 'contrib/rust/src/python/pants/contrib/rust/targets/synthetic:cargo_synthetic_library', + 'contrib/rust/src/python/pants/contrib/rust/targets/synthetic:cargo_synthetic_binary', + 'contrib/rust/src/python/pants/contrib/rust/targets/synthetic:cargo_synthetic_custom_build', + 'contrib/rust/src/python/pants/contrib/rust/tasks:cargo_task', + ] +) + +python_tests( + name='test_cargo_toolchain', + sources=['test_cargo_toolchain.py'], + dependencies=[ + '3rdparty/python:future', + 'src/python/pants/base:build_environment', + 'src/python/pants/util:contextutil', + 'src/python/pants/base:exceptions', + 'tests/python/pants_test:task_test_base', + 'contrib/rust/src/python/pants/contrib/rust/tasks:cargo_toolchain', + ] +) + +python_tests( + name='test_cargo_fetch', + sources=['test_cargo_fetch.py'], + dependencies=[ + '3rdparty/python:future', + 'src/python/pants/util:dirutil', + 'src/python/pants/base:build_environment', + 'src/python/pants/base:exceptions', + 'tests/python/pants_test:task_test_base', + 'contrib/rust/src/python/pants/contrib/rust/tasks:cargo_fetch', + 'contrib/rust/src/python/pants/contrib/rust/targets/original:cargo_library', + ], + timeout=120, +) + +python_tests( + name='test_cargo_workspace', + sources=['test_cargo_workspace.py'], + dependencies=[ + '3rdparty/python:future', + 'src/python/pants/util:dirutil', + 'src/python/pants/base:build_environment', + 'src/python/pants/base:exceptions', + 'tests/python/pants_test:task_test_base', + 'contrib/rust/src/python/pants/contrib/rust/tasks:cargo_workspace', + ] +) + +python_tests( + name='test_cargo_build', + sources=['test_cargo_build.py'], + dependencies=[ + '3rdparty/python:future', + 'src/python/pants/util:dirutil', + 'src/python/pants/base:build_environment', + 'src/python/pants/base:exceptions', + 'src/python/pants/build_graph', + 'tests/python/pants_test:task_test_base', + 'contrib/rust/src/python/pants/contrib/rust/tasks:cargo_build', + 'contrib/rust/src/python/pants/contrib/rust/targets/synthetic:cargo_synthetic_binary', + 'contrib/rust/src/python/pants/contrib/rust/targets/synthetic:cargo_synthetic_custom_build', + ] +) + +python_tests( + name='test_cargo_build_integration', + sources=['test_cargo_build_integration.py'], + dependencies=[ + '3rdparty/python:future', + 'tests/python/pants_test:int-test', + ], + timeout=360, +) + +python_tests( + name='test_cargo_fetch_integration', + sources=['test_cargo_fetch_integration.py'], + dependencies=[ + '3rdparty/python:future', + 'tests/python/pants_test:int-test', + ], + timeout=100, +) diff --git a/contrib/rust/tests/python/pants_test/contrib/rust/tasks/test_cargo_build.py b/contrib/rust/tests/python/pants_test/contrib/rust/tasks/test_cargo_build.py new file mode 100644 index 00000000000..cd56823c399 --- /dev/null +++ b/contrib/rust/tests/python/pants_test/contrib/rust/tasks/test_cargo_build.py @@ -0,0 +1,652 @@ +# coding=utf-8 +# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import, division, print_function, unicode_literals + +import collections +import copy +import os +from textwrap import dedent + +from pants.base.build_environment import get_buildroot +from pants.build_graph.address import Address +from pants.util.dirutil import safe_mkdir +from pants_test.task_test_base import TaskTestBase + +from pants.contrib.rust.targets.synthetic.cargo_synthetic_binary import CargoSyntheticBinary +from pants.contrib.rust.targets.synthetic.cargo_synthetic_custom_build import \ + CargoSyntheticCustomBuild +from pants.contrib.rust.tasks.cargo_build import Build + + +class CargoTaskBuild(TaskTestBase): + @classmethod + def task_type(cls): + return Build + + def get_test_invocation(self, name, dependencies, toolchain, build_script=False): + test_cargo_invocation = { + 'package_name': 'sha2', + 'package_version': '0.8.0', + 'target_kind': [ + 'lib' + ], + 'kind': 'Host', + 'compile_mode': 'build', + 'deps': dependencies, + 'outputs': [ + '/test_pants/contrib/rust/examples/src/rust/lib_kinds/{name}/target/debug/deps/libsha2-f6245ef4e796fdb5.rlib'.format( + name=name) + ], + 'links': { + '/test_pants/contrib/rust/examples/src/rust/lib_kinds/{name}/target/deps/libsha2-f6245ef4e796fdb5.rlib'.format( + name=name): '/test_pants/contrib/rust/examples/src/rust/lib_kinds/{name}/target/debug/deps/libsha2-f6245ef4e796fdb5.rlib'.format( + name=name) + }, + 'program': 'rustc', + 'args': [ + '--crate-name', + 'sha2', + '/root/.cargo/registry/src/github.com-1ecc6299db9ec823/sha2-0.8.0/src/lib.rs', + '--color', + 'always', + '--crate-type', + 'lib', + '--emit=dep-info,link', + '-C', + 'debuginfo=2', + '--cfg', + 'feature=\'default\'', + '--cfg', + 'feature=\'digest\'', + '--cfg', + 'feature=\'std\'', + '-C', + 'metadata=f6245ef4e796fdb5', + '-C', + 'extra-filename=-f6245ef4e796fdb5', + '--out-dir', + '/test_pants/contrib/rust/examples/src/rust/lib_kinds/{name}/target/debug/deps'.format( + name=name), + '-L', + 'dependency=/test_pants/contrib/rust/examples/src/rust/lib_kinds/{name}/target/debug/deps'.format( + name=name), + '--extern', + 'block_buffer=/test_pants/contrib/rust/examples/src/rust/lib_kinds/{name}/target/debug/deps/libblock_buffer-8efff75dcbc10d83.rlib'.format( + name=name), + '--extern', + 'digest=/test_pants/contrib/rust/examples/src/rust/lib_kinds/{name}/target/debug/deps/libdigest-5f1979ef5447de73.rlib'.format( + name=name), + '--extern', + 'fake_simd=/test_pants/contrib/rust/examples/src/rust/lib_kinds/{name}/target/debug/deps/libfake_simd-536d6e5b583fc52f.rlib'.format( + name=name), + '--extern', + 'opaque_debug=/test_pants/contrib/rust/examples/src/rust/lib_kinds/{name}/target/debug/deps/libopaque_debug-c6619a193d9a2c0c.rlib'.format( + name=name), + '--cap-lints', + 'allow' + ], + 'env': { + 'CARGO': '/root/.rustup/toolchains/nightly-2018-12-31-x86_64-unknown-linux-gnu/bin/cargo', + 'CARGO_MANIFEST_DIR': '/root/.cargo/registry/src/github.com-1ecc6299db9ec823/sha2-0.8.0', + 'CARGO_PKG_AUTHORS': 'RustCrypto Developers', + 'CARGO_PKG_DESCRIPTION': 'SHA-2 hash functions', + 'CARGO_PKG_HOMEPAGE': '', + 'CARGO_PKG_NAME': 'sha2', + 'CARGO_PKG_REPOSITORY': 'https://github.com/RustCrypto/hashes', + 'CARGO_PKG_VERSION': '0.8.0', + 'CARGO_PKG_VERSION_MAJOR': '0', + 'CARGO_PKG_VERSION_MINOR': '8', + 'CARGO_PKG_VERSION_PATCH': '0', + 'CARGO_PKG_VERSION_PRE': '', + 'OUT_DIR': '/test_pants/contrib/rust/examples/src/rust/lib_kinds/{name}/target/debug/build/output'.format( + name=name) + }, + 'cwd': os.path.join('root/.cargo/registry/src/github.com-1ecc6299db9ec823/sha2-0.8.0') + } + + if toolchain == 'nightly_linux': + test_cargo_invocation['env'].update({ + 'LD_LIBRARY_PATH': '/test_pants/contrib/rust/examples/src/rust/lib_kinds/{name}/target/debug/deps:/root/.rustup/toolchains/nightly-2018-12-31-x86_64-unknown-linux-gnu/lib:/root/.rustup/toolchains/nightly-2018-12-31-x86_64-unknown-linux-gnu/lib'.format( + name=name)}) + elif toolchain == 'nightly_2018_12_31_macos': + test_cargo_invocation['env'].update({ + 'DYLD_LIBRARY_PATH': '/test_pants/contrib/rust/examples/src/rust/lib_kinds/{name}/target/debug/deps:/root/.rustup/toolchains/nightly-2018-12-31-x86_64-unknown-linux-gnu/lib:/root/.rustup/toolchains/nightly-2018-12-31-x86_64-unknown-linux-gnu/lib'.format( + name=name)}), + elif toolchain == 'nightly_macos': + test_cargo_invocation['env'].update({ + 'DYLD_FALLBACK_LIBRARY_PATH': '/test_pants/contrib/rust/examples/src/rust/lib_kinds/{name}/target/debug/deps:/root/.rustup/toolchains/nightly-2018-12-31-x86_64-unknown-linux-gnu/lib:/root/.rustup/toolchains/nightly-2018-12-31-x86_64-unknown-linux-gnu/lib'.format( + name=name)}), + + if build_script: + test_cargo_invocation[ + 'program'] = '/test_pants/contrib/rust/examples/src/rust/lib_kinds/{name}/target/debug/build/byteorder-bf549d3401bc1e38/build-script-build'.format( + name=name) + test_cargo_invocation['compile_mode'] = 'run-custom-build' + + return copy.deepcopy(test_cargo_invocation) + + def get_cargo_statements(self): + return { + 'rustc-flags': [ + [ + '-l', + 'static=samplerate' + ], + [ + '-l', + 'dylib=stdc++' + ], + [ + '-l', + 'static=pfring', + '-L', + '/usr/local/lib' + ], + [ + '-l', + 'static=pcap', + '-L', + '/usr/local/lib' + ], + ], + 'rustc-env': [ + [ + 'PROTOC', + '/protobuf/protoc' + ], + [ + 'PROTOC_INCLUDE', + '/protobuf/include' + ] + ], + 'rustc-link-search': [ + [ + '-L', + 'native=pants/.pants.d/engine/out' + ], + ], + 'rustc-link-lib': [ + [ + '-l', + 'static=native_engine_ffi' + ], + ], + 'rustc-cfg': [ + [ + '--cfg', + 'rust_1_26' + ], + [ + '--cfg', + 'memchr_runtime_sse42' + ] + ], + 'warning': [ + '/lmdb-sys/lmdb/libraries/liblmdb/mdb.c:10033:33: warning: unused parameter [-Wunused-parameter]', + 'mdb_env_get_maxkeysize(MDB_env *env)', + ' ^', + '1 warning generated.' + ], + 'rerun-if-env-changed': [ + 'PY' + ], + 'rerun-if-changed': [ + 'cbindgen.toml' + ] + } + + def test_get_build_flags(self): + context = self.context(options={'test_scope': {'cargo_opt': ['--a', '--c', '--b']}}) + task = self.create_task(context) + + context.products.safe_create_data('cargo_toolchain', lambda: 'nightly-xyz') + + flags = task.get_build_flags() + self.assertEqual('--a --b --c nightly-xyz', flags) + + def test_get_build_flags_without_options(self): + context = self.context() + task = self.create_task(context) + + context.products.safe_create_data('cargo_toolchain', lambda: 'nightly-xyz') + + flags = task.get_build_flags() + self.assertEqual('nightly-xyz', flags) + + def test_get_build_flags_with_include_tests(self): + context = self.context(options={'test_scope': {'cargo_opt': ['--a', '--c', '--b']}}) + task = self.create_task(context) + + context.products.safe_create_data('cargo_toolchain', lambda: 'nightly-xyz') + context.products.require_data('rust_tests') + + flags = task.get_build_flags() + self.assertEqual('--a --b --c nightly-xyz --tests', flags) + + def test_create_libraries_dir(self): + task = self.create_task(self.context()) + libraries_dir = os.path.join(task.versioned_workdir, 'deps') + + task.create_libraries_dir() + + self.assertTrue(os.path.isdir(libraries_dir)) + + def test_create_build_index_dir(self): + task = self.create_task(self.context()) + build_index_dir = os.path.join(task.versioned_workdir, 'build_index') + + task.create_build_index_dir() + + self.assertTrue(os.path.isdir(build_index_dir)) + + def test_include_compiling_tests_true(self): + context = self.context() + task = self.create_task(context) + context.products.require_data('rust_tests') + + self.assertTrue(task.include_compiling_tests()) + + def test_include_compiling_tests_false(self): + task = self.create_task(self.context()) + + self.assertFalse(task.include_compiling_tests()) + + def test_parse_build_script_output(self): + build_output = dedent(""" + cargo:rustc-link-lib=static=native_engine_ffi + cargo:rustc-link-search=native=pants/.pants.d/engine/out + warning=1 warning generated. + cargo:rustc-cfg=rust_1_26 + warning=/lmdb-sys/lmdb/libraries/liblmdb/mdb.c:10033:33: warning: unused parameter [-Wunused-parameter] + warning=mdb_env_get_maxkeysize(MDB_env *env) + warning= ^ + warning=1 warning generated. + """) + + task = self.create_task(self.context()) + cargo_statements = task.parse_build_script_output(build_output) + + expect = ['cargo:rustc-link-lib=static=native_engine_ffi', + 'cargo:rustc-link-search=native=pants/.pants.d/engine/out', + 'cargo:rustc-cfg=rust_1_26'] + + self.assertEqual(expect, cargo_statements) + + def test_create_build_script_output_path(self): + task = self.create_task(self.context()) + output_path = '/Users/Home/pants/.pants.d/compile/cargo/e8acf1b202cf/lib_1/current/build/lib_1/out' + + result = task.create_build_script_output_path(output_path) + expect = '/Users/Home/pants/.pants.d/compile/cargo/e8acf1b202cf/lib_1/current/build/lib_1/output' + self.assertEqual(expect, result) + + def test_create_build_script_stderr_path(self): + task = self.create_task(self.context()) + output_path = '/Users/Home/pants/.pants.d/compile/cargo/e8acf1b202cf/lib_1/current/build/lib_1/out' + + result = task.create_build_script_stderr_path(output_path) + expect = '/Users/Home/pants/.pants.d/compile/cargo/e8acf1b202cf/lib_1/current/build/lib_1/stderr' + self.assertEqual(expect, result) + + def test_create_directories(self): + task = self.create_task(self.context()) + + dir_1 = os.path.join(get_buildroot(), + 'pants/.pants.d/compile/cargo/e8acf1b202cf/lib_1/current/build') + dir_2 = os.path.join(get_buildroot(), + 'pants/.pants.d/compile/cargo/e8acf1b202cf/lib_1/current/incremental') + mkdir = {dir_1, dir_2} + + self.assertListEqual([False] * len(mkdir), list(map(lambda dir: os.path.isdir(dir), mkdir))) + + task.create_directories(mkdir) + + self.assertListEqual([True] * len(mkdir), list(map(lambda dir: os.path.isdir(dir), mkdir))) + + def test_create_library_symlink(self): + task = self.create_task(self.context()) + + dir_1 = os.path.join(task.versioned_workdir, 'lib_1/current') + dir_2 = os.path.join(task.versioned_workdir, 'lib_2/current') + mkdir = {dir_1, dir_2} + + for dir in mkdir: + safe_mkdir(dir) + + lib_1 = os.path.join(dir_1, 'lib_1.xyz') + lib_2 = os.path.join(dir_1, 'lib_2.xyz') + + mklib = {lib_1, lib_2} + for lib in mklib: + self.create_file(lib, contents="") + + libraries_dir = os.path.join(task.versioned_workdir, 'deps') + safe_mkdir(libraries_dir) + + lib_1_sym = os.path.join(libraries_dir, 'lib_1.xyz') + lib_2_sym = os.path.join(libraries_dir, 'lib_2.xyz') + + lib_sym = {lib_1_sym, lib_2_sym} + + self.assertListEqual([False] * len(lib_sym), + list(map(lambda dir: os.path.exists(dir), lib_sym))) + + task.create_library_symlink(mklib, libraries_dir) + + self.assertListEqual([True] * len(lib_sym), list(map(lambda dir: os.path.islink(dir), lib_sym))) + + def test_create_copies(self): + task = self.create_task(self.context()) + + dir_1 = os.path.join(task.versioned_workdir, 'lib_1/current') + dir_1_1 = os.path.join(task.versioned_workdir, 'lib_1/current/deps') + dir_1_1_1 = os.path.join(task.versioned_workdir, 'lib_1/current/deps/lib_1') + dir_2 = os.path.join(task.versioned_workdir, 'lib_2/current') + dir_2_1 = os.path.join(task.versioned_workdir, 'lib_2/current/deps') + dir_2_1_1 = os.path.join(task.versioned_workdir, 'lib_2/current/deps/lib_2') + mkdir = {dir_1_1_1, dir_2_1} + + for dir in mkdir: + safe_mkdir(dir) + + lib_1 = os.path.join(dir_1_1, 'lib_1.dylib') + lib_1_dir = os.path.join(dir_1_1_1, 'lib_1.dylib') + lib_2 = os.path.join(dir_2_1, 'lib_2.dylib') + + mklib = {lib_1, lib_1_dir} + for lib in mklib: + self.create_file(lib, contents="") + + links = {os.path.join(dir_1, 'lib_1.dylib'): lib_1, + os.path.join(dir_1, 'lib_1'): dir_1_1_1, + os.path.join(dir_2, 'lib_2.dylib'): lib_2, + os.path.join(dir_2, 'lib_2'): dir_2_1_1, } + + self.assertFalse(os.path.exists(os.path.join(dir_1, 'lib_1.dylib'))) + self.assertFalse(os.path.exists(os.path.join(dir_1, 'lib_1'))) + self.assertFalse(os.path.exists(os.path.join(dir_2, 'lib_2.dylib'))) + self.assertFalse(os.path.exists(os.path.join(dir_2, 'lib_2'))) + + task.create_copies(links) + + self.assertTrue(os.path.isfile(os.path.join(dir_1, 'lib_1.dylib'))) + self.assertTrue(os.path.isdir(os.path.join(dir_1, 'lib_1'))) + self.assertTrue(os.path.isfile(os.path.join(dir_1, 'lib_1/lib_1.dylib'))) + self.assertFalse(os.path.exists(os.path.join(dir_2, 'lib_2.dylib'))) + self.assertFalse(os.path.exists(os.path.join(dir_2, 'lib_2'))) + + def test_get_build_index_file_path(self): + task = self.create_task(self.context()) + build_index_dir = os.path.join(task.versioned_workdir, 'build_index') + task._build_index_dir_path = build_index_dir + build_index_file_path = os.path.join(build_index_dir, 'index.json') + + self.assertEqual(build_index_file_path, task.get_build_index_file_path()) + + def test_calculate_target_definition_fingerprint_mode_build_nightly_2018_12_31_macos(self): + build_invocation_target_1 = self.get_test_invocation('cdylib', [2, 4, 5], + 'nightly_2018_12_31_macos') + build_invocation_target_2 = self.get_test_invocation('dylib', [1, 2, 3], + 'nightly_2018_12_31_macos') + + task = self.create_task(self.context()) + self.assertEqual(task.calculate_target_definition_fingerprint(build_invocation_target_1), + task.calculate_target_definition_fingerprint(build_invocation_target_2)) + + def test_calculate_target_definition_fingerprint_mode_build_nightly_macos(self): + build_invocation_target_1 = self.get_test_invocation('cdylib', [2, 4, 5], 'nightly_macos') + build_invocation_target_2 = self.get_test_invocation('dylib', [1, 2, 3], 'nightly_macos') + + task = self.create_task(self.context()) + self.assertEqual(task.calculate_target_definition_fingerprint(build_invocation_target_1), + task.calculate_target_definition_fingerprint(build_invocation_target_2)) + + def test_calculate_target_definition_fingerprint_mode_build_nightly_linux(self): + build_invocation_target_1 = self.get_test_invocation('cdylib', [2, 4, 5], 'nightly_linux') + build_invocation_target_2 = self.get_test_invocation('dylib', [1, 2, 3], 'nightly_linux') + + task = self.create_task(self.context()) + self.assertEqual(task.calculate_target_definition_fingerprint(build_invocation_target_1), + task.calculate_target_definition_fingerprint(build_invocation_target_2)) + + def test_calculate_target_definition_fingerprint_mode_run_custom_build(self): + build_invocation_target_1 = self.get_test_invocation('cdylib', [2, 4, 5], + 'nightly_2018_12_31_macos', + build_script=True) + build_invocation_target_2 = self.get_test_invocation('dylib', [1, 2, 3], + 'nightly_2018_12_31_macos', + build_script=True) + + task = self.create_task(self.context()) + self.assertEqual(task.calculate_target_definition_fingerprint(build_invocation_target_1), + task.calculate_target_definition_fingerprint(build_invocation_target_2)) + + def test_save_and_parse_build_script_output(self): + task = self.create_task(self.context()) + + t1 = self.make_target(spec='test', cargo_invocation={}, target_type=CargoSyntheticCustomBuild) + + build_output = dedent(""" + cargo:rustc-link-lib=static=native_engine_ffi + cargo:rustc-link-search=native=pants/.pants.d/engine/out + warning=1 warning generated. + cargo:rustc-cfg=rust_1_26 + warning=/lmdb-sys/lmdb/libraries/liblmdb/mdb.c:10033:33: warning: unused parameter [-Wunused-parameter] + warning=mdb_env_get_maxkeysize(MDB_env *env) + warning= ^ + warning=1 warning generated. + """) + + out_dir = os.path.join(task.versioned_workdir, 'test/current/build/lib_1/out') + + cargo_statements = task.save_and_parse_build_script_output(build_output, out_dir, t1) + result = { + 'rustc-flags': [], + 'rustc-env': [], + 'rustc-link-search': [ + [ + '-L', + 'native=pants/.pants.d/engine/out' + ], + ], + 'rustc-link-lib': [ + [ + '-l', + 'static=native_engine_ffi' + ], + ], + 'rustc-cfg': [ + [ + '--cfg', + 'rust_1_26' + ] + ], + 'warning': [], + 'rerun-if-changed': [], + 'rerun-if-env-changed': [] + } + self.assertTrue( + os.path.isfile(os.path.join(task.versioned_workdir, 'test/current/build/lib_1/output'))) + self.assertEqual(result, cargo_statements) + + def test_extend_args_with_cargo_statement_without_build_script(self): + task = self.create_task(self.context()) + + t1 = self.make_target(spec='test', cargo_invocation={}, target_type=CargoSyntheticBinary) + + cmd = ['rustc', '--crate-name', 'test', '--release'] + + extend_cmd = task.extend_args_with_cargo_statement(cmd, t1) + + self.assertEqual(cmd, extend_cmd) + + def test_extend_args_with_cargo_statement_with_build_script(self): + task = self.create_task(self.context()) + + t1 = self.make_target(spec='test', cargo_invocation={}, target_type=CargoSyntheticCustomBuild) + t2 = self.make_target(spec='test_2', cargo_invocation={}, target_type=CargoSyntheticBinary, + dependencies=[t1]) + + task._build_script_output_cache['test'] = self.get_cargo_statements() + + cmd = ['rustc', '--crate-name', 'test', '--release'] + + extend_cmd = task.extend_args_with_cargo_statement(cmd, t2) + + result = ['rustc', '--crate-name', 'test', '--release', + '-l', + 'static=native_engine_ffi', + '-L', + 'native=pants/.pants.d/engine/out', + '-l', + 'static=samplerate', + '-l', + 'dylib=stdc++', + '-l', + 'static=pfring', + '-L', + '/usr/local/lib', + '-l', + 'static=pcap', + '-L', + '/usr/local/lib', + '--cfg', + 'rust_1_26', + '--cfg', + 'memchr_runtime_sse42'] + + self.assertEqual(result, extend_cmd) + + def test_create_command(self): + task = self.create_task(self.context()) + t1 = self.make_target(spec='test', cargo_invocation={}, target_type=CargoSyntheticBinary) + + cmd = task.create_command('rustc', ['--crate-name', 'test', '--release'], t1) + + self.assertEqual(['rustc', '--crate-name', 'test', '--release'], cmd) + + def test_create_env(self): + context = self.context() + task = self.create_task(context) + + cargo_home = os.path.join(task.versioned_workdir, 'cargo_home') + env = os.environ.copy() + cargo_path = os.path.join(env['HOME'], '.cargo/bin') + context.products.safe_create_data('cargo_env', + lambda: {'CARGO_HOME': cargo_home, 'PATH': cargo_path}) + cargo_toolchain = 'nightly' + context.products.safe_create_data('cargo_toolchain', lambda: cargo_toolchain) + + t1 = self.make_target(spec='test', cargo_invocation={}, target_type=CargoSyntheticCustomBuild) + t2 = self.make_target(spec='test_2', cargo_invocation={}, target_type=CargoSyntheticBinary, + dependencies=[t1]) + + task._build_script_output_cache['test'] = self.get_cargo_statements() + + invocation_env = { + 'CARGO': '/root/.rustup/toolchains/nightly-2018-12-31-x86_64-unknown-linux-gnu/bin/cargo', + 'CARGO_MANIFEST_DIR': '/root/.cargo/registry/src/github.com-1ecc6299db9ec823/sha2-0.8.0', + 'CARGO_PKG_AUTHORS': 'RustCrypto Developers', + 'CARGO_PKG_DESCRIPTION': 'SHA-2 hash functions', + 'CARGO_PKG_HOMEPAGE': '', + 'CARGO_PKG_NAME': 'sha2', + 'CARGO_PKG_REPOSITORY': 'https://github.com/RustCrypto/hashes', + 'CARGO_PKG_VERSION': '0.8.0', + 'CARGO_PKG_VERSION_MAJOR': '0', + 'CARGO_PKG_VERSION_MINOR': '8', + 'CARGO_PKG_VERSION_PATCH': '0', + 'CARGO_PKG_VERSION_PRE': '' + } + + target_env = task.create_env(invocation_env, t2) + + result = { + 'CARGO': ( + '/root/.rustup/toolchains/nightly-2018-12-31-x86_64-unknown-linux-gnu/bin/cargo', False), + 'CARGO_MANIFEST_DIR': ( + '/root/.cargo/registry/src/github.com-1ecc6299db9ec823/sha2-0.8.0', False), + 'CARGO_PKG_AUTHORS': ('RustCrypto Developers', False), + 'CARGO_PKG_DESCRIPTION': ('SHA-2 hash functions', False), + 'CARGO_PKG_HOMEPAGE': ('', False), + 'CARGO_PKG_NAME': ('sha2', False), + 'CARGO_PKG_REPOSITORY': ('https://github.com/RustCrypto/hashes', False), + 'CARGO_PKG_VERSION': ('0.8.0', False), + 'CARGO_PKG_VERSION_MAJOR': ('0', False), + 'CARGO_PKG_VERSION_MINOR': ('8', False), + 'CARGO_PKG_VERSION_PATCH': ('0', False), + 'CARGO_PKG_VERSION_PRE': ('', False), + 'PATH': (cargo_path, True), + 'RUSTUP_TOOLCHAIN': (cargo_toolchain, False), + 'PROTOC': ('/protobuf/protoc', False), + 'PROTOC_INCLUDE': ('/protobuf/include', False) + } + + self.assertEqual(result, target_env) + + def test_extend_env_with_cargo_statement_without_cargo_statements(self): + task = self.create_task(self.context()) + + t1 = self.make_target(spec='test', cargo_invocation={}, target_type=CargoSyntheticBinary) + + extend_env = task.extend_env_with_cargo_statement({}, t1) + + self.assertEqual({}, extend_env) + + def test_extend_env_with_cargo_statement_with_cargo_statements(self): + task = self.create_task(self.context()) + + t1 = self.make_target(spec='test', cargo_invocation={}, target_type=CargoSyntheticCustomBuild) + t2 = self.make_target(spec='test_2', cargo_invocation={}, target_type=CargoSyntheticBinary, + dependencies=[t1]) + + task._build_script_output_cache['test'] = self.get_cargo_statements() + + extend_env = task.extend_env_with_cargo_statement({}, t2) + + result = {'PROTOC': ('/protobuf/protoc', False), + 'PROTOC_INCLUDE': ('/protobuf/include', False)} + + self.assertEqual(result, extend_env) + + def test_create_target_definition(self): + TargetDefinition = collections.namedtuple('TargetDefinition', + 'name, id, address, ' + 'dependencies, kind, ' + 'invocation, compile_mode') + + task = self.create_task(self.context()) + cargo_invocation = self.get_test_invocation('test', [3], 'nightly_linux') + cwd = os.path.join(get_buildroot(), 'test') + cargo_invocation['cwd'] = cwd + + copy_cargo_invocation = self.get_test_invocation('test', [3], 'nightly_linux') + copy_cargo_invocation['cwd'] = cwd + copy_cargo_invocation.pop('deps') + + fingerprint = 'sha2_{0}'.format(task.calculate_target_definition_fingerprint(cargo_invocation)) + result = TargetDefinition('sha2', fingerprint, + Address(os.path.relpath(cwd, get_buildroot()), fingerprint), [3], + 'lib', copy_cargo_invocation, 'build') + + td = task.create_target_definition(cargo_invocation) + self.assertEqual(result, td) + +# def prepare_task(self): +# def prepare_cargo_targets(self): +# def get_cargo_build_plan(self, target): +# def get_target_definitions_out_of_cargo_build_plan(self, cargo_build_plan): +# def generate_targets(self, target_definitions, cargo_target): +# def is_lib_or_bin(self, target_definition, original_target): +# def inject_lib_or_bin_target(self, target_definition, original_target): +# def build_targets(self, targets): +# def convert_cargo_invocation_into_pants_invocation(self, vt): +# def build_target(self, target, pants_invocation): +# def execute_custom_build(self, cmd, workunit_name, env, cwd, target): +# def execute_rustc(self, package_name, cmd, workunit_name, env, cwd): +# def add_rust_products(self, target, pants_invocation): +# def mark_target_invalid(self, address): +# def check_if_build_scripts_are_invalid(self): +# def read_build_index(self): +# def write_build_index(self): diff --git a/contrib/rust/tests/python/pants_test/contrib/rust/tasks/test_cargo_build_integration.py b/contrib/rust/tests/python/pants_test/contrib/rust/tasks/test_cargo_build_integration.py new file mode 100644 index 00000000000..348489c6e14 --- /dev/null +++ b/contrib/rust/tests/python/pants_test/contrib/rust/tasks/test_cargo_build_integration.py @@ -0,0 +1,23 @@ +# coding=utf-8 +# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import, division, print_function, unicode_literals + +from pants_test.pants_run_integration_test import PantsRunIntegrationTest + + +class CargoTaskBuildIntegrationTest(PantsRunIntegrationTest): + + def test_cargo_compile_targets_nightly(self): + args = ['compile', 'contrib/rust/examples/src/rust/::', '--bootstrap-cargo-toolchain=nightly'] + pants_run = self.run_pants(args) + self.assert_success(pants_run) + + def test_cargo_compile_targets_nightly_2018_12_31(self): + args = [ + 'compile', 'contrib/rust/examples/src/rust/::', + '--bootstrap-cargo-toolchain=nightly-2018-12-31' + ] + pants_run = self.run_pants(args) + self.assert_success(pants_run) diff --git a/contrib/rust/tests/python/pants_test/contrib/rust/tasks/test_cargo_fetch.py b/contrib/rust/tests/python/pants_test/contrib/rust/tasks/test_cargo_fetch.py new file mode 100644 index 00000000000..87c7b8939ba --- /dev/null +++ b/contrib/rust/tests/python/pants_test/contrib/rust/tasks/test_cargo_fetch.py @@ -0,0 +1,141 @@ +# coding=utf-8 +# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import, division, print_function, unicode_literals + +import os +from textwrap import dedent + +from pants.base.build_environment import get_buildroot +from pants.base.exceptions import TaskError +from pants.util.dirutil import safe_mkdir +from pants_test.task_test_base import TaskTestBase + +from pants.contrib.rust.targets.original.cargo_library import CargoLibrary +from pants.contrib.rust.tasks.cargo_fetch import Fetch + + +class CargoTaskFetch(TaskTestBase): + + @classmethod + def task_type(cls): + return Fetch + + def test_create_cargo_home(self): + task = self.create_task(self.context()) + cargo_home_test = task.create_cargo_home() + + cargo_home = os.path.join(task.versioned_workdir, 'cargo_home') + self.assertEqual(cargo_home, cargo_home_test) + self.assertTrue(os.path.isdir(cargo_home)) + + def test_set_cargo_home(self): + context = self.context() + context.products.safe_create_data('cargo_env', lambda: {}) + task = self.create_task(context) + cargo_home = os.path.join(task.versioned_workdir, 'cargo_home') + task.set_cargo_home(cargo_home) + self.assertEqual(cargo_home, context.products.get_data('cargo_env')['CARGO_HOME']) + + def create_cargo_lib_project(self, name): + project_path = os.path.join(get_buildroot(), name) + safe_mkdir(project_path) + + src_path = os.path.join(project_path, 'src') + safe_mkdir(src_path) + + self.create_file( + os.path.join(name, 'BUILD'), + contents=dedent(""" + # Copyright 2018 Pants project contributors (see CONTRIBUTORS.md). + # Licensed under the Apache License, Version 2.0 (see LICENSE). + + cargo_library( + name='{name}', + sources=rglobs('*.rs', 'Cargo.toml', exclude=[rglobs('**/target/*.rs')]), + ) + """).format(name=name).strip()) + + self.create_file( + os.path.join(name, 'Cargo.toml'), + contents=dedent(""" + [package] + name = "{name}" + version = "0.1.0" + authors = ["Pants Build "] + edition = "2018" + + [dependencies] + sha2 = "0.8" + """).format(name=name).strip()) + + self.create_file( + os.path.join(name, 'src', 'lib.rs'), + contents=dedent(""" + #[cfg(test)] + mod tests { + #[test] + fn it_works() { + assert_eq!(2 + 2, 4); + } + #[test] + fn it_works_2() { + assert_eq!(2 + 1 + 1, 4); + } + } + """).strip()) + + with open(os.path.join(project_path, 'Cargo.toml'), 'r') as fp: + print(fp.read()) + + def test_fetch(self): + project_name = 'test' + self.create_cargo_lib_project(project_name) + t1 = self.make_target(spec=project_name, target_type=CargoLibrary) + + context = self.context() + task = self.create_task(context) + + cargo_home = os.path.join(task.versioned_workdir, 'cargo_home') + safe_mkdir(cargo_home) + + env = os.environ.copy() + cargo_path = os.path.join(env['HOME'], '.cargo/bin') + context.products.safe_create_data('cargo_env', lambda: { + 'CARGO_HOME': cargo_home, + 'PATH': cargo_path + }) + context.products.safe_create_data('cargo_toolchain', lambda: 'nightly-2018-12-31') + + cargo_git = os.path.join(cargo_home, 'git') + cargo_registry = os.path.join(cargo_home, 'registry') + + self.assertFalse(os.path.isdir(cargo_git)) + self.assertFalse(os.path.isdir(cargo_registry)) + + task.fetch(t1) + + self.assertTrue(os.path.isdir(cargo_registry)) + + def test_fetch_failure(self): + project_name = 'test' + self.create_cargo_lib_project(project_name) + t1 = self.make_target(spec=project_name, target_type=CargoLibrary) + + context = self.context() + task = self.create_task(context) + + cargo_home = os.path.join(task.versioned_workdir, 'cargo_home') + safe_mkdir(cargo_home) + + env = os.environ.copy() + cargo_path = os.path.join(env['HOME'], '.cargo/bin') + context.products.safe_create_data('cargo_env', lambda: { + 'CARGO_HOME': cargo_home, + 'PATH': cargo_path + }) + context.products.safe_create_data('cargo_toolchain', lambda: 'XYZ') + + with self.assertRaises(TaskError): + task.fetch(t1) diff --git a/contrib/rust/tests/python/pants_test/contrib/rust/tasks/test_cargo_fetch_integration.py b/contrib/rust/tests/python/pants_test/contrib/rust/tasks/test_cargo_fetch_integration.py new file mode 100644 index 00000000000..c9bc71bf491 --- /dev/null +++ b/contrib/rust/tests/python/pants_test/contrib/rust/tasks/test_cargo_fetch_integration.py @@ -0,0 +1,26 @@ +# coding=utf-8 +# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import, division, print_function, unicode_literals + +from pants_test.pants_run_integration_test import PantsRunIntegrationTest + + +class CargoTaskFetchIntegrationTest(PantsRunIntegrationTest): + + def test_cargo_fetch_targets_nightly(self): + args = [ + 'resolve', 'contrib/rust/examples/src/rust/dependencies/workspace_a:workspace_a', + '--bootstrap-cargo-toolchain=nightly' + ] + pants_run = self.run_pants(args) + self.assert_success(pants_run) + + def test_cargo_fetch_targets_nightly_2018_12_31(self): + args = [ + 'resolve', 'contrib/rust/examples/src/rust/dependencies/workspace_a:workspace_a', + '--bootstrap-cargo-toolchain=nightly-2018-12-31' + ] + pants_run = self.run_pants(args) + self.assert_success(pants_run) diff --git a/contrib/rust/tests/python/pants_test/contrib/rust/tasks/test_cargo_task.py b/contrib/rust/tests/python/pants_test/contrib/rust/tasks/test_cargo_task.py new file mode 100644 index 00000000000..985ff32462d --- /dev/null +++ b/contrib/rust/tests/python/pants_test/contrib/rust/tasks/test_cargo_task.py @@ -0,0 +1,336 @@ +# coding=utf-8 +# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import, division, print_function, unicode_literals + +import os +import string +from textwrap import dedent + +from pants.base.workunit import WorkUnitLabel +from pants.build_graph.target import Target +from pants.util.contextutil import temporary_dir +from pants.util.dirutil import chmod_plus_x +from pants_test.task_test_base import TaskTestBase + +from pants.contrib.rust.targets.original.cargo_base import CargoBase +from pants.contrib.rust.targets.original.cargo_binary import CargoBinary +from pants.contrib.rust.targets.original.cargo_library import CargoLibrary +from pants.contrib.rust.targets.synthetic.cargo_project_binary import CargoProjectBinary +from pants.contrib.rust.targets.synthetic.cargo_project_library import CargoProjectLibrary +from pants.contrib.rust.targets.synthetic.cargo_project_test import CargoProjectTest +from pants.contrib.rust.targets.synthetic.cargo_synthetic_base import CargoSyntheticBase +from pants.contrib.rust.targets.synthetic.cargo_synthetic_binary import CargoSyntheticBinary +from pants.contrib.rust.targets.synthetic.cargo_synthetic_custom_build import \ + CargoSyntheticCustomBuild +from pants.contrib.rust.targets.synthetic.cargo_synthetic_library import CargoSyntheticLibrary +from pants.contrib.rust.tasks.cargo_task import CargoTask + + +test_shell_script = dedent(""" + #!/bin/bash + + if [ "$EXECUTE_COMMAND_TEST" = "{execute_command_test_var}" ]; then + echo success + exit 0 + else + >&2 echo error + exit 1 + fi + """) + + +class CargoTaskTest(TaskTestBase): + + class TestCargoTask(CargoTask): + + def execute(self): + raise NotImplementedError() + + @classmethod + def task_type(cls): + return cls.TestCargoTask + + def test_is_cargo_original(self): + expected = { + CargoBase: True, + CargoBinary: True, + CargoLibrary: True, + # CargoWorkspace: True, + CargoProjectBinary: False, + CargoProjectLibrary: False, + CargoProjectTest: False, + CargoSyntheticBase: False, + CargoSyntheticBinary: False, + CargoSyntheticCustomBuild: False, + CargoSyntheticLibrary: False, + Target: False, + } + self.assertEqual(expected, self._type_check(expected.keys(), CargoTask.is_cargo_original)) + + def test_is_cargo_original_binary(self): + expected = { + CargoBase: False, + CargoBinary: True, + CargoLibrary: False, + # CargoWorkspace: False, + CargoProjectBinary: False, + CargoProjectLibrary: False, + CargoProjectTest: False, + CargoSyntheticBase: False, + CargoSyntheticBinary: False, + CargoSyntheticCustomBuild: False, + CargoSyntheticLibrary: False, + Target: False, + } + self.assertEqual(expected, self._type_check(expected.keys(), + CargoTask.is_cargo_original_binary)) + + def test_is_cargo_original_library(self): + expected = { + CargoBase: False, + CargoBinary: False, + CargoLibrary: True, + # CargoWorkspace: False, + CargoProjectBinary: False, + CargoProjectLibrary: False, + CargoProjectTest: False, + CargoSyntheticBase: False, + CargoSyntheticBinary: False, + CargoSyntheticCustomBuild: False, + CargoSyntheticLibrary: False, + Target: False, + } + self.assertEqual(expected, self._type_check(expected.keys(), + CargoTask.is_cargo_original_library)) + + def test_is_cargo_original_workspace(self): + expected = { + CargoBase: False, + CargoBinary: False, + CargoLibrary: False, + # CargoWorkspace: True, + CargoProjectBinary: False, + CargoProjectLibrary: False, + CargoProjectTest: False, + CargoSyntheticBase: False, + CargoSyntheticBinary: False, + CargoSyntheticCustomBuild: False, + CargoSyntheticLibrary: False, + Target: False, + } + self.assertEqual(expected, + self._type_check(expected.keys(), CargoTask.is_cargo_original_workspace)) + + def test_is_cargo_synthetic(self): + expected = { + CargoBase: False, + CargoBinary: False, + CargoLibrary: False, + # CargoWorkspace: False, + CargoProjectBinary: True, + CargoProjectLibrary: True, + CargoProjectTest: True, + CargoSyntheticBase: True, + CargoSyntheticBinary: True, + CargoSyntheticCustomBuild: True, + CargoSyntheticLibrary: True, + Target: False, + } + self.assertEqual(expected, self._type_check(expected.keys(), CargoTask.is_cargo_synthetic)) + + def test_is_cargo_synthetic_library(self): + expected = { + CargoBase: False, + CargoBinary: False, + CargoLibrary: False, + # CargoWorkspace: False, + CargoProjectBinary: False, + CargoProjectLibrary: True, + CargoProjectTest: False, + CargoSyntheticBase: False, + CargoSyntheticBinary: False, + CargoSyntheticCustomBuild: False, + CargoSyntheticLibrary: True, + Target: False, + } + self.assertEqual(expected, + self._type_check(expected.keys(), CargoTask.is_cargo_synthetic_library)) + + def test_is_cargo_synthetic_binary(self): + expected = { + CargoBase: False, + CargoBinary: False, + CargoLibrary: False, + # CargoWorkspace: False, + CargoProjectBinary: True, + CargoProjectLibrary: False, + CargoProjectTest: False, + CargoSyntheticBase: False, + CargoSyntheticBinary: True, + CargoSyntheticCustomBuild: False, + CargoSyntheticLibrary: False, + Target: False, + } + self.assertEqual(expected, self._type_check(expected.keys(), + CargoTask.is_cargo_synthetic_binary)) + + def test_is_cargo_synthetic_custom_build(self): + expected = { + CargoBase: False, + CargoBinary: False, + CargoLibrary: False, + # CargoWorkspace: False, + CargoProjectBinary: False, + CargoProjectLibrary: False, + CargoProjectTest: False, + CargoSyntheticBase: False, + CargoSyntheticBinary: False, + CargoSyntheticCustomBuild: True, + CargoSyntheticLibrary: False, + Target: False, + } + self.assertEqual(expected, + self._type_check(expected.keys(), CargoTask.is_cargo_synthetic_custom_build)) + + def test_is_cargo_project_binary(self): + expected = { + CargoBase: False, + CargoBinary: False, + CargoLibrary: False, + # CargoWorkspace: False, + CargoProjectBinary: True, + CargoProjectLibrary: False, + CargoProjectTest: False, + CargoSyntheticBase: False, + CargoSyntheticBinary: False, + CargoSyntheticCustomBuild: False, + CargoSyntheticLibrary: False, + Target: False, + } + self.assertEqual(expected, self._type_check(expected.keys(), CargoTask.is_cargo_project_binary)) + + def test_is_cargo_project_library(self): + expected = { + CargoBase: False, + CargoBinary: False, + CargoLibrary: False, + # CargoWorkspace: False, + CargoProjectBinary: False, + CargoProjectLibrary: True, + CargoProjectTest: False, + CargoSyntheticBase: False, + CargoSyntheticBinary: False, + CargoSyntheticCustomBuild: False, + CargoSyntheticLibrary: False, + Target: False, + } + self.assertEqual(expected, self._type_check(expected.keys(), + CargoTask.is_cargo_project_library)) + + def test_is_cargo_project_test(self): + expected = { + CargoBase: False, + CargoBinary: False, + CargoLibrary: False, + # CargoWorkspace: False, + CargoProjectBinary: False, + CargoProjectLibrary: False, + CargoProjectTest: True, + CargoSyntheticBase: False, + CargoSyntheticBinary: False, + CargoSyntheticCustomBuild: False, + CargoSyntheticLibrary: False, + Target: False, + } + self.assertEqual(expected, self._type_check(expected.keys(), CargoTask.is_cargo_project_test)) + + def _type_check(self, types, type_check_function): + # Make sure the diff display length is long enough for the test_is_* tests. + # It's a little weird to include this side effect here, but otherwise it would have to + # be duplicated or go in the setup (in which case it would affect all tests). + self.maxDiff = None + + target_names = [':' + letter for letter in list(string.ascii_lowercase)] + types_with_target_names = zip(types, target_names) + + type_check_results = { + type: type_check_function(self.make_target(target_name, type)) + for type, target_name in types_with_target_names + } + + return type_check_results + + def test_execute_command_success(self): + task = self.create_task(self.context()) + with temporary_dir() as chroot: + script = os.path.join(chroot, 'test.sh') + execute_command_test_var = 'Test' + with open(script, 'w') as fp: + fp.write( + test_shell_script.format(execute_command_test_var=execute_command_test_var).strip()) + + chmod_plus_x(script) + returncode = task.execute_command( + [script], + 'test', [WorkUnitLabel.TEST], + env_vars={'EXECUTE_COMMAND_TEST': (execute_command_test_var, False)}, + current_working_dir=chroot) + + self.assertEqual(0, returncode) + + def test_execute_command_failure(self): + task = self.create_task(self.context()) + with temporary_dir() as chroot: + script = os.path.join(chroot, 'test.sh') + execute_command_test_var = 'Test' + with open(script, 'w') as fp: + fp.write( + test_shell_script.format(execute_command_test_var=execute_command_test_var).strip()) + + chmod_plus_x(script) + returncode = task.execute_command([script], + 'test', [WorkUnitLabel.TEST], + current_working_dir=chroot) + + self.assertEqual(1, returncode) + + def test_execute_command_and_get_output_success(self): + task = self.create_task(self.context()) + with temporary_dir() as chroot: + script = os.path.join(chroot, 'test.sh') + execute_command_test_var = 'Test' + with open(script, 'w') as fp: + fp.write( + test_shell_script.format(execute_command_test_var=execute_command_test_var).strip()) + + chmod_plus_x(script) + returncode, std_out, std_err = task.execute_command_and_get_output( + [script], + 'test', [WorkUnitLabel.TEST], + env_vars={'EXECUTE_COMMAND_TEST': (execute_command_test_var, False)}, + current_working_dir=chroot) + + self.assertEqual(0, returncode) + self.assertEqual("success\n", std_out) + self.assertEqual("", std_err) + + def test_execute_command_and_get_output_failure(self): + task = self.create_task(self.context()) + with temporary_dir() as chroot: + script = os.path.join(chroot, 'test.sh') + execute_command_test_var = 'Test' + with open(script, 'w') as fp: + fp.write( + test_shell_script.format(execute_command_test_var=execute_command_test_var).strip()) + + chmod_plus_x(script) + returncode, std_out, std_err = task.execute_command_and_get_output([script], + 'test', + [WorkUnitLabel.TEST], + current_working_dir=chroot) + + self.assertEqual(1, returncode) + self.assertEqual("", std_out) + self.assertEqual("error\n", std_err) diff --git a/contrib/rust/tests/python/pants_test/contrib/rust/tasks/test_cargo_toolchain.py b/contrib/rust/tests/python/pants_test/contrib/rust/tasks/test_cargo_toolchain.py new file mode 100644 index 00000000000..7bf6e65ad06 --- /dev/null +++ b/contrib/rust/tests/python/pants_test/contrib/rust/tasks/test_cargo_toolchain.py @@ -0,0 +1,74 @@ +# coding=utf-8 +# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import, division, print_function, unicode_literals + +import os + +from pants.base.build_environment import get_buildroot +from pants.base.exceptions import TaskError +from pants.util.contextutil import temporary_dir +from pants_test.task_test_base import TaskTestBase + +from pants.contrib.rust.tasks.cargo_toolchain import Toolchain + + +class CargoTaskToolchain(TaskTestBase): + + @classmethod + def task_type(cls): + return Toolchain + + def test_download_rustup_install_script(self): + task = self.create_task(self.context()) + install_script = task.download_rustup_install_script() + self.assertNotEqual("", install_script.text) + + def test_check_integrity_of_rustup_install_script_failure(self): + task = self.create_task(self.context()) + with self.assertRaises(TaskError): + task.check_integrity_of_rustup_install_script('test') + + def test_set_cargo_path(self): + context = self.context() + task = self.create_task(context) + task.set_cargo_path() + env = os.environ.copy() + result = os.path.join(env['HOME'], '.cargo/bin') + self.assertEqual(result, context.products.get_data('cargo_env')['PATH']) + + def test_save_rustup_install_script(self): + task = self.create_task(self.context()) + task.save_rustup_install_script('test') + script_path = os.path.join(task.versioned_workdir, 'rustup_install_script', 'rustup.sh') + self.assertTrue(os.path.isfile(script_path)) + with open(script_path, 'r') as fp: + script_content = fp.read() + self.assertEqual('test', script_content) + + def test_get_toolchain(self): + task = self.create_task( + self.context(options={'test_scope': { + 'toolchain': 'nightly-toolchain' + }})) + toolchain = task.get_toolchain() + self.assertEqual('nightly-toolchain', toolchain) + + def test_get_toolchain_file(self): + with temporary_dir(root_dir=get_buildroot()) as chroot: + rust_toolchain = os.path.join(chroot, 'rust-toolchain') + with open(rust_toolchain, 'w') as fp: + fp.write('nightly-toolchain') + + dir_base = os.path.basename(chroot) + task = self.create_task(self.context(options={'test_scope': {'toolchain': dir_base}})) + toolchain = task.get_toolchain() + self.assertEqual('nightly-toolchain', toolchain) + + # def setup_rustup(self): + # def install_rust_toolchain(self, toolchain): + # def check_if_rustup_exist(self): + # def try_to_find_rustup_executable(self): + # def try_to_find_rustup_executable_in_default_location(self): + # def setup_toolchain(self): diff --git a/contrib/rust/tests/python/pants_test/contrib/rust/tasks/test_cargo_workspace.py b/contrib/rust/tests/python/pants_test/contrib/rust/tasks/test_cargo_workspace.py new file mode 100644 index 00000000000..485efe9389b --- /dev/null +++ b/contrib/rust/tests/python/pants_test/contrib/rust/tasks/test_cargo_workspace.py @@ -0,0 +1,95 @@ +# coding=utf-8 +# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import, division, print_function, unicode_literals + +import collections + +from pants_test.task_test_base import TaskTestBase + +from pants.contrib.rust.tasks.cargo_workspace import Workspace + + +class CargoTaskWorkspace(TaskTestBase): + + class TestCargoWorkspace(Workspace): + + def execute(self): + raise NotImplementedError() + + @classmethod + def task_type(cls): + return cls.TestCargoWorkspace + + def test_is_target_a_member_true(self): + task = self.create_task(self.context()) + target_name = 'a' + member_names = ['a', 'b', 'c'] + + self.assertTrue(task.is_target_a_member(target_name, member_names)) + + def test_is_target_a_member_false(self): + task = self.create_task(self.context()) + target_name = 'd' + member_names = ['a', 'b', 'c'] + + self.assertFalse(task.is_target_a_member(target_name, member_names)) + + def test_is_lib_or_bin_target_true(self): + TargetDefinition = collections.namedtuple('TargetDefinition', 'kind') + targets = [ + TargetDefinition('bin'), + TargetDefinition('lib'), + TargetDefinition('cdylib'), + TargetDefinition('rlib'), + TargetDefinition('dylib'), + TargetDefinition('staticlib'), + TargetDefinition('proc-macro') + ] + + task = self.create_task(self.context()) + + result = list(map(lambda target: task.is_lib_or_bin_target(target), targets)) + self.assertListEqual([True] * len(targets), result) + + def test_is_lib_or_bin_target_false(self): + TargetDefinition = collections.namedtuple('TargetDefinition', 'kind') + target = TargetDefinition('test') + task = self.create_task(self.context()) + self.assertFalse(task.is_lib_or_bin_target(target)) + + def test_is_test_target_true(self): + TargetDefinition = collections.namedtuple('TargetDefinition', 'kind, compile_mode') + targets = [ + TargetDefinition('test', '_'), + TargetDefinition('test', 'test'), + TargetDefinition('_', 'test') + ] + + task = self.create_task(self.context()) + result = list(map(lambda target: task.is_test_target(target), targets)) + self.assertListEqual([True] * len(targets), result) + + def test_is_test_target_false(self): + TargetDefinition = collections.namedtuple('TargetDefinition', 'kind, compile_mode') + targets = [ + TargetDefinition('bin', 'build'), + TargetDefinition('lib', 'build'), + TargetDefinition('cdylib', 'build'), + TargetDefinition('rlib', 'build'), + TargetDefinition('rlib', 'build'), + TargetDefinition('dylib', 'build'), + TargetDefinition('staticlib', 'build'), + TargetDefinition('proc-macro', 'build') + ] + + task = self.create_task(self.context()) + result = list(map(lambda target: task.is_test_target(target), targets)) + self.assertListEqual([False] * len(targets), result) + + +# def is_workspace_member(self, target_definition, workspace_target): +# def inject_member_target(self, target_definition, workspace_target): +# def inject_synthetic_of_original_target_into_build_graph(self, synthetic_target, original_target): +# def get_member_sources_files(self, member_definition): diff --git a/contrib/rust/tests/python/pants_test/contrib/rust/utils/BUILD b/contrib/rust/tests/python/pants_test/contrib/rust/utils/BUILD new file mode 100644 index 00000000000..23e3c8a2da8 --- /dev/null +++ b/contrib/rust/tests/python/pants_test/contrib/rust/utils/BUILD @@ -0,0 +1,43 @@ +# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +python_tests( + name='test_basic_invocation_conversion_rules', + sources=['test_basic_invocation_conversion_rules.py'], + dependencies=[ + 'contrib/rust/src/python/pants/contrib/rust/utils/basic_invocation:args_rules', + 'contrib/rust/src/python/pants/contrib/rust/utils/basic_invocation:env_rules', + 'contrib/rust/src/python/pants/contrib/rust/utils/basic_invocation:links_rule', + 'contrib/rust/src/python/pants/contrib/rust/utils/basic_invocation:outputs_rule', + ] +) + +python_tests( + name='test_basic_invocation_conversion_utils', + sources=['test_basic_invocation_conversion_utils.py'], + dependencies=[ + 'contrib/rust/src/python/pants/contrib/rust/utils:basic_invocation_conversion_utils', + ] +) + +python_tests( + name='test_custom_build_invocation_conversion_rules', + sources=['test_custom_build_invocation_conversion_rules.py'], + dependencies=[ + 'contrib/rust/src/python/pants/contrib/rust/utils/custom_build_invocation:args_rules', + 'contrib/rust/src/python/pants/contrib/rust/utils/custom_build_invocation:env_rules', + 'contrib/rust/src/python/pants/contrib/rust/utils/custom_build_invocation:links_rule', + 'contrib/rust/src/python/pants/contrib/rust/utils/custom_build_invocation:outputs_rule', + 'contrib/rust/src/python/pants/contrib/rust/utils/custom_build_invocation:program_rule', + ] +) + +python_tests( + name='test_custom_build_output_parsing', + sources=['test_custom_build_output_parsing.py'], + dependencies=[ + 'contrib/rust/src/python/pants/contrib/rust/utils:custom_build_output_parsing', + ] +) + + diff --git a/contrib/rust/tests/python/pants_test/contrib/rust/utils/__init__.py b/contrib/rust/tests/python/pants_test/contrib/rust/utils/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/contrib/rust/tests/python/pants_test/contrib/rust/utils/test_basic_invocation_conversion_rules.py b/contrib/rust/tests/python/pants_test/contrib/rust/utils/test_basic_invocation_conversion_rules.py new file mode 100644 index 00000000000..b0ced427a8c --- /dev/null +++ b/contrib/rust/tests/python/pants_test/contrib/rust/utils/test_basic_invocation_conversion_rules.py @@ -0,0 +1,161 @@ +# coding=utf-8 +# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import, division, print_function, unicode_literals + +import collections +import copy +import unittest + +from pants.contrib.rust.utils.basic_invocation.args_rules import args_rules +from pants.contrib.rust.utils.basic_invocation.env_rules import env_rules +from pants.contrib.rust.utils.basic_invocation.links_rule import links_rule +from pants.contrib.rust.utils.basic_invocation.outputs_rule import outputs_rule + + +test_invocation = { + "package_name": "tar_api", + "package_version": "0.0.1", + "target_kind": [ + "lib" + ], + "kind": "Host", + "compile_mode": "build", + "outputs": [ + "/pants/src/rust/engine/target/debug/deps/libtar_api-53a91134f88352d3.rlib" + ], + "links": { + "/pants/src/rust/engine/target/debug/libtar_api.rlib": "/pants/src/rust/engine/target/debug/deps/libtar_api-53a91134f88352d3.rlib" + }, + "program": "rustc", + "args": [ + "--edition=2018", + "--crate-name", + "tar_api", + "tar_api/src/tar_api.rs", + "--color", + "always", + "--crate-type", + "lib", + "--emit=dep-info,link", + "-C", + "debuginfo=2", + "-C", + "metadata=53a91134f88352d3", + "-C", + "extra-filename=-53a91134f88352d3", + "--out-dir", + "/pants/src/rust/engine/target/debug/deps", + "-C", + "incremental=/pants/src/rust/engine/target/debug/incremental", + "-L", + "dependency=/pants/src/rust/engine/target/debug/deps", + "--extern", + "flate2=/pants/src/rust/engine/target/debug/deps/libflate2-c1056eac36b91d52.rlib", + "--extern", + "tar=/pants/src/rust/engine/target/debug/deps/libtar-1f0a911316416d2d.rlib", + "-C", + "link-args=-undefined dynamic_lookup" + ], + "env": { + "CARGO": "/root/.rustup/toolchains/nightly-2018-12-31-x86_64-apple-darwin/bin/cargo", + "CARGO_MANIFEST_DIR": "/pants/src/rust/engine/tar_api", + "CARGO_PKG_AUTHORS": "Pants Build ", + "CARGO_PKG_DESCRIPTION": "", + "CARGO_PKG_HOMEPAGE": "", + "CARGO_PKG_NAME": "tar_api", + "CARGO_PKG_REPOSITORY": "", + "CARGO_PKG_VERSION": "0.0.1", + "CARGO_PKG_VERSION_MAJOR": "0", + "CARGO_PKG_VERSION_MINOR": "0", + "CARGO_PKG_VERSION_PATCH": "1", + "CARGO_PKG_VERSION_PRE": "", + "CARGO_PRIMARY_PACKAGE": "1", + "DYLD_LIBRARY_PATH": "/pants/src/rust/engine/target/debug/deps:/root/.rustup/toolchains/nightly-2018-12-31-x86_64-apple-darwin/lib:/root/.rustup/toolchains/nightly-2018-12-31-x86_64-apple-darwin/lib" + }, + "cwd": "/pants/src/rust/engine" +} + +result_dir = '/pants/pants.d/libtar_api' +libraries_dir = '/pants/pants.d/deps' + + +class UtilsTest(unittest.TestCase): + def get_invocation(self): + return copy.deepcopy(test_invocation) + + def test_link_rule(self): + links = self.get_invocation()['links'] + links_rule(links, result_dir=result_dir) + self.assertEqual(links, { + '/pants/pants.d/libtar_api/libtar_api.rlib': '/pants/pants.d/libtar_api/deps/libtar_api-53a91134f88352d3.rlib'}) + + def test_outputs_rule(self): + outputs = self.get_invocation()['outputs'] + make_dirs = set() + make_sym_links = set() + outputs_rule(outputs, result_dir=result_dir, make_dirs=make_dirs, make_sym_links=make_sym_links) + self.assertEqual(outputs, ['/pants/pants.d/libtar_api/deps/libtar_api-53a91134f88352d3.rlib']) + self.assertEqual(make_dirs, {'/pants/pants.d/libtar_api/deps'}) + self.assertEqual(make_sym_links, + {'/pants/pants.d/libtar_api/deps/libtar_api-53a91134f88352d3.rlib'}) + + def test_env_rules(self): + env = self.get_invocation()['env'] + TargetMock = collections.namedtuple('TargetMock', 'dependencies') + target = TargetMock([]) + crate_out_dirs = dict() + env_rules(env, target=target, result_dir=result_dir, crate_out_dirs=crate_out_dirs, libraries_dir=libraries_dir) + self.assertEqual(env['DYLD_LIBRARY_PATH'], + '/pants/pants.d/deps:/root/.rustup/toolchains/nightly-2018-12-31-x86_64-apple-darwin/lib:/root/.rustup/toolchains/nightly-2018-12-31-x86_64-apple-darwin/lib') + + def test_args_rules(self): + args = self.get_invocation()['args'] + AddressMock = collections.namedtuple('AddressMock', 'target_name') + TargetMock = collections.namedtuple('TargetMock', 'address, dependencies') + + dep1_target = TargetMock(AddressMock('flate2'), []) + dep2_target = TargetMock(AddressMock('tar'), []) + target = TargetMock(AddressMock('tar_api'), [dep1_target, dep2_target]) + + make_dirs = set() + crate_out_dirs = { + 'flate2': ('flate2', ['/pants/pants.d/libflate2/deps/libflate2-c1056eac36b91d52.rlib']), + 'tar': ('tar', ['/pants/pants.d/tar/deps/libtar-1f0a911316416d2d.rlib']) + } + + result = [ + "--edition=2018", + "--crate-name", + "tar_api", + "tar_api/src/tar_api.rs", + "--color", + "always", + "--crate-type", + "lib", + "--emit=dep-info,link", + "-C", + "debuginfo=2", + "-C", + "metadata=53a91134f88352d3", + "-C", + "extra-filename=-53a91134f88352d3", + "--out-dir", + "/pants/pants.d/libtar_api/deps", + "-C", + "incremental=/pants/pants.d/libtar_api/incremental", + "-L", + "dependency=/pants/pants.d/deps", + "--extern", + "flate2=/pants/pants.d/libflate2/deps/libflate2-c1056eac36b91d52.rlib", + "--extern", + "tar=/pants/pants.d/tar/deps/libtar-1f0a911316416d2d.rlib", + "-C", + "link-args=-undefined dynamic_lookup" + ] + + args_rules(args, target=target, result_dir=result_dir, crate_out_dirs=crate_out_dirs, libraries_dir=libraries_dir, make_dirs=make_dirs) + self.assertEqual(args, result) + self.assertEqual(make_dirs, + {'/pants/pants.d/libtar_api/deps', '/pants/pants.d/libtar_api/incremental'}) diff --git a/contrib/rust/tests/python/pants_test/contrib/rust/utils/test_basic_invocation_conversion_utils.py b/contrib/rust/tests/python/pants_test/contrib/rust/utils/test_basic_invocation_conversion_utils.py new file mode 100644 index 00000000000..c3f90890d0c --- /dev/null +++ b/contrib/rust/tests/python/pants_test/contrib/rust/utils/test_basic_invocation_conversion_utils.py @@ -0,0 +1,52 @@ +# coding=utf-8 +# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import, division, print_function, unicode_literals + +import copy +import unittest + +from pants.contrib.rust.utils.basic_invocation_conversion_utils import (reduce_invocation, + sanitize_crate_name) + + +test_invocation = { + "package_name": "tar_api", + "package_version": "0.0.1", + "target_kind": ['lib'], + "kind": "Host", + "compile_mode": "build", + "outputs": [], + "links": {}, + "program": "rustc", + "args": [], + "env": {}, + "cwd": "/pants/src/rust/engine" +} + + +class UtilsTest(unittest.TestCase): + + def get_invocation(self): + return copy.deepcopy(test_invocation) + + def test_reduce_invocation(self): + invocation = self.get_invocation() + result = { + "package_name": "tar_api", + "package_version": "0.0.1", + "target_kind": ['lib'], + "compile_mode": "build", + "outputs": [], + "links": {}, + "program": "rustc", + "args": [], + "env": {}, + "cwd": "/pants/src/rust/engine" + } + reduce_invocation(invocation) + self.assertEqual(invocation, result) + + def test_sanitize_crate_name(self): + self.assertEqual(sanitize_crate_name('tar-api-lib'), 'tar_api_lib') diff --git a/contrib/rust/tests/python/pants_test/contrib/rust/utils/test_custom_build_invocation_conversion_rules.py b/contrib/rust/tests/python/pants_test/contrib/rust/utils/test_custom_build_invocation_conversion_rules.py new file mode 100644 index 00000000000..4b24d8204af --- /dev/null +++ b/contrib/rust/tests/python/pants_test/contrib/rust/utils/test_custom_build_invocation_conversion_rules.py @@ -0,0 +1,262 @@ +# coding=utf-8 +# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import, division, print_function, unicode_literals + +import collections +import copy +import unittest + +from pants.contrib.rust.utils.custom_build_invocation.args_rules import args_rules +from pants.contrib.rust.utils.custom_build_invocation.env_rules import env_rules +from pants.contrib.rust.utils.custom_build_invocation.links_rule import links_rule +from pants.contrib.rust.utils.custom_build_invocation.outputs_rule import outputs_rule +from pants.contrib.rust.utils.custom_build_invocation.program_rule import program_rule + + +custom_build_invocation = { + "package_name": "engine", + "package_version": "0.0.1", + "target_kind": [ + "custom-build" + ], + "kind": "Host", + "compile_mode": "build", + "outputs": [ + "/pants/src/rust/engine/target/debug/build/engine-0c8d5cf2130633dc/build_script_cffi_build-0c8d5cf2130633dc" + ], + "links": { + "/pants/src/rust/engine/target/debug/build/engine-0c8d5cf2130633dc/build-script-cffi_build": "/pants/src/rust/engine/target/debug/build/engine-0c8d5cf2130633dc/build_script_cffi_build-0c8d5cf2130633dc" + }, + "program": "rustc", + "args": [ + "--edition=2018", + "--crate-name", + "build_script_cffi_build", + "src/cffi_build.rs", + "--color", + "always", + "--crate-type", + "bin", + "--emit=dep-info,link", + "-C", + "debuginfo=2", + "-C", + "metadata=0c8d5cf2130633dc", + "-C", + "extra-filename=-0c8d5cf2130633dc", + "--out-dir", + "/pants/src/rust/engine/target/debug/build/engine-0c8d5cf2130633dc", + "-C", + "incremental=/pants/src/rust/engine/target/debug/incremental", + "-L", + "dependency=/pants/src/rust/engine/target/debug/deps", + "--extern", + "build_utils=/pants/src/rust/engine/target/debug/deps/libbuild_utils-cb8514cd7dbe5a1c.rlib", + "--extern", + "cbindgen=/pants/src/rust/engine/target/debug/deps/libcbindgen-cdeba0a445e93ac7.rlib", + "--extern", + "cc=/pants/src/rust/engine/target/debug/deps/libcc-6d75c99c01814b55.rlib", + "-C", + "link-args=-undefined dynamic_lookup" + ], + "env": { + "CARGO": "/root/.rustup/toolchains/nightly-2018-12-31-x86_64-apple-darwin/bin/cargo", + "CARGO_MANIFEST_DIR": "/pants/src/rust/engine", + "CARGO_PKG_AUTHORS": "Pants Build ", + "CARGO_PKG_DESCRIPTION": "", + "CARGO_PKG_HOMEPAGE": "", + "CARGO_PKG_NAME": "engine", + "CARGO_PKG_REPOSITORY": "", + "CARGO_PKG_VERSION": "0.0.1", + "CARGO_PKG_VERSION_MAJOR": "0", + "CARGO_PKG_VERSION_MINOR": "0", + "CARGO_PKG_VERSION_PATCH": "1", + "CARGO_PKG_VERSION_PRE": "", + "CARGO_PRIMARY_PACKAGE": "1", + "DYLD_LIBRARY_PATH": "/pants/src/rust/engine/target/debug/deps:/root/.rustup/toolchains/nightly-2018-12-31-x86_64-apple-darwin/lib:/root/.rustup/toolchains/nightly-2018-12-31-x86_64-apple-darwin/lib" + }, + "cwd": "/pants/src/rust/engine" +} + +run_custom_build_invocation = { + "package_name": "engine", + "package_version": "0.0.1", + "target_kind": [ + "custom-build" + ], + "kind": "Host", + "compile_mode": "run-custom-build", + "outputs": [], + "links": {}, + "program": "/pants/src/rust/engine/target/debug/build/engine-0c8d5cf2130633dc/build-script-cffi_build", + "args": [], + "env": { + "CARGO": "/root/.rustup/toolchains/nightly-2018-12-31-x86_64-apple-darwin/bin/cargo", + "CARGO_CFG_DEBUG_ASSERTIONS": "", + "CARGO_CFG_PROC_MACRO": "", + "CARGO_CFG_TARGET_ARCH": "x86_64", + "CARGO_CFG_TARGET_ENDIAN": "little", + "CARGO_CFG_TARGET_ENV": "", + "CARGO_CFG_TARGET_FAMILY": "unix", + "CARGO_CFG_TARGET_FEATURE": "cmpxchg16b,fxsr,mmx,sse,sse2,sse3,ssse3", + "CARGO_CFG_TARGET_HAS_ATOMIC": "128,16,32,64,8,cas,ptr", + "CARGO_CFG_TARGET_OS": "macos", + "CARGO_CFG_TARGET_POINTER_WIDTH": "64", + "CARGO_CFG_TARGET_THREAD_LOCAL": "", + "CARGO_CFG_TARGET_VENDOR": "apple", + "CARGO_CFG_UNIX": "", + "CARGO_MANIFEST_DIR": "/pants/src/rust/engine", + "CARGO_PKG_AUTHORS": "Pants Build ", + "CARGO_PKG_DESCRIPTION": "", + "CARGO_PKG_HOMEPAGE": "", + "CARGO_PKG_NAME": "engine", + "CARGO_PKG_REPOSITORY": "", + "CARGO_PKG_VERSION": "0.0.1", + "CARGO_PKG_VERSION_MAJOR": "0", + "CARGO_PKG_VERSION_MINOR": "0", + "CARGO_PKG_VERSION_PATCH": "1", + "CARGO_PKG_VERSION_PRE": "", + "DEBUG": "true", + "DYLD_LIBRARY_PATH": "/pants/src/rust/engine/target/debug/deps:/root/.rustup/toolchains/nightly-2018-12-31-x86_64-apple-darwin/lib:/root/.rustup/toolchains/nightly-2018-12-31-x86_64-apple-darwin/lib", + "HOST": "x86_64-apple-darwin", + "NUM_JOBS": "8", + "OPT_LEVEL": "0", + "OUT_DIR": "/pants/src/rust/engine/target/debug/build/engine-ba91b9939db2857c/out", + "PROFILE": "debug", + "RUSTC": "rustc", + "RUSTDOC": "rustdoc", + "TARGET": "x86_64-apple-darwin" + }, + "cwd": "/pants/src/rust/engine" +} + +result_dir = '/pants/pants.d/engine' +libraries_dir = '/pants/pants.d/deps' + + +class UtilsTest(unittest.TestCase): + def get_custom_build_invocation(self): + return copy.deepcopy(custom_build_invocation) + + def get_run_custom_build_invocation(self): + return copy.deepcopy(run_custom_build_invocation) + + def test_args_rules_custom_invocation(self): + args = self.get_custom_build_invocation()['args'] + + AddressMock = collections.namedtuple('AddressMock', 'target_name') + TargetMock = collections.namedtuple('TargetMock', 'address, dependencies') + + dep1_target = TargetMock(AddressMock('build_utils'), []) + dep2_target = TargetMock(AddressMock('cbindgen'), []) + dep3_target = TargetMock(AddressMock('cc'), []) + target = TargetMock(AddressMock('engine'), [dep1_target, dep2_target, dep3_target]) + + make_dirs = set() + crate_out_dirs = { + 'build_utils': ( + 'build_utils', ['/pants/pants.d/build_utils/deps/libbuild_utils-cb8514cd7dbe5a1c.rlib']), + 'cbindgen': ('cbindgen', ['/pants/pants.d/cbindgen/deps/libcbindgen-cdeba0a445e93ac7.rlib']), + 'cc': ('cc', ['/pants/pants.d/cc/deps/libcc-6d75c99c01814b55.rlib']) + } + + create_information = { + 'package_name': 'engine', + 'extra-filename': '-0c8d5cf2130633dc' + } + + result = [ + "--edition=2018", + "--crate-name", + "build_script_cffi_build", + "src/cffi_build.rs", + "--color", + "always", + "--crate-type", + "bin", + "--emit=dep-info,link", + "-C", + "debuginfo=2", + "-C", + "metadata=0c8d5cf2130633dc", + "-C", + "extra-filename=-0c8d5cf2130633dc", + "--out-dir", + "/pants/pants.d/engine/build/engine-0c8d5cf2130633dc", + "-C", + "incremental=/pants/pants.d/engine/incremental", + "-L", + "dependency=/pants/pants.d/deps", + "--extern", + "build_utils=/pants/pants.d/build_utils/deps/libbuild_utils-cb8514cd7dbe5a1c.rlib", + "--extern", + "cbindgen=/pants/pants.d/cbindgen/deps/libcbindgen-cdeba0a445e93ac7.rlib", + "--extern", + "cc=/pants/pants.d/cc/deps/libcc-6d75c99c01814b55.rlib", + "-C", + "link-args=-undefined dynamic_lookup" + ] + + args_rules(args, target=target, result_dir=result_dir, crate_out_dirs=crate_out_dirs, + libraries_dir=libraries_dir, information=create_information, + make_dirs=make_dirs) + print(args) + self.assertEqual(args, result) + self.assertEqual(make_dirs, {'/pants/pants.d/engine/build/engine-0c8d5cf2130633dc', + '/pants/pants.d/engine/incremental'}) + + def test_outputs_rule_custom_invocation(self): + outputs = self.get_custom_build_invocation()['outputs'] + create_information = { + 'package_name': 'engine', + 'extra-filename': '-0c8d5cf2130633dc' + } + make_dirs = set() + outputs_rule(outputs, result_dir=result_dir, information=create_information, + make_dirs=make_dirs) + + self.assertEqual(outputs, [ + '/pants/pants.d/engine/build/engine-0c8d5cf2130633dc/build_script_cffi_build-0c8d5cf2130633dc']) + self.assertEqual(make_dirs, {'/pants/pants.d/engine/build/engine-0c8d5cf2130633dc'}) + + def test_links_rule_custom_invocation(self): + links = self.get_custom_build_invocation()['links'] + create_information = { + 'package_name': 'engine', + 'extra-filename': '-0c8d5cf2130633dc' + } + + links_rule(links, result_dir=result_dir, information=create_information) + + self.assertEqual(links, { + "/pants/pants.d/engine/build/engine-0c8d5cf2130633dc/build-script-cffi_build": "/pants/pants.d/engine/build/engine-0c8d5cf2130633dc/build_script_cffi_build-0c8d5cf2130633dc"}) + + def test_program_rule_run_custom_invocation(self): + invocation = self.get_run_custom_build_invocation() + + AddressMock = collections.namedtuple('AddressMock', 'target_name') + TargetMock = collections.namedtuple('TargetMock', 'address, dependencies') + + dep_target = TargetMock(AddressMock('engine_custom_build'), []) + target = TargetMock(AddressMock('engine_run_custom_build'), [dep_target]) + + crate_out_dirs = { + 'engine_custom_build': (None, { + "/pants/pants.d/engine/build/engine-ba91b9939db2857c/build-script-cffi_build": "/pants/pants.d/engine/build/engine-ba91b9939db2857c/build_script_cffi_build-0c8d5cf2130633dc"}) + } + + program_rule(target=target, crate_out_dirs=crate_out_dirs, invocation=invocation) + self.assertEqual(invocation['program'], + '/pants/pants.d/engine/build/engine-ba91b9939db2857c/build-script-cffi_build') + + def test_env_rules_run_custom_invocation(self): + env = self.get_run_custom_build_invocation()['env'] + make_dirs = set() + env_rules(env, result_dir=result_dir, libraries_dir=libraries_dir, make_dirs=make_dirs) + self.assertEqual(env['DYLD_LIBRARY_PATH'], + '/pants/pants.d/deps:/root/.rustup/toolchains/nightly-2018-12-31-x86_64-apple-darwin/lib:/root/.rustup/toolchains/nightly-2018-12-31-x86_64-apple-darwin/lib') + self.assertEqual(env['OUT_DIR'], + '/pants/pants.d/engine/build/engine-ba91b9939db2857c/out') + self.assertEqual(make_dirs, {'/pants/pants.d/engine/build/engine-ba91b9939db2857c/out'}) diff --git a/contrib/rust/tests/python/pants_test/contrib/rust/utils/test_custom_build_output_parsing.py b/contrib/rust/tests/python/pants_test/contrib/rust/utils/test_custom_build_output_parsing.py new file mode 100644 index 00000000000..9e63a8cc190 --- /dev/null +++ b/contrib/rust/tests/python/pants_test/contrib/rust/utils/test_custom_build_output_parsing.py @@ -0,0 +1,54 @@ +# coding=utf-8 +# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import, division, print_function, unicode_literals + +import unittest + +from pants.contrib.rust.utils.custom_build_output_parsing import parse_multiple_cargo_statements + + +build_output = [ + 'cargo:rustc-link-lib=static=native_engine_ffi', + 'cargo:rustc-link-search=native=pants/.pants.d/engine/out', 'cargo:rustc-cfg=rust_1_26', + 'cargo:rustc-cfg=memchr_runtime_sse42', 'cargo:rustc-env=PROTOC=/protobuf/protoc', + 'cargo:rustc-env=PROTOC_INCLUDE=/protobuf/include', 'cargo:rustc-flags=-l static=samplerate', + 'cargo:rustc-flags= -l dylib=stdc++ ', + 'cargo:rustc-flags=-l static=pfring -L /usr/local/lib', + 'cargo:rustc-flags= -l static=pcap -L /usr/local/lib ', + 'cargo:warning=/lmdb-sys/lmdb/libraries/liblmdb/mdb.c:10033:33: warning: unused parameter [-Wunused-parameter]', + 'cargo:warning=mdb_env_get_maxkeysize(MDB_env *env)', + 'cargo:warning= ^', 'cargo:warning=1 warning generated.', + 'cargo:rerun-if-env-changed=PY', 'cargo:rerun-if-changed=cbindgen.toml' +] + + +class UtilsTest(unittest.TestCase): + + def get_build_output(self): + return build_output + + def test_env_rules_run_custom_invocation(self): + output = self.get_build_output() + result = { + 'rustc-flags': [ + ['-l', 'static=samplerate'], + ['-l', 'dylib=stdc++'], + ['-l', 'static=pfring', '-L', '/usr/local/lib'], + ['-l', 'static=pcap', '-L', '/usr/local/lib'], + ], + 'rustc-env': [['PROTOC', '/protobuf/protoc'], ['PROTOC_INCLUDE', '/protobuf/include']], + 'rustc-link-search': [['-L', 'native=pants/.pants.d/engine/out'],], + 'rustc-link-lib': [['-l', 'static=native_engine_ffi'],], + 'rustc-cfg': [['--cfg', 'rust_1_26'], ['--cfg', 'memchr_runtime_sse42']], + 'warning': [ + '/lmdb-sys/lmdb/libraries/liblmdb/mdb.c:10033:33: warning: unused parameter [-Wunused-parameter]', + 'mdb_env_get_maxkeysize(MDB_env *env)', ' ^', + '1 warning generated.' + ], + 'rerun-if-env-changed': ['PY'], + 'rerun-if-changed': ['cbindgen.toml'] + } + statements = parse_multiple_cargo_statements(output) + self.assertEqual(statements, result) diff --git a/pants.ini b/pants.ini index 57b2367a78b..437992c1cef 100644 --- a/pants.ini +++ b/pants.ini @@ -41,6 +41,7 @@ pythonpath: [ "%(buildroot)s/contrib/scalajs/src/python", "%(buildroot)s/contrib/scrooge/src/python", "%(buildroot)s/contrib/thrifty/src/python", + "%(buildroot)s/contrib/rust/src/python", "%(buildroot)s/pants-plugins/src/python", ] @@ -67,6 +68,7 @@ backend_packages: +[ "pants.contrib.python.checks", "pants.contrib.scrooge", "pants.contrib.thrifty", + "pants.contrib.rust", ] # Path patterns to ignore for filesystem operations on top of the builtin patterns. diff --git a/src/python/pants/bin/BUILD b/src/python/pants/bin/BUILD index c2c6c559e25..db0aaa54ffb 100644 --- a/src/python/pants/bin/BUILD +++ b/src/python/pants/bin/BUILD @@ -71,6 +71,9 @@ python_binary( dependencies=[ ':bin', 'contrib/python/src/python/pants/contrib/python/checks/tasks/checkstyle', + # The pants local binary depends on the rust contrib module in order to transitively pick up + # its `toml` dep in a slightly more self-documenting way than directly adding the `toml` dep. + 'contrib/rust/src/python/pants/contrib/rust:plugin', 'pants-plugins/src/python/internal_backend:plugins', ], )