Skip to content

Commit

Permalink
python3-maturin: Fix cross compilation issue for armv7l, mips64, ppc
Browse files Browse the repository at this point in the history 
When bitbaking python3-rpds-py it built extension module as:

  site-packages/rpds/rpds.cpython-312-armv7l-linux-gnueabihf.so

Which caused error on target:

  root@qemuarm:~# python3 -c "from rpds import HashTrieMap, HashTrieSet, List"
  Traceback (most recent call last):
    File "<string>", line 1, in <module>
    File "/usr/lib/python3.12/site-packages/rpds/__init__.py", line 1, in <module>
      from .rpds import *
  ModuleNotFoundError: No module named 'rpds.rpds'

Where as it should have been:

  site-packages/rpds/rpds.cpython-312-arm-linux-gnueabihf.so

Associated upstream bug report:
PyO3/maturin#2203

Associated upstream pull request:
PyO3/maturin#2204

Note - mitigation has not been tested with musl:
PyO3/maturin#2204 (comment)

(From OE-Core rev: 32a8a7379008cc6e367b7664c5b10b29f0bb8136)

(From OE-Core rev: d2f73e3840c21997b918d1f1cfae965c618c1076)

Signed-off-by: Vesa Jääskeläinen <[email protected]>
Signed-off-by: Niko Mauno <[email protected]>
Signed-off-by: Richard Purdie <[email protected]>
Signed-off-by: Steve Sakoman <[email protected]>
  • Loading branch information
@nikomauno @sakoman
nikomauno authored and sakoman committed Sep 13, 2024
1 parent 79dd59e commit 078be0c
Show file tree
Hide file tree
Showing 6 changed files with 438 additions and 0 deletions.
107 changes: 107 additions & 0 deletions ...ls/python/python3-maturin/0001-Extract-extension-architecture-name-resolvation-code.patch
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,107 @@
From 42a97ee7100ad158d4b1ba6133ea13cc864a567f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Vesa=20J=C3=A4=C3=A4skel=C3=A4inen?=
<[email protected]>
Date: Sun, 1 Sep 2024 09:23:10 +0300
Subject: [PATCH 1/5] Extract extension architecture name resolvation code as
helper
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This commit introduces helper InterpreterConfig.get_python_ext_arch() that
can be used to determine the extension architecture name python uses in
`ext_suffix` for this architecture.

Upstream-Status: Backport [https://github.com/PyO3/maturin/commit/42a97ee7100ad158d4b1ba6133ea13cc864a567f]

Signed-off-by: Vesa Jääskeläinen <[email protected]>
---
src/python_interpreter/config.rs | 18 ++++++------------
src/target.rs | 16 ++++++++++++++++
2 files changed, 22 insertions(+), 12 deletions(-)

diff --git a/src/python_interpreter/config.rs b/src/python_interpreter/config.rs
index 912f9218..d76606f2 100644
--- a/src/python_interpreter/config.rs
+++ b/src/python_interpreter/config.rs
@@ -47,15 +47,7 @@ impl InterpreterConfig {
// Python 2 is not supported
return None;
}
- let python_arch = if matches!(target.target_arch(), Arch::Armv6L | Arch::Armv7L) {
- "arm"
- } else if matches!(target.target_arch(), Arch::Powerpc64Le) && python_impl == PyPy {
- "ppc_64"
- } else if matches!(target.target_arch(), Arch::X86) && python_impl == PyPy {
- "x86"
- } else {
- target.get_python_arch()
- };
+ let python_ext_arch = target.get_python_ext_arch(python_impl);
// See https://github.com/pypa/auditwheel/issues/349
let target_env = match python_impl {
CPython => {
@@ -77,7 +69,7 @@ impl InterpreterConfig {
let ldversion = format!("{}{}{}", major, minor, abiflags);
let ext_suffix = format!(
".cpython-{}-{}-linux-{}.so",
- ldversion, python_arch, target_env
+ ldversion, python_ext_arch, target_env
);
Some(Self {
major,
@@ -90,7 +82,8 @@ impl InterpreterConfig {
}
(Os::Linux, PyPy) => {
let abi_tag = format!("pypy{}{}-{}", major, minor, PYPY_ABI_TAG);
- let ext_suffix = format!(".{}-{}-linux-{}.so", abi_tag, python_arch, target_env);
+ let ext_suffix =
+ format!(".{}-{}-linux-{}.so", abi_tag, python_ext_arch, target_env);
Some(Self {
major,
minor,
@@ -204,7 +197,8 @@ impl InterpreterConfig {
}
(Os::Emscripten, CPython) => {
let ldversion = format!("{}{}", major, minor);
- let ext_suffix = format!(".cpython-{}-{}-emscripten.so", ldversion, python_arch);
+ let ext_suffix =
+ format!(".cpython-{}-{}-emscripten.so", ldversion, python_ext_arch);
Some(Self {
major,
minor,
diff --git a/src/target.rs b/src/target.rs
index dc7df0cf..84bae559 100644
--- a/src/target.rs
+++ b/src/target.rs
@@ -1,4 +1,5 @@
use crate::cross_compile::is_cross_compiling;
+use crate::python_interpreter::InterpreterKind;
use crate::PlatformTag;
use anyhow::{anyhow, bail, format_err, Result};
use platform_info::*;
@@ -368,6 +369,21 @@ impl Target {
}
}

+ /// Returns the extension architecture name python uses in `ext_suffix` for this architecture.
+ pub fn get_python_ext_arch(&self, python_impl: InterpreterKind) -> &str {
+ if matches!(self.target_arch(), Arch::Armv6L | Arch::Armv7L) {
+ "arm"
+ } else if matches!(self.target_arch(), Arch::Powerpc64Le)
+ && python_impl == InterpreterKind::PyPy
+ {
+ "ppc_64"
+ } else if matches!(self.target_arch(), Arch::X86) && python_impl == InterpreterKind::PyPy {
+ "x86"
+ } else {
+ self.get_python_arch()
+ }
+ }
+
/// Returns the name python uses in `sys.platform` for this os
pub fn get_python_os(&self) -> &str {
match self.os {
--
2.34.1

76 changes: 76 additions & 0 deletions ...ls/python/python3-maturin/0002-Fix-cross-compilation-issue-with-linux-armv7l-archit.patch
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
From 0c6b8cc84eff72ed21098029aaba079b899dbee2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Vesa=20J=C3=A4=C3=A4skel=C3=A4inen?=
<[email protected]>
Date: Sun, 1 Sep 2024 09:23:40 +0300
Subject: [PATCH 2/5] Fix cross compilation issue with linux-armv7l
architecture
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

When compiling under Yocto project for linux-armv7l target architecture
.so files were generated incorrectly as:

rpds.cpython-312-armv7l-linux-gnueabihf.so

Where as platform and EXT_SUFFIX are defined as:

>>> sysconfig.get_platform()
'linux-armv7l'
>>> sysconfig.get_config_vars()['EXT_SUFFIX']
'.cpython-312-arm-linux-gnueabihf.so'

Which should have caused the .so files as:

rpds.cpython-312-arm-linux-gnueabihf.so

Upstream-Status: Backport [https://github.com/PyO3/maturin/commit/0c6b8cc84eff72ed21098029aaba079b899dbee2]

Signed-off-by: Vesa Jääskeläinen <[email protected]>
---
src/python_interpreter/config.rs | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/python_interpreter/config.rs b/src/python_interpreter/config.rs
index d76606f2..5736aedc 100644
--- a/src/python_interpreter/config.rs
+++ b/src/python_interpreter/config.rs
@@ -306,7 +306,7 @@ impl InterpreterConfig {
format!(
".cpython-{}-{}-{}-{}.{}",
abi_tag,
- target.get_python_arch(),
+ target.get_python_ext_arch(interpreter_kind),
target.get_python_os(),
target_env,
file_ext,
@@ -319,7 +319,7 @@ impl InterpreterConfig {
major,
minor,
abi_tag,
- target.get_python_arch(),
+ target.get_python_ext_arch(interpreter_kind),
target.get_python_os(),
target_env,
file_ext,
@@ -330,7 +330,7 @@ impl InterpreterConfig {
format!(
".{}-{}-{}.{}",
abi_tag.replace('_', "-"),
- target.get_python_arch(),
+ target.get_python_ext_arch(interpreter_kind),
target.get_python_os(),
file_ext,
)
@@ -341,7 +341,7 @@ impl InterpreterConfig {
format!(
".cpython-{}-{}-{}.{}",
abi_tag,
- target.get_python_arch(),
+ target.get_python_ext_arch(interpreter_kind),
target.get_python_os(),
file_ext
)
--
2.34.1

98 changes: 98 additions & 0 deletions ...ls/python/python3-maturin/0003-Extract-extension-ABI-name-resolvation-code-as-helpe.patch
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
From fa64426f3a98a0455721c23ec86bd2240708b45e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Vesa=20J=C3=A4=C3=A4skel=C3=A4inen?=
<[email protected]>
Date: Sun, 1 Sep 2024 15:55:07 +0300
Subject: [PATCH 3/5] Extract extension ABI name resolvation code as helper
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This commit introduces helper InterpreterConfig.get_python_target_env()
that can be used to determine the extension ABI python uses in
`ext_suffix` for this architecture.

Upstream-Status: Backport [https://github.com/PyO3/maturin/commit/fa64426f3a98a0455721c23ec86bd2240708b45e]

Signed-off-by: Vesa Jääskeläinen <[email protected]>
---
src/python_interpreter/config.rs | 19 ++-----------------
src/target.rs | 20 ++++++++++++++++++++
2 files changed, 22 insertions(+), 17 deletions(-)

diff --git a/src/python_interpreter/config.rs b/src/python_interpreter/config.rs
index 5736aedc..938e9955 100644
--- a/src/python_interpreter/config.rs
+++ b/src/python_interpreter/config.rs
@@ -48,17 +48,7 @@ impl InterpreterConfig {
return None;
}
let python_ext_arch = target.get_python_ext_arch(python_impl);
- // See https://github.com/pypa/auditwheel/issues/349
- let target_env = match python_impl {
- CPython => {
- if python_version >= (3, 11) {
- target.target_env().to_string()
- } else {
- target.target_env().to_string().replace("musl", "gnu")
- }
- }
- PyPy | GraalPy => "gnu".to_string(),
- };
+ let target_env = target.get_python_target_env(python_impl, python_version);
match (target.target_os(), python_impl) {
(Os::Linux, CPython) => {
let abiflags = if python_version < (3, 8) {
@@ -294,12 +284,7 @@ impl InterpreterConfig {
};
let file_ext = if target.is_windows() { "pyd" } else { "so" };
let ext_suffix = if target.is_linux() || target.is_macos() {
- // See https://github.com/pypa/auditwheel/issues/349
- let target_env = if (major, minor) >= (3, 11) {
- target.target_env().to_string()
- } else {
- target.target_env().to_string().replace("musl", "gnu")
- };
+ let target_env = target.get_python_target_env(interpreter_kind, (major, minor));
match interpreter_kind {
InterpreterKind::CPython => ext_suffix.unwrap_or_else(|| {
// Eg: .cpython-38-x86_64-linux-gnu.so
diff --git a/src/target.rs b/src/target.rs
index 84bae559..ad8ebaba 100644
--- a/src/target.rs
+++ b/src/target.rs
@@ -1,5 +1,6 @@
use crate::cross_compile::is_cross_compiling;
use crate::python_interpreter::InterpreterKind;
+use crate::python_interpreter::InterpreterKind::{CPython, GraalPy, PyPy};
use crate::PlatformTag;
use anyhow::{anyhow, bail, format_err, Result};
use platform_info::*;
@@ -384,6 +385,25 @@ impl Target {
}
}

+ /// Returns the environment python uses in `ext_suffix` for this architecture.
+ pub fn get_python_target_env(
+ &self,
+ python_impl: InterpreterKind,
+ python_version: (usize, usize),
+ ) -> String {
+ match python_impl {
+ CPython => {
+ // For musl handling see https://github.com/pypa/auditwheel/issues/349
+ if python_version >= (3, 11) {
+ self.target_env().to_string()
+ } else {
+ self.target_env().to_string().replace("musl", "gnu")
+ }
+ }
+ PyPy | GraalPy => "gnu".to_string(),
+ }
+ }
+
/// Returns the name python uses in `sys.platform` for this os
pub fn get_python_os(&self) -> &str {
match self.os {
--
2.34.1

68 changes: 68 additions & 0 deletions ...ls/python/python3-maturin/0004-Fix-cross-compilation-issue-with-linux-ppc-architect.patch
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
From f2c892109a05db144e8b18bcbcf9c24fe8d977c4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Vesa=20J=C3=A4=C3=A4skel=C3=A4inen?=
<[email protected]>
Date: Sun, 1 Sep 2024 15:55:16 +0300
Subject: [PATCH 4/5] Fix cross compilation issue with linux-ppc architecture
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

When compiling under Yocto project for linux-ppc target architecture
.so files were generated incorrectly as:

rpds.cpython-312-ppc-linux-gnu.so

Where as platform and EXT_SUFFIX are defined as:

>>> sysconfig.get_platform()
'linux-ppc'
>>> sysconfig.get_config_vars()['EXT_SUFFIX']
'.cpython-312-powerpc-linux-gnu.so'

Which should have caused the .so files as:

rpds.cpython-312-powerpc-linux-gnu.so

Upstream-Status: Backport [https://github.com/PyO3/maturin/commit/f2c892109a05db144e8b18bcbcf9c24fe8d977c4]

Signed-off-by: Vesa Jääskeläinen <[email protected]>
---
src/python_interpreter/config.rs | 8 ++++++++
src/target.rs | 2 ++
2 files changed, 10 insertions(+)

diff --git a/src/python_interpreter/config.rs b/src/python_interpreter/config.rs
index 938e9955..8f883887 100644
--- a/src/python_interpreter/config.rs
+++ b/src/python_interpreter/config.rs
@@ -424,6 +424,14 @@ mod test {
".cpython-310-powerpc64le-linux-gnu.so"
);

+ let sysconfig = InterpreterConfig::lookup_one(
+ &Target::from_target_triple(Some("powerpc-unknown-linux-gnu".to_string())).unwrap(),
+ InterpreterKind::CPython,
+ (3, 10),
+ )
+ .unwrap();
+ assert_eq!(sysconfig.ext_suffix, ".cpython-310-powerpc-linux-gnu.so");
+
let sysconfig = InterpreterConfig::lookup_one(
&Target::from_target_triple(Some("s390x-unknown-linux-gnu".to_string())).unwrap(),
InterpreterKind::CPython,
diff --git a/src/target.rs b/src/target.rs
index ad8ebaba..93afd9bb 100644
--- a/src/target.rs
+++ b/src/target.rs
@@ -380,6 +380,8 @@ impl Target {
"ppc_64"
} else if matches!(self.target_arch(), Arch::X86) && python_impl == InterpreterKind::PyPy {
"x86"
+ } else if matches!(self.target_arch(), Arch::Powerpc) {
+ "powerpc"
} else {
self.get_python_arch()
}
--
2.34.1

Loading

0 comments on commit 078be0c

Please sign in to comment.