Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Python: merge expressions of interpreters #53123

Merged
merged 5 commits into from
Jan 4, 2019
Merged

Python: merge expressions of interpreters #53123

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
efbe87f
CPython: merge expressions of interpreters
FRidh Dec 31, 2018
613498a
pypy: merge 2.7 and 3.5 into a single expression
FRidh Jan 1, 2019
f665828
Python: improve cross-compilation
FRidh Jan 2, 2019
0a2caa4
Python: drop `python.majorVersion`
FRidh Jan 3, 2019
d91b496
pythonInterpreters.pypy{27,35}_prebuilt: init at 6.0.0
FRidh Jan 3, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion pkgs/applications/audio/ingen/default.nix
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ stdenv.mkDerivation rec {
for program in ingenams ingenish
do
wrapProgram $out/bin/$program \
--prefix PYTHONPATH : $out/lib/python${python.majorVersion}/site-packages:$PYTHONPATH
--prefix PYTHONPATH : $out/${python.sitePackages}:$PYTHONPATH
done
'';

Expand Down
10 changes: 5 additions & 5 deletions pkgs/applications/misc/blender/default.nix
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,10 +51,10 @@ stdenv.mkDerivation rec {
"-DWITH_SYSTEM_OPENJPEG=ON"
"-DWITH_PLAYER=ON"
"-DWITH_OPENSUBDIV=ON"
"-DPYTHON_LIBRARY=python${python.majorVersion}m"
"-DPYTHON_LIBRARY=${python.libPrefix}"
"-DPYTHON_LIBPATH=${python}/lib"
"-DPYTHON_INCLUDE_DIR=${python}/include/python${python.majorVersion}m"
"-DPYTHON_VERSION=${python.majorVersion}"
"-DPYTHON_INCLUDE_DIR=${python}/include/${python.libPrefix}"
"-DPYTHON_VERSION=${python.pythonVersion}"
"-DWITH_PYTHON_INSTALL=OFF"
"-DWITH_PYTHON_INSTALL_NUMPY=OFF"
]
Expand All @@ -66,7 +66,7 @@ stdenv.mkDerivation rec {
]
++ optional colladaSupport "-DWITH_OPENCOLLADA=ON";

NIX_CFLAGS_COMPILE = "-I${ilmbase.dev}/include/OpenEXR -I${python}/include/${python.libPrefix}m";
NIX_CFLAGS_COMPILE = "-I${ilmbase.dev}/include/OpenEXR -I${python}/include/${python.libPrefix}";

# Since some dependencies are built with gcc 6, we need gcc 6's
# libstdc++ in our RPATH. Sigh.
Expand All @@ -77,7 +77,7 @@ stdenv.mkDerivation rec {
postInstall = optionalString enableNumpy
''
wrapProgram $out/bin/blender \
--prefix PYTHONPATH : ${pythonPackages.numpy}/lib/python${python.majorVersion}/site-packages
--prefix PYTHONPATH : ${pythonPackages.numpy}/${python.sitePackages}
'';

meta = with stdenv.lib; {
Expand Down
2 changes: 1 addition & 1 deletion pkgs/applications/networking/instant-messengers/profanity/default.nix
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ stdenv.mkDerivation rec {
++ optionals pythonPluginSupport [ "-I${python}/include/${python.libPrefix}" ];

LDFLAGS = [ ]
++ optionals pythonPluginSupport [ "-L${python}/lib" "-lpython${python.majorVersion}m" ];
++ optionals pythonPluginSupport [ "-L${python}/lib" "-l${python.libPrefix}" ];

meta = {
description = "A console based XMPP client";
Expand Down
4 changes: 2 additions & 2 deletions pkgs/applications/science/misc/golly/beta.nix
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
}:

stdenv.mkDerivation rec {
name = "golly-${version}";
pname = "golly";
version = "2.8.99.2.20161122";
#src = fetchurl {
# url="mirror://sourceforge/project/golly/golly/golly-2.8/golly-2.8-src.tar.gz";
Expand All @@ -30,7 +30,7 @@ stdenv.mkDerivation rec {
makeFlags=[
"AM_LDFLAGS="
];
NIX_LDFLAGS="-lpython${python2.majorVersion} -lperl";
NIX_LDFLAGS="-l${python2.libPrefix} -lperl";
preConfigure=''
export NIX_LDFLAGS="$NIX_LDFLAGS -L$(dirname "$(find ${perl} -name libperl.so)")"
export NIX_CFLAGS_COMPILE="$NIX_CFLAGS_COMPILE
Expand Down
5 changes: 2 additions & 3 deletions pkgs/applications/science/misc/golly/default.nix
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,7 @@
{stdenv, fetchurl, wxGTK, perl, python2, zlib, libGLU_combined, libX11}:
stdenv.mkDerivation rec {
baseName="golly";
pname = "golly";
version = "3.2";
name="${baseName}-${version}";

src = fetchurl {
sha256 = "0cg9mbwmf4q6qxhqlnzrxh9y047banxdb8pd3hgj3smmja2zf0jd";
Expand All @@ -21,7 +20,7 @@ stdenv.mkDerivation rec {
makeFlags=[
"AM_LDFLAGS="
];
NIX_LDFLAGS="-lpython${python2.majorVersion} -lperl";
NIX_LDFLAGS="-l${python2.libPrefix} -lperl";
preConfigure=''
export NIX_LDFLAGS="$NIX_LDFLAGS -L$(dirname "$(find ${perl} -name libperl.so)")"
export NIX_CFLAGS_COMPILE="$NIX_CFLAGS_COMPILE
Expand Down
5 changes: 2 additions & 3 deletions pkgs/development/interpreters/python/build-python-package-common.nix
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
# This function provides generic bits to install a Python wheel.

{ python
, bootstrapped-pip
}:

{ buildInputs ? []
Expand All @@ -10,7 +9,7 @@
, ... } @ attrs:

attrs // {
buildInputs = buildInputs ++ [ bootstrapped-pip ];
buildInputs = buildInputs ++ [ python.pythonForBuild.pkgs.bootstrapped-pip ];
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Bootstrapped pip is used at build time and so should be a nativeBuildInput.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Alternatively, since you use it explicitly below, maybe it doesn't need to be any sort of imput.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good catch. I don't think we actually need it as a build input. We call it already from the store path, and Python packages should never call or import from pip. Doesn't mean they won't do it though...

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

So, we do need it as *buildInput. While it is indeed supposed to be nativeBuildInput, putting it there breaks cross-compilation for me, likely because at the moment something wrong is happening yet it looks alright. I'll keep this issue for another day.


configurePhase = attrs.configurePhase or ''
runHook preConfigure
Expand All @@ -24,7 +23,7 @@ attrs // {
export PYTHONPATH="$out/${python.sitePackages}:$PYTHONPATH"

pushd dist
${bootstrapped-pip}/bin/pip install *.whl --no-index --prefix=$out --no-cache ${toString installFlags} --build tmpbuild
${python.pythonForBuild.pkgs.bootstrapped-pip}/bin/pip install *.whl --no-index --prefix=$out --no-cache ${toString installFlags} --build tmpbuild
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

N.B. this is fine, but won't need it once bootstrapped-pip is a nativeBuildInput.

popd

runHook postInstall
Expand Down
11 changes: 5 additions & 6 deletions pkgs/development/interpreters/python/build-python-package-setuptools.nix
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@

{ lib
, python
, bootstrapped-pip
}:

{
Expand All @@ -26,13 +25,13 @@ in attrs // {
buildPhase = attrs.buildPhase or ''
runHook preBuild
cp ${setuppy} nix_run_setup
${python.interpreter} nix_run_setup ${lib.optionalString (setupPyBuildFlags != []) ("build_ext " + (lib.concatStringsSep " " setupPyBuildFlags))} bdist_wheel
${python.pythonForBuild.interpreter} nix_run_setup ${lib.optionalString (setupPyBuildFlags != []) ("build_ext " + (lib.concatStringsSep " " setupPyBuildFlags))} bdist_wheel
runHook postBuild
'';

installCheckPhase = attrs.checkPhase or ''
runHook preCheck
${python.interpreter} nix_run_setup test
${python.pythonForBuild.interpreter} nix_run_setup test
runHook postCheck
'';

Expand All @@ -47,9 +46,9 @@ in attrs // {
if test -e setup.py; then
tmp_path=$(mktemp -d)
export PATH="$tmp_path/bin:$PATH"
export PYTHONPATH="$tmp_path/${python.sitePackages}:$PYTHONPATH"
mkdir -p $tmp_path/${python.sitePackages}
${bootstrapped-pip}/bin/pip install -e . --prefix $tmp_path >&2
export PYTHONPATH="$tmp_path/${python.pythonForBuild.sitePackages}:$PYTHONPATH"
mkdir -p $tmp_path/${python.pythonForBuild.sitePackages}
${python.pythonForBuild.pkgs.bootstrapped-pip}/bin/pip install -e . --prefix $tmp_path >&2
fi
${postShellHook}
'';
Expand Down
5 changes: 2 additions & 3 deletions pkgs/development/interpreters/python/build-python-package.nix
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,17 +10,16 @@
, ensureNewerSourcesForZipFilesHook
, toPythonModule
, namePrefix
, bootstrapped-pip
, flit
, writeScript
, update-python-libraries
}:

let
setuptools-specific = import ./build-python-package-setuptools.nix { inherit lib python bootstrapped-pip; };
setuptools-specific = import ./build-python-package-setuptools.nix { inherit lib python; };
flit-specific = import ./build-python-package-flit.nix { inherit python flit; };
wheel-specific = import ./build-python-package-wheel.nix { };
common = import ./build-python-package-common.nix { inherit python bootstrapped-pip; };
common = import ./build-python-package-common.nix { inherit python; };
mkPythonDerivation = import ./mk-python-derivation.nix {
inherit lib config python wrapPython setuptools unzip ensureNewerSourcesForZipFilesHook;
inherit toPythonModule namePrefix writeScript update-python-libraries;
Expand Down
72 changes: 35 additions & 37 deletions pkgs/development/interpreters/python/cpython/2.7/default.nix
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
{ stdenv, buildPackages, fetchurl
{ stdenv, fetchurl, fetchpatch
, bzip2
, expat
, libffi
, gdbm
, fetchpatch
, db
, ncurses
, openssl
, readline
Expand All @@ -10,15 +12,16 @@
, zlib
, callPackage
, self
, db
, expat
, libffi
, CF, configd, coreutils
, python-setup-hook
# Some proprietary libs assume UCS2 unicode, especially on darwin :(
, ucsEncoding ? 4
# For the Python package set
, packageOverrides ? (self: super: {})
, buildPackages
, sourceVersion
, sha256
, passthruFun
}:

assert x11Support -> tcl != null
Expand All @@ -29,16 +32,26 @@ assert x11Support -> tcl != null
with stdenv.lib;

let
majorVersion = "2.7";
minorVersion = "15";
minorVersionSuffix = "";
version = "${majorVersion}.${minorVersion}${minorVersionSuffix}";
libPrefix = "python${majorVersion}";
sitePackages = "lib/${libPrefix}/site-packages";

pythonForBuild = buildPackages.${"python${sourceVersion.major}${sourceVersion.minor}"};

passthru = passthruFun rec {
inherit self sourceVersion packageOverrides;
implementation = "cpython";
libPrefix = "python${pythonVersion}";
executable = libPrefix;
pythonVersion = with sourceVersion; "${major}.${minor}";
sitePackages = "lib/${libPrefix}/site-packages";
inherit pythonForBuild;
} // {
inherit ucsEncoding;
};

version = with sourceVersion; "${major}.${minor}.${patch}${suffix}";

src = fetchurl {
url = "https://www.python.org/ftp/python/${majorVersion}.${minorVersion}/Python-${version}.tar.xz";
sha256 = "0x2mvz9dp11wj7p5ccvmk9s0hzjk2fa1m462p395l4r6bfnb3n92";
url = with sourceVersion; "https://www.python.org/ftp/python/${major}.${minor}.${patch}/Python-${version}.tar.xz";
inherit sha256;
};

hasDistutilsCxxPatch = !(stdenv.cc.isGNU or false);
Expand Down Expand Up @@ -191,12 +204,11 @@ let
# Build the basic Python interpreter without modules that have
# external dependencies.

in stdenv.mkDerivation ({
name = "python-${version}";
pythonVersion = majorVersion;
in with passthru; stdenv.mkDerivation ({
pname = "python";
inherit version;

inherit majorVersion version src patches buildInputs nativeBuildInputs
preConfigure configureFlags;
inherit src patches buildInputs nativeBuildInputs preConfigure configureFlags;

LDFLAGS = stdenv.lib.optionalString (!stdenv.isDarwin) "-lgcc_s";
inherit (mkPaths buildInputs) C_INCLUDE_PATH LIBRARY_PATH;
Expand All @@ -215,7 +227,7 @@ in stdenv.mkDerivation ({
''
# needed for some packages, especially packages that backport
# functionality to 2.x from 3.x
for item in $out/lib/python${majorVersion}/test/*; do
for item in $out/lib/${libPrefix}/test/*; do
if [[ "$item" != */test_support.py*
&& "$item" != */test/support
&& "$item" != */test/regrtest.py* ]]; then
Expand All @@ -224,9 +236,9 @@ in stdenv.mkDerivation ({
echo $item
fi
done
touch $out/lib/python${majorVersion}/test/__init__.py
ln -s $out/lib/python${majorVersion}/pdb.py $out/bin/pdb
ln -s $out/lib/python${majorVersion}/pdb.py $out/bin/pdb${majorVersion}
touch $out/lib/${libPrefix}/test/__init__.py
ln -s $out/lib/${libPrefix}/pdb.py $out/bin/pdb
ln -s $out/lib/${libPrefix}/pdb.py $out/bin/pdb${sourceVersion.major}.${sourceVersion.minor}
ln -s $out/share/man/man1/{python2.7.1.gz,python.1.gz}

# Python on Nix is not manylinux1 compatible. https://github.com/NixOS/nixpkgs/issues/18484
Expand All @@ -249,21 +261,7 @@ in stdenv.mkDerivation ({
cp libpython2.7.dll.a $out/lib
'';

passthru = let
pythonPackages = callPackage ../../../../../top-level/python-packages.nix {
python = self;
overrides = packageOverrides;
};
in rec {
inherit libPrefix sitePackages x11Support hasDistutilsCxxPatch ucsEncoding;
executable = libPrefix;
buildEnv = callPackage ../../wrapper.nix { python = self; inherit (pythonPackages) requiredPythonModules; };
withPackages = import ../../with-packages.nix { inherit buildEnv pythonPackages;};
pkgs = pythonPackages;
isPy2 = true;
isPy27 = true;
interpreter = "${self}/bin/${executable}";
};
inherit passthru;

enableParallelBuilding = true;

Expand Down
Loading