gh-115119: Switch Windows build to mpdecimal external (GH-115182)

This includes adding what should be a relatively temporary `Modules/_decimal/windows/mpdecimal.h` shim to choose between `mpdecimal32vc.h` or `mpdecimal64vc.h` based on which of `CONFIG_64` or `CONFIG_32` is defined.
python · Mar 18, 2024 · 849e071 · 849e071
1 parent 7707b14
commit 849e071
Show file tree

Hide file tree

Showing 7 changed files with 117 additions and 63 deletions.
diff --git a/Misc/NEWS.d/next/Windows/2024-02-08-14-48-15.gh-issue-115119.qMt32O.rst b/Misc/NEWS.d/next/Windows/2024-02-08-14-48-15.gh-issue-115119.qMt32O.rst
@@ -0,0 +1,3 @@
+Switched from vendored ``libmpdecimal`` code to a separately-hosted external
+package in the ``cpython-source-deps`` repository when building the
+``_decimal`` module.
diff --git a/Misc/externals.spdx.json b/Misc/externals.spdx.json
@@ -43,6 +43,28 @@
       "primaryPackagePurpose": "SOURCE",
       "versionInfo": "3.4.4"
     },
+    {
+      "SPDXID": "SPDXRef-PACKAGE-mpdecimal",
+      "checksums": [
+        {
+          "algorithm": "SHA256",
+          "checksumValue": "93118043651ffa33dcaaab445bae4f8929fca25d2d749079b78e97f220c3d8b1"
+        }
+      ],
+      "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/mpdecimal-2.5.1.tar.gz",
+      "externalRefs": [
+        {
+          "referenceCategory": "SECURITY",
+          "referenceLocator": "cpe:2.3:a:bytereef:mpdecimal:2.5.1:*:*:*:*:*:*:*",
+          "referenceType": "cpe23Type"
+        }
+      ],
+      "licenseConcluded": "NOASSERTION",
+      "name": "mpdecimal",
+      "originator": "Organization: bytereef.org",
+      "primaryPackagePurpose": "SOURCE",
+      "versionInfo": "2.5.1"
+    },
     {
       "SPDXID": "SPDXRef-PACKAGE-openssl",
       "checksums": [

diff --git a/Modules/_decimal/windows/mpdecimal.h b/Modules/_decimal/windows/mpdecimal.h
@@ -0,0 +1,17 @@
+/* Windows mpdecimal.h shim
+ *
+ * Generally, the mpdecimal library build will copy the correct header into
+ * place named "mpdecimal.h", but since we're building it ourselves directly
+ * into _decimal.pyd, we need to pick the right one.
+ *
+ * */
+
+#if defined(_MSC_VER)
+  #if defined(CONFIG_64)
+    #include <mpdecimal64vc.h>
+  #elif defined(CONFIG_32)
+    #include <mpdecimal32vc.h>
+  #else
+    #error "Unknown configuration!"
+  #endif
+#endif
diff --git a/PCbuild/_decimal.vcxproj b/PCbuild/_decimal.vcxproj
@@ -93,51 +93,55 @@
   </PropertyGroup>
   <ItemDefinitionGroup>
     <ClCompile>
-      <PreprocessorDefinitions>_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>BUILD_LIBMPDEC;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <PreprocessorDefinitions Condition="'$(Platform)' == 'Win32'">CONFIG_32;PPRO;MASM;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <PreprocessorDefinitions Condition="'$(Platform)'=='ARM'">CONFIG_32;ANSI;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <PreprocessorDefinitions Condition="'$(Platform)'=='ARM64'">CONFIG_64;ANSI;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <PreprocessorDefinitions Condition="'$(Platform)' == 'x64'">CONFIG_64;MASM;%(PreprocessorDefinitions)</PreprocessorDefinitions>
-      <AdditionalIncludeDirectories>..\Modules\_decimal;..\Modules\_decimal\libmpdec;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <AdditionalIncludeDirectories>..\Modules\_decimal;..\Modules\_decimal\windows;$(mpdecimalDir)\libmpdec;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
     </ClCompile>
   </ItemDefinitionGroup>
   <ItemGroup>
-    <ClInclude Include="..\Modules\_decimal\libmpdec\basearith.h" />
-    <ClInclude Include="..\Modules\_decimal\libmpdec\bits.h" />
-    <ClInclude Include="..\Modules\_decimal\libmpdec\constants.h" />
-    <ClInclude Include="..\Modules\_decimal\libmpdec\convolute.h" />
-    <ClInclude Include="..\Modules\_decimal\libmpdec\crt.h" />
-    <ClInclude Include="..\Modules\_decimal\libmpdec\difradix2.h" />
+    <ClInclude Include="$(mpdecimalDir)\libmpdec\basearith.h" />
+    <ClInclude Include="$(mpdecimalDir)\libmpdec\bits.h" />
+    <ClInclude Include="$(mpdecimalDir)\libmpdec\constants.h" />
+    <ClInclude Include="$(mpdecimalDir)\libmpdec\convolute.h" />
+    <ClInclude Include="$(mpdecimalDir)\libmpdec\crt.h" />
+    <ClInclude Include="$(mpdecimalDir)\libmpdec\difradix2.h" />
     <ClInclude Include="..\Modules\_decimal\docstrings.h" />
-    <ClInclude Include="..\Modules\_decimal\libmpdec\fnt.h" />
-    <ClInclude Include="..\Modules\_decimal\libmpdec\fourstep.h" />
-    <ClInclude Include="..\Modules\_decimal\libmpdec\mpdecimal.h" />
-    <ClInclude Include="..\Modules\_decimal\libmpdec\numbertheory.h" />
-    <ClInclude Include="..\Modules\_decimal\libmpdec\sixstep.h" />
-    <ClInclude Include="..\Modules\_decimal\libmpdec\transpose.h" />
-    <ClInclude Include="..\Modules\_decimal\libmpdec\typearith.h" />
-    <ClInclude Include="..\Modules\_decimal\libmpdec\umodarith.h" />
-    <ClInclude Include="..\Modules\_decimal\libmpdec\vccompat.h" />
+    <ClInclude Include="$(mpdecimalDir)\libmpdec\fnt.h" />
+    <ClInclude Include="$(mpdecimalDir)\libmpdec\fourstep.h" />
+    <ClInclude Include="..\Modules\_decimal\windows\mpdecimal.h" />
+    <ClInclude Condition="'$(Platform)' == 'Win32'" Include="$(mpdecimalDir)\libmpdec\mpdecimal32vc.h" />
+    <ClInclude Condition="'$(Platform)' == 'ARM'" Include="$(mpdecimalDir)\libmpdec\mpdecimal32vc.h" />
+    <ClInclude Condition="'$(Platform)' == 'ARM64'" Include="$(mpdecimalDir)\libmpdec\mpdecimal64vc.h" />
+    <ClInclude Condition="'$(Platform)' == 'x64'" Include="$(mpdecimalDir)\libmpdec\mpdecimal64vc.h" />
+    <ClInclude Include="$(mpdecimalDir)\libmpdec\numbertheory.h" />
+    <ClInclude Include="$(mpdecimalDir)\libmpdec\sixstep.h" />
+    <ClInclude Include="$(mpdecimalDir)\libmpdec\transpose.h" />
+    <ClInclude Include="$(mpdecimalDir)\libmpdec\typearith.h" />
+    <ClInclude Include="$(mpdecimalDir)\libmpdec\umodarith.h" />
+    <ClInclude Include="$(mpdecimalDir)\libmpdec\vccompat.h" />
   </ItemGroup>
   <ItemGroup>
     <ClCompile Include="..\Modules\_decimal\_decimal.c" />
-    <ClCompile Include="..\Modules\_decimal\libmpdec\basearith.c" />
-    <ClCompile Include="..\Modules\_decimal\libmpdec\constants.c" />
-    <ClCompile Include="..\Modules\_decimal\libmpdec\context.c" />
-    <ClCompile Include="..\Modules\_decimal\libmpdec\convolute.c" />
-    <ClCompile Include="..\Modules\_decimal\libmpdec\crt.c" />
-    <ClCompile Include="..\Modules\_decimal\libmpdec\difradix2.c" />
-    <ClCompile Include="..\Modules\_decimal\libmpdec\fnt.c" />
-    <ClCompile Include="..\Modules\_decimal\libmpdec\fourstep.c" />
-    <ClCompile Include="..\Modules\_decimal\libmpdec\io.c" />
-    <ClCompile Include="..\Modules\_decimal\libmpdec\mpalloc.c" />
-    <ClCompile Include="..\Modules\_decimal\libmpdec\mpdecimal.c" />
-    <ClCompile Include="..\Modules\_decimal\libmpdec\numbertheory.c" />
-    <ClCompile Include="..\Modules\_decimal\libmpdec\sixstep.c" />
-    <ClCompile Include="..\Modules\_decimal\libmpdec\transpose.c" />
+    <ClCompile Include="$(mpdecimalDir)\libmpdec\basearith.c" />
+    <ClCompile Include="$(mpdecimalDir)\libmpdec\constants.c" />
+    <ClCompile Include="$(mpdecimalDir)\libmpdec\context.c" />
+    <ClCompile Include="$(mpdecimalDir)\libmpdec\convolute.c" />
+    <ClCompile Include="$(mpdecimalDir)\libmpdec\crt.c" />
+    <ClCompile Include="$(mpdecimalDir)\libmpdec\difradix2.c" />
+    <ClCompile Include="$(mpdecimalDir)\libmpdec\fnt.c" />
+    <ClCompile Include="$(mpdecimalDir)\libmpdec\fourstep.c" />
+    <ClCompile Include="$(mpdecimalDir)\libmpdec\io.c" />
+    <ClCompile Include="$(mpdecimalDir)\libmpdec\mpalloc.c" />
+    <ClCompile Include="$(mpdecimalDir)\libmpdec\mpdecimal.c" />
+    <ClCompile Include="$(mpdecimalDir)\libmpdec\numbertheory.c" />
+    <ClCompile Include="$(mpdecimalDir)\libmpdec\sixstep.c" />
+    <ClCompile Include="$(mpdecimalDir)\libmpdec\transpose.c" />
   </ItemGroup>
   <ItemGroup>
-    <CustomBuild Include="..\Modules\_decimal\libmpdec\vcdiv64.asm">
+    <CustomBuild Include="$(mpdecimalDir)\libmpdec\vcdiv64.asm">
       <ExcludedFromBuild Condition="'$(Platform)'=='Win32'">true</ExcludedFromBuild>
       <ExcludedFromBuild Condition="'$(Platform)'=='ARM'">true</ExcludedFromBuild>
       <ExcludedFromBuild Condition="'$(Platform)'=='ARM64'">true</ExcludedFromBuild>

diff --git a/PCbuild/_decimal.vcxproj.filters b/PCbuild/_decimal.vcxproj.filters
@@ -21,96 +21,102 @@
     <ClInclude Include="..\Modules\_decimal\docstrings.h">
       <Filter>Header Files</Filter>
     </ClInclude>
-    <ClInclude Include="..\Modules\_decimal\libmpdec\basearith.h">
+    <ClInclude Include="$(mpdecimalDir)\libmpdec\basearith.h">
       <Filter>Header Files\libmpdec</Filter>
     </ClInclude>
-    <ClInclude Include="..\Modules\_decimal\libmpdec\bits.h">
+    <ClInclude Include="$(mpdecimalDir)\libmpdec\bits.h">
       <Filter>Header Files\libmpdec</Filter>
     </ClInclude>
-    <ClInclude Include="..\Modules\_decimal\libmpdec\constants.h">
+    <ClInclude Include="$(mpdecimalDir)\libmpdec\constants.h">
       <Filter>Header Files\libmpdec</Filter>
     </ClInclude>
-    <ClInclude Include="..\Modules\_decimal\libmpdec\convolute.h">
+    <ClInclude Include="$(mpdecimalDir)\libmpdec\convolute.h">
       <Filter>Header Files\libmpdec</Filter>
     </ClInclude>
-    <ClInclude Include="..\Modules\_decimal\libmpdec\crt.h">
+    <ClInclude Include="$(mpdecimalDir)\libmpdec\crt.h">
       <Filter>Header Files\libmpdec</Filter>
     </ClInclude>
-    <ClInclude Include="..\Modules\_decimal\libmpdec\difradix2.h">
+    <ClInclude Include="$(mpdecimalDir)\libmpdec\difradix2.h">
       <Filter>Header Files\libmpdec</Filter>
     </ClInclude>
-    <ClInclude Include="..\Modules\_decimal\libmpdec\fnt.h">
+    <ClInclude Include="$(mpdecimalDir)\libmpdec\fnt.h">
       <Filter>Header Files\libmpdec</Filter>
     </ClInclude>
-    <ClInclude Include="..\Modules\_decimal\libmpdec\fourstep.h">
+    <ClInclude Include="$(mpdecimalDir)\libmpdec\fourstep.h">
       <Filter>Header Files\libmpdec</Filter>
     </ClInclude>
-    <ClInclude Include="..\Modules\_decimal\libmpdec\mpdecimal.h">
+    <ClInclude Include="..\Modules\_decimal\windows\mpdecimal.h">
       <Filter>Header Files\libmpdec</Filter>
     </ClInclude>
-    <ClInclude Include="..\Modules\_decimal\libmpdec\numbertheory.h">
+    <ClInclude Include="$(mpdecimalDir)\libmpdec\mpdecimal32vc.h">
       <Filter>Header Files\libmpdec</Filter>
     </ClInclude>
-    <ClInclude Include="..\Modules\_decimal\libmpdec\sixstep.h">
+    <ClInclude Include="$(mpdecimalDir)\libmpdec\mpdecimal64vc.h">
       <Filter>Header Files\libmpdec</Filter>
     </ClInclude>
-    <ClInclude Include="..\Modules\_decimal\libmpdec\transpose.h">
+    <ClInclude Include="$(mpdecimalDir)\libmpdec\numbertheory.h">
       <Filter>Header Files\libmpdec</Filter>
     </ClInclude>
-    <ClInclude Include="..\Modules\_decimal\libmpdec\typearith.h">
+    <ClInclude Include="$(mpdecimalDir)\libmpdec\sixstep.h">
       <Filter>Header Files\libmpdec</Filter>
     </ClInclude>
-    <ClInclude Include="..\Modules\_decimal\libmpdec\umodarith.h">
+    <ClInclude Include="$(mpdecimalDir)\libmpdec\transpose.h">
       <Filter>Header Files\libmpdec</Filter>
     </ClInclude>
-    <ClInclude Include="..\Modules\_decimal\libmpdec\vccompat.h">
+    <ClInclude Include="$(mpdecimalDir)\libmpdec\typearith.h">
+      <Filter>Header Files\libmpdec</Filter>
+    </ClInclude>
+    <ClInclude Include="$(mpdecimalDir)\libmpdec\umodarith.h">
+      <Filter>Header Files\libmpdec</Filter>
+    </ClInclude>
+    <ClInclude Include="$(mpdecimalDir)\libmpdec\vccompat.h">
       <Filter>Header Files\libmpdec</Filter>
     </ClInclude>
   </ItemGroup>
   <ItemGroup>
     <ClCompile Include="..\Modules\_decimal\_decimal.c">
       <Filter>Source Files</Filter>
     </ClCompile>
-    <ClCompile Include="..\Modules\_decimal\libmpdec\basearith.c">
+    <ClCompile Include="$(mpdecimalDir)\libmpdec\basearith.c">
       <Filter>Source Files\libmpdec</Filter>
     </ClCompile>
-    <ClCompile Include="..\Modules\_decimal\libmpdec\constants.c">
+    <ClCompile Include="$(mpdecimalDir)\libmpdec\constants.c">
       <Filter>Source Files\libmpdec</Filter>
     </ClCompile>
-    <ClCompile Include="..\Modules\_decimal\libmpdec\context.c">
+    <ClCompile Include="$(mpdecimalDir)\libmpdec\context.c">
       <Filter>Source Files\libmpdec</Filter>
     </ClCompile>
-    <ClCompile Include="..\Modules\_decimal\libmpdec\convolute.c">
+    <ClCompile Include="$(mpdecimalDir)\libmpdec\convolute.c">
       <Filter>Source Files\libmpdec</Filter>
     </ClCompile>
-    <ClCompile Include="..\Modules\_decimal\libmpdec\crt.c">
+    <ClCompile Include="$(mpdecimalDir)\libmpdec\crt.c">
       <Filter>Source Files\libmpdec</Filter>
     </ClCompile>
-    <ClCompile Include="..\Modules\_decimal\libmpdec\difradix2.c">
+    <ClCompile Include="$(mpdecimalDir)\libmpdec\difradix2.c">
       <Filter>Source Files\libmpdec</Filter>
     </ClCompile>
-    <ClCompile Include="..\Modules\_decimal\libmpdec\fnt.c">
+    <ClCompile Include="$(mpdecimalDir)\libmpdec\fnt.c">
       <Filter>Source Files\libmpdec</Filter>
     </ClCompile>
-    <ClCompile Include="..\Modules\_decimal\libmpdec\fourstep.c">
+    <ClCompile Include="$(mpdecimalDir)\libmpdec\fourstep.c">
       <Filter>Source Files\libmpdec</Filter>
     </ClCompile>
-    <ClCompile Include="..\Modules\_decimal\libmpdec\io.c">
+    <ClCompile Include="$(mpdecimalDir)\libmpdec\io.c">
       <Filter>Source Files\libmpdec</Filter>
     </ClCompile>
-    <ClCompile Include="..\Modules\_decimal\libmpdec\mpalloc.c">
+    <ClCompile Include="$(mpdecimalDir)\libmpdec\mpalloc.c">
       <Filter>Source Files\libmpdec</Filter>
     </ClCompile>
-    <ClCompile Include="..\Modules\_decimal\libmpdec\mpdecimal.c">
+    <ClCompile Include="$(mpdecimalDir)\libmpdec\mpdecimal.c">
       <Filter>Source Files\libmpdec</Filter>
     </ClCompile>
-    <ClCompile Include="..\Modules\_decimal\libmpdec\numbertheory.c">
+    <ClCompile Include="$(mpdecimalDir)\libmpdec\numbertheory.c">
       <Filter>Source Files\libmpdec</Filter>
     </ClCompile>
-    <ClCompile Include="..\Modules\_decimal\libmpdec\sixstep.c">
+    <ClCompile Include="$(mpdecimalDir)\libmpdec\sixstep.c">
       <Filter>Source Files\libmpdec</Filter>
     </ClCompile>
-    <ClCompile Include="..\Modules\_decimal\libmpdec\transpose.c">
+    <ClCompile Include="$(mpdecimalDir)\libmpdec\transpose.c">
       <Filter>Source Files\libmpdec</Filter>
     </ClCompile>
   </ItemGroup>
@@ -120,8 +126,8 @@
     </ResourceCompile>
   </ItemGroup>
   <ItemGroup>
-    <CustomBuild Include="..\Modules\_decimal\libmpdec\vcdiv64.asm">
+    <CustomBuild Include="$(mpdecimalDir)\libmpdec\vcdiv64.asm">
       <Filter>Source Files\libmpdec</Filter>
     </CustomBuild>
   </ItemGroup>
-</Project>
+</Project>
diff --git a/PCbuild/get_externals.bat b/PCbuild/get_externals.bat
@@ -54,6 +54,7 @@ set libraries=
 set libraries=%libraries%                                       bzip2-1.0.8
 if NOT "%IncludeLibffiSrc%"=="false" set libraries=%libraries%  libffi-3.4.4
 if NOT "%IncludeSSLSrc%"=="false" set libraries=%libraries%     openssl-3.0.13
+set libraries=%libraries%                                       mpdecimal-2.5.1
 set libraries=%libraries%                                       sqlite-3.45.1.0
 if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tcl-core-8.6.13.1
 if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tk-8.6.13.1

diff --git a/PCbuild/python.props b/PCbuild/python.props
@@ -74,6 +74,7 @@
     <libffiDir Condition="$(libffiDir) == ''">$(ExternalsDir)libffi-3.4.4\</libffiDir>
     <libffiOutDir Condition="$(libffiOutDir) == ''">$(libffiDir)$(ArchName)\</libffiOutDir>
     <libffiIncludeDir Condition="$(libffiIncludeDir) == ''">$(libffiOutDir)include</libffiIncludeDir>
+    <mpdecimalDir Condition="$(mpdecimalDir) == ''">$(ExternalsDir)\mpdecimal-2.5.1\</mpdecimalDir>
     <opensslDir Condition="$(opensslDir) == ''">$(ExternalsDir)openssl-3.0.13\</opensslDir>
     <opensslOutDir Condition="$(opensslOutDir) == ''">$(ExternalsDir)openssl-bin-3.0.13\$(ArchName)\</opensslOutDir>
     <opensslIncludeDir Condition="$(opensslIncludeDir) == ''">$(opensslOutDir)include</opensslIncludeDir>