Revert static linking with stdc++ (#824)

Motiviation: In the past we removed the usage of apr by static linking against stdc++. While at first this seemed like a good idea it has proven to introduce problems on various platforms. Because of these problems we should better revert the changes and just link against apr again. Modifications: - Revert "Fix memory leak when creating and destroying a lot of context (#790)" fd8fb2a. - Revert "Fix possible corruption / segfault when using atomics (#774)" 11ff708. - Revert "Fix segfault which was caused by not creating the ticket_keys_new atomic (#772)" 196f935. - Revert "Fix format usage" eecaaa8. - Revert "Link libstdc++ statically on linux" a288f03. - Revert "rename functions for consistency" 7ee7804. - Revert "Remove dependency on apr by using c++11 / c++14 features" 8740651. - Update APR version Result: Fixes #798 , #789
netty · Sep 21, 2023 · 33b2338 · 33b2338
1 parent 624570f
commit 33b2338
Show file tree

Hide file tree

Showing 41 changed files with 2,050 additions and 599 deletions.
diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
@@ -71,7 +71,7 @@ jobs:
     #    uses a compiled language
 
     - name: Install tools / libraries 
-      run: sudo apt-get update && sudo apt-get -y install autoconf automake libtool libtool-bin make tar libssl-dev cmake perl ninja-build
+      run: sudo apt-get update && sudo apt-get -y install autoconf automake libtool libtool-bin make tar libapr1-dev libssl-dev cmake perl ninja-build
 
     - name: Build project
       run: ./mvnw clean package -pl openssl-dynamic -DskipTests=true

diff --git a/Brewfile b/Brewfile
@@ -6,3 +6,4 @@ brew 'perl'
 brew 'ninja'
 brew 'golang'
 brew 'cmake'
+brew 'apr'
diff --git a/boringssl-static/pom.xml b/boringssl-static/pom.xml
@@ -27,7 +27,7 @@
   <name>Netty/TomcatNative [BoringSSL - Static]</name>
   <description>
     A Mavenized fork of Tomcat Native which incorporates various patches. This artifact is statically linked
-    to BoringSSL.
+    to BoringSSL and Apache APR.
   </description>
 
   <build>
@@ -130,6 +130,7 @@
             <artifactId>maven-bundle-plugin</artifactId>
             <configuration>
               <instructions>
+                <Apr-Version>${aprVersion}</Apr-Version>
                 <BoringSSL-Revision>${boringsslCommitSha}</BoringSSL-Revision>
                 <BoringSSL-Branch>${boringsslBranch}</BoringSSL-Branch>
               </instructions>
@@ -317,7 +318,7 @@
             </executions>
           </plugin>
 
-          <!-- Configure the distribution statically linked against OpenSSL -->
+          <!-- Configure the distribution statically linked against OpenSSL and APR -->
           <plugin>
             <groupId>org.fusesource.hawtjni</groupId>
             <artifactId>hawtjni-maven-plugin</artifactId>
@@ -340,6 +341,7 @@
                   <!-- <verbose>true</verbose> -->
                   <configureArgs>
                     <configureArg>--with-ssl=no</configureArg>
+                    <configureArg>--with-apr=${aprHome}</configureArg>
                     <configureArg>--with-static-libs</configureArg>
                     <configureArg>--libdir=${project.build.directory}/native-build/target/lib</configureArg>
                     <configureArg>${macOsxDeploymentTarget}</configureArg>
@@ -417,6 +419,10 @@
                       </files>
                       <content>release 7.6</content>
                     </requireFilesContent>
+                    <requireProperty>
+                      <property>aprArmHome</property>
+                      <message>The folder of APR for aarch64 must be specified by hand. Please try -DaprArmHome=</message>
+                    </requireProperty>
                   </rules>
                   <ignoreCache>true</ignoreCache>
                 </configuration>
@@ -469,6 +475,7 @@
             <artifactId>maven-bundle-plugin</artifactId>
             <configuration>
               <instructions>
+                <Apr-Version>${aprVersion}</Apr-Version>
                 <BoringSSL-Revision>${boringsslCommitSha}</BoringSSL-Revision>
                 <BoringSSL-Branch>${boringsslBranch}</BoringSSL-Branch>
               </instructions>
@@ -645,7 +652,7 @@
             </executions>
           </plugin>
 
-          <!-- Configure the distribution statically linked against OpenSSL -->
+          <!-- Configure the distribution statically linked against OpenSSL and APR -->
           <plugin>
             <groupId>org.fusesource.hawtjni</groupId>
             <artifactId>hawtjni-maven-plugin</artifactId>
@@ -668,14 +675,14 @@
                   <!-- <verbose>true</verbose> -->
                   <configureArgs>
                     <configureArg>--with-ssl=no</configureArg>
+                    <configureArg>--with-apr=${aprArmHome}</configureArg>
                     <configureArg>--with-static-libs</configureArg>
                     <configureArg>--libdir=${project.build.directory}/native-build/target/lib</configureArg>
                     <configureArg>CFLAGS=-O3 -Werror -fno-omit-frame-pointer -fvisibility=hidden -Wunused -Wno-unused-value</configureArg>
                     <configureArg>CPPFLAGS=-DHAVE_OPENSSL -I${boringsslSourceDir}/include</configureArg>
                     <configureArg>LDFLAGS=-L${boringsslHome}/ssl -L${boringsslHome}/crypto -lssl -lcrypto</configureArg>
                     <configureArg>--host=aarch64-linux-gnu</configureArg>
                     <configureArg>CC=aarch64-none-linux-gnu-gcc</configureArg>
-                    <configureArg>CXX=aarch64-none-linux-gnu-g++</configureArg>
                   </configureArgs>
                 </configuration>
               </execution>
@@ -928,6 +935,7 @@
             <artifactId>maven-bundle-plugin</artifactId>
             <configuration>
               <instructions>
+                <Apr-Version>${aprVersion}</Apr-Version>
                 <BoringSSL-Revision>${boringsslCommitSha}</BoringSSL-Revision>
                 <BoringSSL-Branch>${boringsslBranch}</BoringSSL-Branch>
               </instructions>
@@ -1063,7 +1071,7 @@
             </executions>
           </plugin>
 
-          <!-- Configure the distribution statically linked against OpenSSL -->
+          <!-- Configure the distribution statically linked against OpenSSL and APR -->
           <plugin>
             <groupId>org.fusesource.hawtjni</groupId>
             <artifactId>hawtjni-maven-plugin</artifactId>
@@ -1085,6 +1093,7 @@
                   <!-- <verbose>true</verbose> -->
                   <configureArgs>
                     <configureArg>--with-ssl=no</configureArg>
+                    <configureArg>--with-apr=${aprHome}</configureArg>
                     <configureArg>--with-static-libs</configureArg>
                     <configureArg>--libdir=${project.build.directory}/native-build/target/lib</configureArg>
                     <configureArg>--host=aarch64-apple-darwin</configureArg>
@@ -1173,6 +1182,7 @@
             <artifactId>maven-bundle-plugin</artifactId>
             <configuration>
               <instructions>
+                <Apr-Version>${aprVersion}</Apr-Version>
                 <BoringSSL-Revision>${boringsslCommitSha}</BoringSSL-Revision>
                 <BoringSSL-Branch>${boringsslBranch}</BoringSSL-Branch>
               </instructions>
@@ -1308,7 +1318,7 @@
             </executions>
           </plugin>
 
-          <!-- Configure the distribution statically linked against OpenSSL -->
+          <!-- Configure the distribution statically linked against OpenSSL and APR -->
           <plugin>
             <groupId>org.fusesource.hawtjni</groupId>
             <artifactId>hawtjni-maven-plugin</artifactId>
@@ -1330,6 +1340,7 @@
                   <!-- <verbose>true</verbose> -->
                   <configureArgs>
                     <configureArg>--with-ssl=no</configureArg>
+                    <configureArg>--with-apr=${aprHome}</configureArg>
                     <configureArg>--with-static-libs</configureArg>
                     <configureArg>--libdir=${project.build.directory}/native-build/target/lib</configureArg>
                     <configureArg>--host=x86_64-apple-darwin</configureArg>

diff --git a/docker/Dockerfile.arch b/docker/Dockerfile.arch
@@ -6,6 +6,7 @@ ENV JAVA_VERSION $java_version
 # install dependencies
 # use openSSL 1.0.x for now, for highest compatibility
 RUN pacman -Sy --noconfirm --needed \
+ apr \
  autoconf \
  automake \
  bzip2 \

diff --git a/docker/Dockerfile.centos6 b/docker/Dockerfile.centos6
@@ -12,12 +12,14 @@ RUN sed -i -e 's/^mirrorlist/#mirrorlist/g' -e 's/^#baseurl=http:\/\/mirror.cent
 
 # install dependencies
 RUN yum install -y \
+ apr-devel \
  autoconf \
  automake \
  bzip2 \
  git \
  glibc-devel \
  gnupg \
+ libapr1-dev \
  libtool \
  lsb-core \
  make \

diff --git a/docker/Dockerfile.cross_compile_aarch64 b/docker/Dockerfile.cross_compile_aarch64
@@ -2,10 +2,12 @@ FROM centos:7.6.1810
 
 ARG gcc_version=10.2-2020.11
 ARG openssl_version=1_1_1d
+ARG apr_version=1.7.4
 ENV SOURCE_DIR /root/source
 ENV GCC_VERSION $gcc_version
 ENV OPENSSL_VERSION $openssl_version
 ENV MAVEN_VERSION 3.9.1
+ENV APR_VERSION $apr_version
 
 RUN mkdir $SOURCE_DIR
 WORKDIR $SOURCE_DIR
@@ -32,6 +34,32 @@ ENV PATH="/gcc-arm-$GCC_VERSION-x86_64-aarch64-none-linux-gnu/bin:${PATH}"
 # Install CMake
 RUN yum install -y cmake3 && ln -s /usr/bin/cmake3 /usr/bin/cmake
 
+# Cross compile Apache Apr for aarch64 - static
+RUN set -x && \
+  wget https://downloads.apache.org//apr/apr-$APR_VERSION.tar.gz && \
+  tar xvf apr-$APR_VERSION.tar.gz && \
+  pushd apr-$APR_VERSION && \
+  CC=aarch64-none-linux-gnu-gcc CFLAGS='-O3 -fno-omit-frame-pointer -fPIC' ./configure --disable-shared --prefix=/opt/apr-$APR_VERSION-static --host=aarch64-none-linux-gnu ac_cv_file__dev_zero=yes ac_cv_func_setpgrp_void=yes apr_cv_tcp_nodelay_with_cork=yes ac_cv_sizeof_struct_iovec=8 && \
+  make || true && \
+  pushd tools && \
+  gcc -Wall -O2 -DCROSS_COMPILE gen_test_char.c -s -o gen_test_char && \
+  popd && \
+  make && make install && \
+  popd
+
+# Cross compile Apache Apr for aarch64 - share
+RUN set -x && \
+  wget https://downloads.apache.org//apr/apr-$APR_VERSION.tar.gz && \
+  tar xvf apr-$APR_VERSION.tar.gz && \
+  pushd apr-$APR_VERSION && \
+  CC=aarch64-none-linux-gnu-gcc CFLAGS='-O3 -fno-omit-frame-pointer -fPIC' ./configure --prefix=/opt/apr-$APR_VERSION-share --host=aarch64-none-linux-gnu ac_cv_file__dev_zero=yes ac_cv_func_setpgrp_void=yes apr_cv_tcp_nodelay_with_cork=yes ac_cv_sizeof_struct_iovec=8 && \
+  make || true && \
+  pushd tools && \
+  gcc -Wall -O2 -DCROSS_COMPILE gen_test_char.c -s -o gen_test_char && \
+  popd && \
+  make && make install && \
+  popd
+
 # Cross compile OpenSSL for aarch64 - share
 RUN set -x && \
   wget https://github.com/openssl/openssl/archive/OpenSSL_$OPENSSL_VERSION.tar.gz && \

diff --git a/docker/Dockerfile.debian b/docker/Dockerfile.debian
@@ -24,6 +24,7 @@ RUN echo "deb http://archive.debian.org/debian/ wheezy contrib main non-free" >
  git \
  gnupg \
  g++ \
+ libapr1-dev \
  libssl1.0.0=1.0.1e-2+deb7u20 \
  libssl-dev \
  libtool \

diff --git a/docker/Dockerfile.opensuse b/docker/Dockerfile.opensuse
@@ -9,6 +9,7 @@ ENV JAVA_VERSION $java_version
 # install dependencies
 # use openSSL 1.0.x for now, for highest compatibility
 RUN zypper install --force-resolution --no-recommends --no-confirm \
+ apr-devel \
  autoconf \
  automake \
  bzip2 \

diff --git a/docker/docker-compose.centos-7.yaml b/docker/docker-compose.centos-7.yaml
@@ -9,6 +9,7 @@ services:
       dockerfile: docker/Dockerfile.cross_compile_aarch64
       args:
         gcc_version: "10.2-2020.11"
+        apr_version: "1.7.4"
         openssl_version: "1_1_1k"
 
   cross-compile-aarch64-common: &cross-compile-aarch64-common
@@ -39,7 +40,7 @@ services:
 
   cross-compile-aarch64-build:
     <<: *cross-compile-aarch64-common
-    command: /bin/bash -cl "./mvnw clean package -Plinux-aarch64 -am -pl openssl-dynamic -DopensslArmHome=/opt/openssl-$$OPENSSL_VERSION-share -DskipTests && ./mvnw clean package -Plinux-aarch64 -am -pl boringssl-static -DboringsslSourceDir=/root/workspace/boringssl-source -DboringsslHome=/root/workspace/boringssl -DskipTests"
+    command: /bin/bash -cl "./mvnw clean package -Plinux-aarch64 -am -pl openssl-dynamic  -DaprArmHome=/opt/apr-$$APR_VERSION-share -DopensslArmHome=/opt/openssl-$$OPENSSL_VERSION-share -DskipTests && ./mvnw clean package -Plinux-aarch64 -am -pl boringssl-static -DaprArmHome=/opt/apr-$$APR_VERSION-static -DboringsslSourceDir=/root/workspace/boringssl-source -DboringsslHome=/root/workspace/boringssl -DskipTests"
 
   cross-compile-aarch64-deploy:
     <<: *cross-compile-aarch64-common
@@ -49,7 +50,7 @@ services:
       - ~/.m2/repository:/root/.m2/repository
       - ~/.m2/settings.xml:/root/.m2/settings.xml
       - ..:/code
-    command: /bin/bash -cl "./mvnw clean deploy -Plinux-aarch64 -am -pl openssl-dynamic -DopensslArmHome=/opt/openssl-$$OPENSSL_VERSION-share -DskipTests && ./mvnw clean deploy -Plinux-aarch64 -am -pl boringssl-static -DboringsslSourceDir=/root/workspace/boringssl-source -DboringsslHome=/root/workspace/boringssl -DskipTests"
+    command: /bin/bash -cl "./mvnw clean deploy -Plinux-aarch64 -am -pl openssl-dynamic -DaprArmHome=/opt/apr-$$APR_VERSION-share -DopensslArmHome=/opt/openssl-$$OPENSSL_VERSION-share -DskipTests && ./mvnw clean deploy -Plinux-aarch64 -am -pl boringssl-static -DaprArmHome=/opt/apr-$$APR_VERSION-static -DboringsslSourceDir=/root/workspace/boringssl-source -DboringsslHome=/root/workspace/boringssl -DskipTests"
 
   cross-compile-aarch64-stage-snapshot:
     <<: *cross-compile-aarch64-common
@@ -59,7 +60,7 @@ services:
       - ~/.m2/repository:/root/.m2/repository
       - ~/local-staging:/root/local-staging
       - ..:/code
-    command: /bin/bash -cl "./mvnw -Plinux-aarch64 -am -pl openssl-dynamic -DopensslArmHome=/opt/openssl-$$OPENSSL_VERSION-share clean package org.sonatype.plugins:nexus-staging-maven-plugin:deploy -DaltStagingDirectory=/root/local-staging -DskipRemoteStaging=true -DskipTests=true && ./mvnw -Plinux-aarch64 -am -pl boringssl-static -DboringsslSourceDir=/root/workspace/boringssl-source -DboringsslHome=/root/workspace/boringssl clean package org.sonatype.plugins:nexus-staging-maven-plugin:deploy -DaltStagingDirectory=/root/local-staging -DskipRemoteStaging=true -DskipTests=true"
+    command: /bin/bash -cl "./mvnw -Plinux-aarch64 -am -pl openssl-dynamic -DaprArmHome=/opt/apr-$$APR_VERSION-share -DopensslArmHome=/opt/openssl-$$OPENSSL_VERSION-share clean package org.sonatype.plugins:nexus-staging-maven-plugin:deploy -DaltStagingDirectory=/root/local-staging -DskipRemoteStaging=true -DskipTests=true && ./mvnw -Plinux-aarch64 -am -pl boringssl-static -DaprArmHome=/opt/apr-$$APR_VERSION-static -DboringsslSourceDir=/root/workspace/boringssl-source -DboringsslHome=/root/workspace/boringssl clean package org.sonatype.plugins:nexus-staging-maven-plugin:deploy -DaltStagingDirectory=/root/local-staging -DskipRemoteStaging=true -DskipTests=true"
 
   cross-compile-aarch64-stage-release:
     <<: *cross-compile-aarch64-common
@@ -69,4 +70,5 @@ services:
       - ~/.m2/settings.xml:/root/.m2/settings.xml
       - ~/local-staging:/root/local-staging
       - ..:/code
-    command: /bin/bash -cl "cat <(echo -e \"${GPG_PRIVATE_KEY}\") | gpg --batch --import && ./mvnw -Plinux-aarch64 -am -pl openssl-dynamic -DopensslArmHome=/opt/openssl-$$OPENSSL_VERSION-share clean javadoc:jar package gpg:sign org.sonatype.plugins:nexus-staging-maven-plugin:deploy -DnexusUrl=https://oss.sonatype.org -DserverId=sonatype-nexus-staging -DaltStagingDirectory=/root/local-staging -DskipRemoteStaging=true -DskipTests=true -Dgpg.passphrase=${GPG_PASSPHRASE} -Dgpg.keyname=${GPG_KEYNAME} && ./mvnw -Plinux-aarch64 -am -pl boringssl-static -DboringsslSourceDir=/root/workspace/boringssl-source -DboringsslHome=/root/workspace/boringssl clean javadoc:jar package gpg:sign org.sonatype.plugins:nexus-staging-maven-plugin:deploy -DnexusUrl=https://oss.sonatype.org -DserverId=sonatype-nexus-staging -DaltStagingDirectory=/root/local-staging -DskipRemoteStaging=true -DskipTests=true -Dgpg.passphrase=${GPG_PASSPHRASE} -Dgpg.keyname=${GPG_KEYNAME}"
+    command: /bin/bash -cl "cat <(echo -e \"${GPG_PRIVATE_KEY}\") | gpg --batch --import && ./mvnw -Plinux-aarch64 -am -pl openssl-dynamic -DaprArmHome=/opt/apr-$$APR_VERSION-share -DopensslArmHome=/opt/openssl-$$OPENSSL_VERSION-share clean javadoc:jar package gpg:sign org.sonatype.plugins:nexus-staging-maven-plugin:deploy -DnexusUrl=https://oss.sonatype.org -DserverId=sonatype-nexus-staging -DaltStagingDirectory=/root/local-staging -DskipRemoteStaging=true -DskipTests=true -Dgpg.passphrase=${GPG_PASSPHRASE} -Dgpg.keyname=${GPG_KEYNAME} && ./mvnw -Plinux-aarch64 -am -pl boringssl-static -DaprArmHome=/opt/apr-$$APR_VERSION-static -DboringsslSourceDir=/root/workspace/boringssl-source -DboringsslHome=/root/workspace/boringssl clean javadoc:jar package gpg:sign org.sonatype.plugins:nexus-staging-maven-plugin:deploy -DnexusUrl=https://oss.sonatype.org -DserverId=sonatype-nexus-staging -DaltStagingDirectory=/root/local-staging -DskipRemoteStaging=true -DskipTests=true -Dgpg.passphrase=${GPG_PASSPHRASE} -Dgpg.keyname=${GPG_KEYNAME}"
+
diff --git a/libressl-static/pom.xml b/libressl-static/pom.xml
@@ -27,7 +27,7 @@
   <name>Netty/TomcatNative [LibreSSL - Static]</name>
   <description>
     A Mavenized fork of Tomcat Native which incorporates various patches. This artifact is statically linked
-    to LibreSSL.
+    to LibreSSL and Apache APR.
   </description>
 
   <properties>
@@ -61,12 +61,14 @@
     </pluginManagement>
 
     <plugins>
+
       <!-- Add the LibreSSL version to the manifest. -->
       <plugin>
         <groupId>org.apache.felix</groupId>
         <artifactId>maven-bundle-plugin</artifactId>
         <configuration>
           <instructions>
+            <Apr-Version>${aprVersion}</Apr-Version>
             <LibreSSL-Version>${libresslVersion}</LibreSSL-Version>
           </instructions>
         </configuration>
@@ -159,7 +161,7 @@
         </executions>
       </plugin>
 
-      <!-- Configure the distribution statically linked against OpenSSL -->
+      <!-- Configure the distribution statically linked against OpenSSL and APR -->
       <plugin>
         <groupId>org.fusesource.hawtjni</groupId>
         <artifactId>hawtjni-maven-plugin</artifactId>
@@ -181,6 +183,7 @@
               <windowsBuildTool>msbuild</windowsBuildTool>
               <configureArgs>
                 <configureArg>--with-ssl=no</configureArg>
+                <configureArg>--with-apr=${aprHome}</configureArg>
                 <configureArg>--with-static-libs</configureArg>
                 <configureArg>--libdir=${project.build.directory}/native-build/target/lib</configureArg>
                 <configureArg>${macOsxDeploymentTarget}</configureArg>

diff --git a/openssl-classes/pom.xml b/openssl-classes/pom.xml
@@ -27,7 +27,7 @@
   <name>Netty/TomcatNative [OpenSSL - Classes]</name>
   <description>
     A Mavenized fork of Tomcat Native which incorporates various patches. This artifact is dynamically linked
-    to OpenSSL.
+    to OpenSSL and Apache APR.
   </description>
 
   <properties>

diff --git a/openssl-classes/src/main/java/io/netty/internal/tcnative/Library.java b/openssl-classes/src/main/java/io/netty/internal/tcnative/Library.java
@@ -154,6 +154,17 @@ private static String calculatePackagePrefix() {
         return maybeShaded.substring(0, maybeShaded.length() - expected.length());
     }
 
+    /* create global TCN's APR pool
+     * This has to be the first call to TCN library.
+     */
+    private static native boolean initialize0();
+
+    private static native boolean aprHasThreads();
+
+    private static native int aprMajorVersion();
+
+    /* APR_VERSION_STRING */
+    private static native String aprVersionString();
 
     /**
      * Calls {@link #initialize(String, String)} with {@code "provided"} and {@code null}.
@@ -176,8 +187,16 @@ public static boolean initialize() throws Exception {
     public static boolean initialize(String libraryName, String engine) throws Exception {
         if (_instance == null) {
             _instance = libraryName == null ? new Library() : new Library(libraryName);
+
+            if (aprMajorVersion() < 1) {
+                throw new UnsatisfiedLinkError("Unsupported APR Version (" +
+                                               aprVersionString() + ")");
+            }
+
+            if (!aprHasThreads()) {
+                throw new UnsatisfiedLinkError("Missing APR_HAS_THREADS");
+            }
         }
-        SSL.initialize(engine);
-        return true;
+        return initialize0() && SSL.initialize(engine) == 0;
     }
 }
diff --git a/openssl-classes/src/main/java/io/netty/internal/tcnative/SSL.java b/openssl-classes/src/main/java/io/netty/internal/tcnative/SSL.java
@@ -138,8 +138,9 @@ private SSL() { }
      *
      * @param engine Support for external a Crypto Device ("engine"),
      *                usually a hardware accelerator card for crypto operations.
+     * @return APR status code
      */
-    static native void initialize(String engine);
+    static native int initialize(String engine);
 
     /**
      * Initialize new in-memory BIO that is located in the secure heap.

diff --git a/openssl-classes/src/main/java/io/netty/internal/tcnative/SSLContext.java b/openssl-classes/src/main/java/io/netty/internal/tcnative/SSLContext.java
@@ -68,7 +68,7 @@ public static native long make(int protocol, int mode)
     /**
      * Free the resources used by the Context
      * @param ctx Server or Client context to free.
-     * @return {@code 0} on success.
+     * @return APR Status code.
      */
     public static native int free(long ctx);