test: integration-tests for transport compression

.github/workflows/ci.yml

@@ -53,6 +53,13 @@ jobs:
             CXX: clang++-15
             ERROR_ON_WARNINGS: 1
             RUN_ANALYZER: asan,llvm-cov
+          - name: Linux (clang + transport-compression + asan + llvm-cov)
+            os: ubuntu-22.04
+            CC: clang-15
+            CXX: clang++-15
+            ERROR_ON_WARNINGS: 1
+            TRANSPORT_COMPRESSION: 1
+            RUN_ANALYZER: asan,llvm-cov
           - name: Linux (clang + kcov)
             os: ubuntu-22.04
             CC: clang-15
@@ -78,16 +85,33 @@ jobs:
             ERROR_ON_WARNINGS: 1
             SYSTEM_VERSION_COMPAT: 0
             RUN_ANALYZER: asan,llvm-cov
+          - name: macOS (clang + transport-compression + asan + llvm-cov)
+            os: macOs-11
+            CC: clang
+            CXX: clang++
+            ERROR_ON_WARNINGS: 1
+            SYSTEM_VERSION_COMPAT: 0
+            TRANSPORT_COMPRESSION: 1
+            RUN_ANALYZER: asan,llvm-cov
           - name: Windows (old VS, 32-bit)
             os: windows-2019
             TEST_X86: 1
           - name: Windows (latest)
             os: windows-latest
+          - name: Windows (latest + transport-compression)
+            os: windows-latest
+            TRANSPORT_COMPRESSION: 1
           - name: LLVM-Mingw
             os: windows-latest
             TEST_MINGW: 1
             MINGW_PKG_PREFIX: x86_64-w64-mingw32
             MINGW_ASM_MASM_COMPILER: llvm-ml;-m64
+          - name: LLVM-Mingw (transport-compression)
+            os: windows-latest
+            TEST_MINGW: 1
+            MINGW_PKG_PREFIX: x86_64-w64-mingw32
+            MINGW_ASM_MASM_COMPILER: llvm-ml;-m64
+            TRANSPORT_COMPRESSION: 1
           # The Android emulator is currently only available on macos, see:
           # https://docs.microsoft.com/en-us/azure/devops/pipelines/ecosystems/android?view=azure-devops#test-on-the-android-emulator
           - name: Android (old API/NDK)
@@ -114,7 +138,7 @@ jobs:
       ANDROID_ARCH: ${{ matrix.ANDROID_ARCH }}
       CMAKE_DEFINES: ${{ matrix.CMAKE_DEFINES }}
       SYSTEM_VERSION_COMPAT: ${{ matrix.SYSTEM_VERSION_COMPAT }}
-
+      TRANSPORT_COMPRESSION: ${{ matrix.TRANSPORT_COMPRESSION }}
     steps:
       - uses: actions/checkout@v3
         with:
@@ -158,6 +182,11 @@ jobs:
           MINGW_ASM_MASM_COMPILER: ${{ matrix.MINGW_ASM_MASM_COMPILER }}
         run: . "scripts\install-llvm-mingw.ps1"
 
+      - name: Set up zlib for Windows
+        if: ${{ runner.os == 'Windows' }}
+        shell: powershell
+        run: . "scripts\install-zlib.ps1"
+
       - name: Installing Android SDK Dependencies
         if: ${{ env['ANDROID_API'] }}
         run: |

CHANGELOG.md

@@ -1,10 +1,19 @@
 # Changelog
 
-## 0.7.1
+## Unreleased
 
 **Features**:
 
 - Add optional Gzip transport compression via build option `SENTRY_TRANSPORT_COMPRESSION`. Requires system `zlib`. ([#954](https://github.com/getsentry/sentry-native/pull/954))
+
+**Thank you**:
+
+- [@Strive-Sun](https://github.com/Strive-Sun)
+
+## 0.7.1
+
+**Features**:
+
 - Add user feedback capability to the Native SDK ([#966](https://github.com/getsentry/sentry-native/pull/966))
 
 **Internal**:
@@ -15,10 +24,6 @@
 
 - Add usage of the breadcrumb `data` property to the example. [#951](https://github.com/getsentry/sentry-native/pull/951)
 
-**Thank you**:
-
-- [@Strive-Sun](https://github.com/Strive-Sun)
-
 ## 0.7.0
 
 **Breaking changes**:

scripts/install-llvm-mingw.ps1

@@ -26,10 +26,10 @@ New-Item -ItemType Directory -Force -Path "${LLVM_MINGW_INSTALL_PATH}"
 Expand-Archive -LiteralPath "${LLVM_MINGW_DL_PATH}" -DestinationPath "${LLVM_MINGW_INSTALL_PATH}"
 # Export the LLVM-mingw install path
 $LLVM_MINGW_INSTALL_PATH = "${LLVM_MINGW_INSTALL_PATH}\${LLVM_MINGW_PKG}"
-Write-Output "LLVM_MINGW_INSTALL_PATH=${LLVM_MINGW_INSTALL_PATH}" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
+"LLVM_MINGW_INSTALL_PATH=${LLVM_MINGW_INSTALL_PATH}" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
 # Prepend bin path to the system PATH
-Write-Output "Path to LLVM-mingw bin folder: ${LLVM_MINGW_INSTALL_PATH}\bin"
-Write-Output "${LLVM_MINGW_INSTALL_PATH}\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
+Write-Host "Path to LLVM-mingw bin folder: ${LLVM_MINGW_INSTALL_PATH}\bin"
+"${LLVM_MINGW_INSTALL_PATH}\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
 
 # Download ninja-build
 $NINJA_DL_URL = "https://github.com/ninja-build/ninja/releases/download/v1.11.1/ninja-win.zip"
@@ -49,36 +49,10 @@ Write-Host "Extracting Ninja-Build..."
 $NINJA_INSTALL_PATH = "$env:GITHUB_WORKSPACE\buildtools\ninja"
 New-Item -ItemType Directory -Force -Path "${NINJA_INSTALL_PATH}"
 Expand-Archive -LiteralPath "${NINJA_DL_PATH}" -DestinationPath "${NINJA_INSTALL_PATH}"
-# Export the NINJA executable path
-Write-Output "NINJA_INSTALL_PATH=${NINJA_INSTALL_PATH}" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
-Write-Output "PATH=${NINJA_INSTALL_PATH}; $env:PATH" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
-$env:PATH = "${NINJA_INSTALL_PATH}; $env:PATH"
-
-# Download zlib
-$ZLIB_RELEASE = "1.3.1";
-$ZLIB_RELEASE_ASSET = "zlib131.zip"
-$ZLIB_DL_URL = "https://github.com/madler/zlib/releases/download/v${ZLIB_RELEASE}/${ZLIB_RELEASE_ASSET}"
-$ZLIB_DL_SHA512 = "1f171880153b0120e1364baaf7d0a17f65086eff279f8f8c8538e5950097d1feee37cc173181676ba1e2aeb4565ba68749c814cd3e25bfb06271bea02feb7d94"
-$ZLIB_DL_PATH = "${DL_BASEDIR}\${ZLIB_RELEASE_ASSET}"
-$CurlArguments = '-s', '-Lf', '-o', "${ZLIB_DL_PATH}", "${ZLIB_DL_URL}"
-& curl.exe @CurlArguments
-$zlib_zip_hash = Get-FileHash -LiteralPath "${ZLIB_DL_PATH}" -Algorithm SHA512
-if ($zlib_zip_hash.Hash -eq $ZLIB_DL_SHA512) {
-	Write-Host "Successfully downloaded ${ZLIB_RELEASE_ASSET}"
-}
-Else {
-	Write-Error "The downloaded ${ZLIB_RELEASE_ASSET} hash '$($zlib_zip_hash.Hash)' does not match the expected hash: '$ZLIB_DL_SHA512'"
-}
 
-Write-Host "Extracting zlib source..."
-$ZLIB_SOURCE_PATH = "$env:GITHUB_WORKSPACE\buildtools\zlib-${ZLIB_RELEASE}"
-Expand-Archive -LiteralPath "${ZLIB_DL_PATH}" -DestinationPath "$env:GITHUB_WORKSPACE\buildtools"
-
-Write-Host "Building zlib source..."
-$ZLIB_BUILD_PATH = "$env:GITHUB_WORKSPACE\buildtools\zlib_build"
-cmake.exe -B "${ZLIB_BUILD_PATH}" -S "${ZLIB_SOURCE_PATH}" -DCMAKE_C_COMPILER="${env:MINGW_PKG_PREFIX}-gcc" -DCMAKE_CXX_COMPILER="${env:MINGW_PKG_PREFIX}-g++" -DCMAKE_RC_COMPILER="${env:MINGW_PKG_PREFIX}-windres" -DCMAKE_ASM_MASM_COMPILER="${env:MINGW_ASM_MASM_COMPILER}" -GNinja
-cmake.exe --build "${ZLIB_BUILD_PATH}" --target zlibstatic
-Copy-Item "${ZLIB_SOURCE_PATH}\zlib.h" "${ZLIB_BUILD_PATH}"
+# Export the NINJA executable path
+"NINJA_INSTALL_PATH=${NINJA_INSTALL_PATH}" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
+"PATH=${NINJA_INSTALL_PATH}; $env:PATH" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
 
-# Add CMAKE_DEFINES
-Write-Output "CMAKE_DEFINES=-DZLIB_LIBRARY=${ZLIB_BUILD_PATH}\libzlibstatic.a -DZLIB_INCLUDE_DIR=${ZLIB_BUILD_PATH} -DCMAKE_C_COMPILER=${env:MINGW_PKG_PREFIX}-gcc -DCMAKE_CXX_COMPILER=${env:MINGW_PKG_PREFIX}-g++ -DCMAKE_RC_COMPILER=${env:MINGW_PKG_PREFIX}-windres -DCMAKE_ASM_MASM_COMPILER=${env:MINGW_ASM_MASM_COMPILER} -GNinja" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
+# export CMAKE_DEFINES to the runner environment
+"CMAKE_DEFINES=-DCMAKE_C_COMPILER=${env:MINGW_PKG_PREFIX}-gcc -DCMAKE_CXX_COMPILER=${env:MINGW_PKG_PREFIX}-g++ -DCMAKE_RC_COMPILER=${env:MINGW_PKG_PREFIX}-windres -DCMAKE_ASM_MASM_COMPILER=${env:MINGW_ASM_MASM_COMPILER}" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append

scripts/install-zlib.ps1

@@ -0,0 +1,41 @@
+$DL_BASEDIR = "$env:GITHUB_WORKSPACE\dl"
+if (!(Test-Path -Path "$DL_BASEDIR")) { New-Item -ItemType Directory -Force -Path "$DL_BASEDIR" }
+
+$ZLIB_RELEASE = "1.3.1";
+$ZLIB_RELEASE_ASSET = "zlib131.zip"
+$ZLIB_DL_URL = "https://github.com/madler/zlib/releases/download/v${ZLIB_RELEASE}/${ZLIB_RELEASE_ASSET}"
+$ZLIB_DL_SHA512 = "1f171880153b0120e1364baaf7d0a17f65086eff279f8f8c8538e5950097d1feee37cc173181676ba1e2aeb4565ba68749c814cd3e25bfb06271bea02feb7d94"
+$ZLIB_DL_PATH = "${DL_BASEDIR}\${ZLIB_RELEASE_ASSET}"
+$CurlArguments = '-s', '-Lf', '-o', "${ZLIB_DL_PATH}", "${ZLIB_DL_URL}"
+& curl.exe @CurlArguments
+$zlib_zip_hash = Get-FileHash -LiteralPath "${ZLIB_DL_PATH}" -Algorithm SHA512
+if ($zlib_zip_hash.Hash -eq $ZLIB_DL_SHA512) {
+    Write-Host "Successfully downloaded ${ZLIB_RELEASE_ASSET}"
+}
+Else {
+    Write-Error "The downloaded ${ZLIB_RELEASE_ASSET} hash '$($zlib_zip_hash.Hash)' does not match the expected hash: '$ZLIB_DL_SHA512'"
+}
+
+Write-Host "Extracting zlib source..."
+$ZLIB_SOURCE_PATH = "$env:GITHUB_WORKSPACE\buildtools\zlib-${ZLIB_RELEASE}"
+Expand-Archive -LiteralPath "${ZLIB_DL_PATH}" -DestinationPath "$env:GITHUB_WORKSPACE\buildtools"
+
+Write-Host "Building zlib source..."
+$ZLIB_BUILD_PATH = "$env:GITHUB_WORKSPACE\buildtools\zlib_build"
+if ($env:TEST_MINGW -eq 1) {
+    cmake.exe -B "${ZLIB_BUILD_PATH}" -S "${ZLIB_SOURCE_PATH}" -GNinja
+}
+Else {
+    cmake.exe -B "${ZLIB_BUILD_PATH}" -S "${ZLIB_SOURCE_PATH}"
+}
+cmake.exe --build "${ZLIB_BUILD_PATH}" --target zlibstatic
+Copy-Item "${ZLIB_SOURCE_PATH}\zlib.h" "${ZLIB_BUILD_PATH}"
+
+# Append zlib CMAKE_DEFINES to the runner env.
+if ($env:TEST_MINGW -eq 1) {
+    $NEW_CMAKE_DEFINES="CMAKE_DEFINES=${env:CMAKE_DEFINES} -DZLIB_LIBRARY=${ZLIB_BUILD_PATH}\libzlibstatic.a -DZLIB_INCLUDE_DIR=${ZLIB_BUILD_PATH} -GNinja"
+}
+Else {
+    $NEW_CMAKE_DEFINES="CMAKE_DEFINES=-DZLIB_LIBRARY=${ZLIB_BUILD_PATH}\Debug\zlibstaticd.lib -DZLIB_INCLUDE_DIR=${ZLIB_BUILD_PATH}"
+}
+$NEW_CMAKE_DEFINES | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append

src/sentry_transport.c

@@ -169,8 +169,8 @@ gzipped_with_compression(const char *body, const size_t body_len,
 
     z_stream stream;
     memset(&stream, 0, sizeof(stream));
-    stream.next_in = body;
-    stream.avail_in = body_len;
+    stream.next_in = (unsigned char *)body;
+    stream.avail_in = (unsigned int)body_len;
 
     int err = deflateInit2(&stream, Z_DEFAULT_COMPRESSION, Z_DEFLATED,
         MAX_WBITS + 16, 9, Z_DEFAULT_STRATEGY);
@@ -179,16 +179,16 @@ gzipped_with_compression(const char *body, const size_t body_len,
         return false;
     }
 
-    size_t len = compressBound(body_len);
+    size_t len = compressBound((unsigned long)body_len);
     char *buffer = sentry_malloc(len);
     if (!buffer) {
         deflateEnd(&stream);
         return false;
     }
 
     while (err == Z_OK) {
-        stream.next_out = buffer + stream.total_out;
-        stream.avail_out = len - stream.total_out;
+        stream.next_out = (unsigned char *)(buffer + stream.total_out);
+        stream.avail_out = (unsigned int)(len - stream.total_out);
         err = deflate(&stream, Z_FINISH);
     }
 
@@ -225,8 +225,8 @@ sentry__prepare_http_request(sentry_envelope_t *envelope,
         return NULL;
     }
 
-    bool compressed = false;
 #ifdef SENTRY_TRANSPORT_COMPRESSION
+    bool compressed = false;
     char *compressed_body = NULL;
     size_t compressed_body_len = 0;
     compressed = gzipped_with_compression(
@@ -273,11 +273,13 @@ sentry__prepare_http_request(sentry_envelope_t *envelope,
     h->key = "content-type";
     h->value = sentry__string_clone(ENVELOPE_MIME);
 
+#ifdef SENTRY_TRANSPORT_COMPRESSION
     if (compressed) {
         h = &req->headers[req->headers_len++];
         h->key = "content-encoding";
         h->value = sentry__string_clone("gzip");
     }
+#endif
 
     h = &req->headers[req->headers_len++];
     h->key = "content-length";

tests/__init__.py

@@ -1,3 +1,4 @@
+import gzip
 import subprocess
 import os
 import io
@@ -162,10 +163,17 @@ def deserialize_from(
 
     @classmethod
     def deserialize(
-        cls, bytes  # type: bytes
+        cls, data  # type: bytes
     ):
         # type: (...) -> Envelope
-        return cls.deserialize_from(io.BytesIO(bytes))
+
+        # check if the data is gzip encoded and extract it before deserialization.
+        # 0x1f8b: gzip-magic, 0x08: `DEFLATE` compression method.
+        if data[:3] == b"\x1f\x8b\x08":
+            with gzip.open(io.BytesIO(data), "rb") as output:
+                return cls.deserialize_from(output)
+
+        return cls.deserialize_from(io.BytesIO(data))
 
     def print_verbose(self, indent=0):
         """Pretty prints the envelope."""

tests/cmake.py

@@ -138,6 +138,9 @@ def cmake(cwd, targets, options=None):
     if "tsan" in os.environ.get("RUN_ANALYZER", ""):
         configcmd.append("-DWITH_TSAN_OPTION=ON")
 
+    if os.environ.get("TRANSPORT_COMPRESSION"):
+        configcmd.append("-DSENTRY_TRANSPORT_COMPRESSION=ON")
+
     # we have to set `-Werror` for this cmake invocation only, otherwise
     # completely unrelated things will break
     cflags = []

tests/requirements.txt

@@ -1,4 +1,5 @@
 black==24.3.0
-pytest==8.0.1
+pytest==8.1.1
 pytest-httpserver==1.0.10
 msgpack==1.0.8
+pytest-xdist==3.5.0

tests/unit/test_envelopes.c

@@ -33,10 +33,12 @@ SENTRY_TEST(basic_http_request_preparation_for_event)
     TEST_CHECK_STRING_EQUAL(req->method, "POST");
     TEST_CHECK_STRING_EQUAL(
         req->url, "https://sentry.invalid:443/api/42/envelope/");
+#ifndef SENTRY_TRANSPORT_COMPRESSION
     TEST_CHECK_STRING_EQUAL(req->body,
         "{\"event_id\":\"c993afb6-b4ac-48a6-b61b-2558e601d65d\"}\n"
         "{\"type\":\"event\",\"length\":51}\n"
         "{\"event_id\":\"c993afb6-b4ac-48a6-b61b-2558e601d65d\"}");
+#endif
     sentry__prepared_http_request_free(req);
     sentry_envelope_free(envelope);
 
@@ -62,13 +64,14 @@ SENTRY_TEST(basic_http_request_preparation_for_transaction)
     TEST_CHECK_STRING_EQUAL(req->method, "POST");
     TEST_CHECK_STRING_EQUAL(
         req->url, "https://sentry.invalid:443/api/42/envelope/");
+#ifndef SENTRY_TRANSPORT_COMPRESSION
     TEST_CHECK_STRING_EQUAL(req->body,
         "{\"event_id\":\"c993afb6-b4ac-48a6-b61b-2558e601d65d\",\"sent_at\":"
         "\"2021-12-16T05:53:59.343Z\"}\n"
         "{\"type\":\"transaction\",\"length\":72}\n"
         "{\"event_id\":\"c993afb6-b4ac-48a6-b61b-2558e601d65d\",\"type\":"
         "\"transaction\"}");
-
+#endif
     sentry__prepared_http_request_free(req);
     sentry_envelope_free(envelope);
 
@@ -91,12 +94,14 @@ SENTRY_TEST(basic_http_request_preparation_for_user_feedback)
     TEST_CHECK_STRING_EQUAL(req->method, "POST");
     TEST_CHECK_STRING_EQUAL(
         req->url, "https://sentry.invalid:443/api/42/envelope/");
+#ifndef SENTRY_TRANSPORT_COMPRESSION
     TEST_CHECK_STRING_EQUAL(req->body,
         "{\"event_id\":\"c993afb6-b4ac-48a6-b61b-2558e601d65d\"}\n"
         "{\"type\":\"user_report\",\"length\":117}\n"
         "{\"event_id\":\"c993afb6-b4ac-48a6-b61b-2558e601d65d\",\"name\":"
         "\"some-name\",\"email\":\"some-email\",\"comments\":"
         "\"some-comment\"}");
+#endif
     sentry__prepared_http_request_free(req);
     sentry_value_decref(user_feedback);
     sentry_envelope_free(envelope);
@@ -124,12 +129,14 @@ SENTRY_TEST(basic_http_request_preparation_for_event_with_attachment)
     TEST_CHECK_STRING_EQUAL(req->method, "POST");
     TEST_CHECK_STRING_EQUAL(
         req->url, "https://sentry.invalid:443/api/42/envelope/");
+#ifndef SENTRY_TRANSPORT_COMPRESSION
     TEST_CHECK_STRING_EQUAL(req->body,
         "{\"event_id\":\"c993afb6-b4ac-48a6-b61b-2558e601d65d\"}\n"
         "{\"type\":\"event\",\"length\":51}\n"
         "{\"event_id\":\"c993afb6-b4ac-48a6-b61b-2558e601d65d\"}\n"
         "{\"type\":\"attachment\",\"length\":12}\n"
         "Hello World!");
+#endif
     sentry__prepared_http_request_free(req);
     sentry_envelope_free(envelope);
 
@@ -153,12 +160,14 @@ SENTRY_TEST(basic_http_request_preparation_for_minidump)
     TEST_CHECK_STRING_EQUAL(req->method, "POST");
     TEST_CHECK_STRING_EQUAL(
         req->url, "https://sentry.invalid:443/api/42/envelope/");
+#ifndef SENTRY_TRANSPORT_COMPRESSION
     TEST_CHECK_STRING_EQUAL(req->body,
         "{}\n"
         "{\"type\":\"minidump\",\"length\":4}\n"
         "MDMP\n"
         "{\"type\":\"attachment\",\"length\":12}\n"
         "Hello World!");
+#endif
     sentry__prepared_http_request_free(req);
     sentry_envelope_free(envelope);