test: integration-tests for transport compression (#969)

.github/workflows/ci.yml

@@ -158,6 +158,11 @@ jobs:
           MINGW_ASM_MASM_COMPILER: ${{ matrix.MINGW_ASM_MASM_COMPILER }}
         run: . "scripts\install-llvm-mingw.ps1"
 
+      - name: Set up zlib for Windows
+        if: ${{ runner.os == 'Windows' }}
+        shell: powershell
+        run: . "scripts\install-zlib.ps1"
+
       - name: Installing Android SDK Dependencies
         if: ${{ env['ANDROID_API'] }}
         run: |

scripts/install-llvm-mingw.ps1

@@ -26,10 +26,10 @@ New-Item -ItemType Directory -Force -Path "${LLVM_MINGW_INSTALL_PATH}"
 Expand-Archive -LiteralPath "${LLVM_MINGW_DL_PATH}" -DestinationPath "${LLVM_MINGW_INSTALL_PATH}"
 # Export the LLVM-mingw install path
 $LLVM_MINGW_INSTALL_PATH = "${LLVM_MINGW_INSTALL_PATH}\${LLVM_MINGW_PKG}"
-Write-Output "LLVM_MINGW_INSTALL_PATH=${LLVM_MINGW_INSTALL_PATH}" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
+"LLVM_MINGW_INSTALL_PATH=${LLVM_MINGW_INSTALL_PATH}" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
 # Prepend bin path to the system PATH
-Write-Output "Path to LLVM-mingw bin folder: ${LLVM_MINGW_INSTALL_PATH}\bin"
-Write-Output "${LLVM_MINGW_INSTALL_PATH}\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
+Write-Host "Path to LLVM-mingw bin folder: ${LLVM_MINGW_INSTALL_PATH}\bin"
+"${LLVM_MINGW_INSTALL_PATH}\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
 
 # Download ninja-build
 $NINJA_DL_URL = "https://github.com/ninja-build/ninja/releases/download/v1.11.1/ninja-win.zip"
@@ -49,36 +49,10 @@ Write-Host "Extracting Ninja-Build..."
 $NINJA_INSTALL_PATH = "$env:GITHUB_WORKSPACE\buildtools\ninja"
 New-Item -ItemType Directory -Force -Path "${NINJA_INSTALL_PATH}"
 Expand-Archive -LiteralPath "${NINJA_DL_PATH}" -DestinationPath "${NINJA_INSTALL_PATH}"
-# Export the NINJA executable path
-Write-Output "NINJA_INSTALL_PATH=${NINJA_INSTALL_PATH}" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
-Write-Output "PATH=${NINJA_INSTALL_PATH}; $env:PATH" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
-$env:PATH = "${NINJA_INSTALL_PATH}; $env:PATH"
-
-# Download zlib
-$ZLIB_RELEASE = "1.3.1";
-$ZLIB_RELEASE_ASSET = "zlib131.zip"
-$ZLIB_DL_URL = "https://github.com/madler/zlib/releases/download/v${ZLIB_RELEASE}/${ZLIB_RELEASE_ASSET}"
-$ZLIB_DL_SHA512 = "1f171880153b0120e1364baaf7d0a17f65086eff279f8f8c8538e5950097d1feee37cc173181676ba1e2aeb4565ba68749c814cd3e25bfb06271bea02feb7d94"
-$ZLIB_DL_PATH = "${DL_BASEDIR}\${ZLIB_RELEASE_ASSET}"
-$CurlArguments = '-s', '-Lf', '-o', "${ZLIB_DL_PATH}", "${ZLIB_DL_URL}"
-& curl.exe @CurlArguments
-$zlib_zip_hash = Get-FileHash -LiteralPath "${ZLIB_DL_PATH}" -Algorithm SHA512
-if ($zlib_zip_hash.Hash -eq $ZLIB_DL_SHA512) {
-	Write-Host "Successfully downloaded ${ZLIB_RELEASE_ASSET}"
-}
-Else {
-	Write-Error "The downloaded ${ZLIB_RELEASE_ASSET} hash '$($zlib_zip_hash.Hash)' does not match the expected hash: '$ZLIB_DL_SHA512'"
-}
 
-Write-Host "Extracting zlib source..."
-$ZLIB_SOURCE_PATH = "$env:GITHUB_WORKSPACE\buildtools\zlib-${ZLIB_RELEASE}"
-Expand-Archive -LiteralPath "${ZLIB_DL_PATH}" -DestinationPath "$env:GITHUB_WORKSPACE\buildtools"
-
-Write-Host "Building zlib source..."
-$ZLIB_BUILD_PATH = "$env:GITHUB_WORKSPACE\buildtools\zlib_build"
-cmake.exe -B "${ZLIB_BUILD_PATH}" -S "${ZLIB_SOURCE_PATH}" -DCMAKE_C_COMPILER="${env:MINGW_PKG_PREFIX}-gcc" -DCMAKE_CXX_COMPILER="${env:MINGW_PKG_PREFIX}-g++" -DCMAKE_RC_COMPILER="${env:MINGW_PKG_PREFIX}-windres" -DCMAKE_ASM_MASM_COMPILER="${env:MINGW_ASM_MASM_COMPILER}" -GNinja
-cmake.exe --build "${ZLIB_BUILD_PATH}" --target zlibstatic
-Copy-Item "${ZLIB_SOURCE_PATH}\zlib.h" "${ZLIB_BUILD_PATH}"
+# Export the NINJA executable path
+"NINJA_INSTALL_PATH=${NINJA_INSTALL_PATH}" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
+"PATH=${NINJA_INSTALL_PATH}; $env:PATH" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
 
-# Add CMAKE_DEFINES
-Write-Output "CMAKE_DEFINES=-DZLIB_LIBRARY=${ZLIB_BUILD_PATH}\libzlibstatic.a -DZLIB_INCLUDE_DIR=${ZLIB_BUILD_PATH} -DCMAKE_C_COMPILER=${env:MINGW_PKG_PREFIX}-gcc -DCMAKE_CXX_COMPILER=${env:MINGW_PKG_PREFIX}-g++ -DCMAKE_RC_COMPILER=${env:MINGW_PKG_PREFIX}-windres -DCMAKE_ASM_MASM_COMPILER=${env:MINGW_ASM_MASM_COMPILER} -GNinja" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
+# export CMAKE_DEFINES to the runner environment
+"CMAKE_DEFINES=-DCMAKE_C_COMPILER=${env:MINGW_PKG_PREFIX}-gcc -DCMAKE_CXX_COMPILER=${env:MINGW_PKG_PREFIX}-g++ -DCMAKE_RC_COMPILER=${env:MINGW_PKG_PREFIX}-windres -DCMAKE_ASM_MASM_COMPILER=${env:MINGW_ASM_MASM_COMPILER}" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append

scripts/install-zlib.ps1

@@ -0,0 +1,44 @@
+$DL_BASEDIR = "$env:GITHUB_WORKSPACE\dl"
+if (!(Test-Path -Path "$DL_BASEDIR")) { New-Item -ItemType Directory -Force -Path "$DL_BASEDIR" }
+
+$ZLIB_RELEASE = "1.3.1";
+$ZLIB_RELEASE_ASSET = "zlib131.zip"
+$ZLIB_DL_URL = "https://github.com/madler/zlib/releases/download/v${ZLIB_RELEASE}/${ZLIB_RELEASE_ASSET}"
+$ZLIB_DL_SHA512 = "1f171880153b0120e1364baaf7d0a17f65086eff279f8f8c8538e5950097d1feee37cc173181676ba1e2aeb4565ba68749c814cd3e25bfb06271bea02feb7d94"
+$ZLIB_DL_PATH = "${DL_BASEDIR}\${ZLIB_RELEASE_ASSET}"
+$CurlArguments = '-s', '-Lf', '-o', "${ZLIB_DL_PATH}", "${ZLIB_DL_URL}"
+& curl.exe @CurlArguments
+$zlib_zip_hash = Get-FileHash -LiteralPath "${ZLIB_DL_PATH}" -Algorithm SHA512
+if ($zlib_zip_hash.Hash -eq $ZLIB_DL_SHA512) {
+    Write-Host "Successfully downloaded ${ZLIB_RELEASE_ASSET}"
+}
+Else {
+    Write-Error "The downloaded ${ZLIB_RELEASE_ASSET} hash '$($zlib_zip_hash.Hash)' does not match the expected hash: '$ZLIB_DL_SHA512'"
+}
+
+Write-Host "Extracting zlib source..."
+$ZLIB_SOURCE_PATH = "$env:GITHUB_WORKSPACE\buildtools\zlib-${ZLIB_RELEASE}"
+Expand-Archive -LiteralPath "${ZLIB_DL_PATH}" -DestinationPath "$env:GITHUB_WORKSPACE\buildtools"
+
+Write-Host "Building zlib source..."
+$ZLIB_BUILD_PATH = "$env:GITHUB_WORKSPACE\buildtools\zlib_build"
+if ($env:TEST_MINGW -eq 1) {
+    cmake.exe -B "${ZLIB_BUILD_PATH}" -S "${ZLIB_SOURCE_PATH}" -GNinja
+}
+Elseif ($env:TEST_X86 -eq 1) {
+    cmake.exe -B "${ZLIB_BUILD_PATH}" -S "${ZLIB_SOURCE_PATH}" -AWin32
+}
+Else {
+    cmake.exe -B "${ZLIB_BUILD_PATH}" -S "${ZLIB_SOURCE_PATH}"
+}
+cmake.exe --build "${ZLIB_BUILD_PATH}" --target zlibstatic
+Copy-Item "${ZLIB_SOURCE_PATH}\zlib.h" "${ZLIB_BUILD_PATH}"
+
+# Append zlib CMAKE_DEFINES to the runner env.
+if ($env:TEST_MINGW -eq 1) {
+    $NEW_CMAKE_DEFINES="CMAKE_DEFINES=${env:CMAKE_DEFINES} -DZLIB_LIBRARY=${ZLIB_BUILD_PATH}\libzlibstatic.a -DZLIB_INCLUDE_DIR=${ZLIB_BUILD_PATH} -GNinja"
+}
+Else {
+    $NEW_CMAKE_DEFINES="CMAKE_DEFINES=-DZLIB_LIBRARY=${ZLIB_BUILD_PATH}\Debug\zlibstaticd.lib -DZLIB_INCLUDE_DIR=${ZLIB_BUILD_PATH}"
+}
+$NEW_CMAKE_DEFINES | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append

src/sentry_transport.c

@@ -169,8 +169,8 @@ gzipped_with_compression(const char *body, const size_t body_len,
 
     z_stream stream;
     memset(&stream, 0, sizeof(stream));
-    stream.next_in = body;
-    stream.avail_in = body_len;
+    stream.next_in = (unsigned char *)body;
+    stream.avail_in = (unsigned int)body_len;
 
     int err = deflateInit2(&stream, Z_DEFAULT_COMPRESSION, Z_DEFLATED,
         MAX_WBITS + 16, 9, Z_DEFAULT_STRATEGY);
@@ -179,16 +179,16 @@ gzipped_with_compression(const char *body, const size_t body_len,
         return false;
     }
 
-    size_t len = compressBound(body_len);
+    size_t len = compressBound((unsigned long)body_len);
     char *buffer = sentry_malloc(len);
     if (!buffer) {
         deflateEnd(&stream);
         return false;
     }
 
     while (err == Z_OK) {
-        stream.next_out = buffer + stream.total_out;
-        stream.avail_out = len - stream.total_out;
+        stream.next_out = (unsigned char *)(buffer + stream.total_out);
+        stream.avail_out = (unsigned int)(len - stream.total_out);
         err = deflate(&stream, Z_FINISH);
     }
 
@@ -225,8 +225,8 @@ sentry__prepare_http_request(sentry_envelope_t *envelope,
         return NULL;
     }
 
-    bool compressed = false;
 #ifdef SENTRY_TRANSPORT_COMPRESSION
+    bool compressed = false;
     char *compressed_body = NULL;
     size_t compressed_body_len = 0;
     compressed = gzipped_with_compression(
@@ -273,11 +273,13 @@ sentry__prepare_http_request(sentry_envelope_t *envelope,
     h->key = "content-type";
     h->value = sentry__string_clone(ENVELOPE_MIME);
 
+#ifdef SENTRY_TRANSPORT_COMPRESSION
     if (compressed) {
         h = &req->headers[req->headers_len++];
         h->key = "content-encoding";
         h->value = sentry__string_clone("gzip");
     }
+#endif
 
     h = &req->headers[req->headers_len++];
     h->key = "content-length";

tests/__init__.py

@@ -1,3 +1,4 @@
+import gzip
 import subprocess
 import os
 import io
@@ -162,10 +163,17 @@ def deserialize_from(
 
     @classmethod
     def deserialize(
-        cls, bytes  # type: bytes
+        cls, data  # type: bytes
     ):
         # type: (...) -> Envelope
-        return cls.deserialize_from(io.BytesIO(bytes))
+
+        # check if the data is gzip encoded and extract it before deserialization.
+        # 0x1f8b: gzip-magic, 0x08: `DEFLATE` compression method.
+        if data[:3] == b"\x1f\x8b\x08":
+            with gzip.open(io.BytesIO(data), "rb") as output:
+                return cls.deserialize_from(output)
+
+        return cls.deserialize_from(io.BytesIO(data))
 
     def print_verbose(self, indent=0):
         """Pretty prints the envelope."""

tests/assertions.py

@@ -4,7 +4,7 @@
 import re
 import sys
 from dataclasses import dataclass
-from datetime import datetime
+from datetime import datetime, UTC
 
 import msgpack
 
@@ -185,7 +185,7 @@ def assert_minidump(envelope):
     assert minidump.payload.bytes.startswith(b"MDMP")
 
 
-def assert_timestamp(ts, now=datetime.utcnow()):
+def assert_timestamp(ts, now=datetime.now(UTC)):
     assert ts[:11] == now.isoformat()[:11]
 
 
@@ -307,3 +307,11 @@ def assert_crashpad_upload(req):
         and b"\n\nMDMP" in part.as_bytes()
         for part in msg.walk()
     )
+
+
+def assert_gzip_file_header(output):
+    assert output[:3] == b"\x1f\x8b\x08"
+
+
+def assert_gzip_content_encoding(req):
+    assert req.content_encoding == "gzip"

tests/cmake.py

@@ -1,9 +1,10 @@
-import os
 import json
-import sys
+import os
+import shutil
 import subprocess
+import sys
+
 import pytest
-import shutil
 
 
 class CMake:

tests/requirements.txt

@@ -1,4 +1,5 @@
 black==24.3.0
-pytest==8.0.1
+pytest==8.1.1
 pytest-httpserver==1.0.10
 msgpack==1.0.8
+pytest-xdist==3.5.0

tests/test_integration_crashpad.py

@@ -1,11 +1,17 @@
-import pytest
 import os
 import shutil
 import sys
 import time
+
+import pytest
+
 from . import make_dsn, run, Envelope
+from .assertions import (
+    assert_crashpad_upload,
+    assert_session,
+    assert_gzip_file_header,
+)
 from .conditions import has_crashpad
-from .assertions import assert_crashpad_upload, assert_session
 
 pytestmark = pytest.mark.skipif(not has_crashpad, reason="tests need crashpad backend")
 
@@ -118,13 +124,15 @@ def test_crashpad_wer_crash(cmake, httpserver, run_args):
 
 
 @pytest.mark.parametrize(
-    "run_args",
+    "run_args,build_args",
     [
         # if we crash, we want a dump
-        ([]),
+        ([], {"SENTRY_TRANSPORT_COMPRESSION": "Off"}),
+        ([], {"SENTRY_TRANSPORT_COMPRESSION": "On"}),
         # if we crash and before-send doesn't discard, we want a dump
         pytest.param(
             ["before-send"],
+            {},
             marks=pytest.mark.skipif(
                 sys.platform == "darwin",
                 reason="crashpad doesn't provide SetFirstChanceExceptionHandler on macOS",
@@ -133,15 +141,17 @@ def test_crashpad_wer_crash(cmake, httpserver, run_args):
         # if on_crash() is non-discarding, a discarding before_send() is overruled, so we get a dump
         pytest.param(
             ["discarding-before-send", "on-crash"],
+            {},
             marks=pytest.mark.skipif(
                 sys.platform == "darwin",
                 reason="crashpad doesn't provide SetFirstChanceExceptionHandler on macOS",
             ),
         ),
     ],
 )
-def test_crashpad_dumping_crash(cmake, httpserver, run_args):
-    tmp_path = cmake(["sentry_example"], {"SENTRY_BACKEND": "crashpad"})
+def test_crashpad_dumping_crash(cmake, httpserver, run_args, build_args):
+    build_args.update({"SENTRY_BACKEND": "crashpad"})
+    tmp_path = cmake(["sentry_example"], build_args)
 
     # make sure we are isolated from previous runs
     shutil.rmtree(tmp_path / ".sentry-native", ignore_errors=True)
@@ -169,19 +179,24 @@ def test_crashpad_dumping_crash(cmake, httpserver, run_args):
     run(tmp_path, "sentry_example", ["log", "no-setup"], check=True, env=env)
 
     assert len(httpserver.log) == 2
-    outputs = (httpserver.log[0][0], httpserver.log[1][0])
     session, multipart = (
-        (outputs[0].get_data(), outputs[1])
-        if b'"type":"session"' in outputs[0].get_data()
-        else (outputs[1].get_data(), outputs[0])
+        (httpserver.log[0][0], httpserver.log[1][0])
+        if is_session_envelope(httpserver.log[0][0].get_data())
+        else (httpserver.log[1][0], httpserver.log[0][0])
     )
 
-    envelope = Envelope.deserialize(session)
+    if build_args.get("SENTRY_TRANSPORT_COMPRESSION") == "On":
+        assert_gzip_file_header(session.get_data())
 
+    envelope = Envelope.deserialize(session.get_data())
     assert_session(envelope, {"status": "crashed", "errors": 1})
     assert_crashpad_upload(multipart)
 
 
+def is_session_envelope(data):
+    return b'"type":"session"' in data
+
+
 @pytest.mark.skipif(
     sys.platform == "darwin",
     reason="crashpad doesn't provide SetFirstChanceExceptionHandler on macOS",