From 61f1172113c8dd2a69b9ea00279e78a4f68dd630 Mon Sep 17 00:00:00 2001 From: Mischan Toosarani-Hausberger Date: Fri, 22 Mar 2024 19:47:00 +0100 Subject: [PATCH 01/17] Enable it for all. --- src/sentry_transport.c | 8 +++++--- tests/__init__.py | 12 ++++++++++-- tests/cmake.py | 3 +++ tests/requirements.txt | 3 ++- tests/unit/test_envelopes.c | 11 ++++++++++- 5 files changed, 30 insertions(+), 7 deletions(-) diff --git a/src/sentry_transport.c b/src/sentry_transport.c index 06ba3524b..72b274d85 100644 --- a/src/sentry_transport.c +++ b/src/sentry_transport.c @@ -169,7 +169,7 @@ gzipped_with_compression(const char *body, const size_t body_len, z_stream stream; memset(&stream, 0, sizeof(stream)); - stream.next_in = body; + stream.next_in = (unsigned char*)body; stream.avail_in = body_len; int err = deflateInit2(&stream, Z_DEFAULT_COMPRESSION, Z_DEFLATED, @@ -187,7 +187,7 @@ gzipped_with_compression(const char *body, const size_t body_len, } while (err == Z_OK) { - stream.next_out = buffer + stream.total_out; + stream.next_out = (unsigned char*)(buffer + stream.total_out); stream.avail_out = len - stream.total_out; err = deflate(&stream, Z_FINISH); } @@ -225,8 +225,8 @@ sentry__prepare_http_request(sentry_envelope_t *envelope, return NULL; } - bool compressed = false; #ifdef SENTRY_TRANSPORT_COMPRESSION + bool compressed = false; char *compressed_body = NULL; size_t compressed_body_len = 0; compressed = gzipped_with_compression( @@ -273,11 +273,13 @@ sentry__prepare_http_request(sentry_envelope_t *envelope, h->key = "content-type"; h->value = sentry__string_clone(ENVELOPE_MIME); +#ifdef SENTRY_TRANSPORT_COMPRESSION if (compressed) { h = &req->headers[req->headers_len++]; h->key = "content-encoding"; h->value = sentry__string_clone("gzip"); } +#endif h = &req->headers[req->headers_len++]; h->key = "content-length"; diff --git a/tests/__init__.py b/tests/__init__.py index d7374c5b4..83c85d243 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,3 +1,4 @@ +import gzip import subprocess import os import io @@ -162,10 +163,17 @@ def deserialize_from( @classmethod def deserialize( - cls, bytes # type: bytes + cls, data # type: bytes ): # type: (...) -> Envelope - return cls.deserialize_from(io.BytesIO(bytes)) + + # check if the data is gzip encoded and extract it before deserialization. + # 0x1f8b: gzip-magic, 0x08: `DEFLATE` compression method. + if data[:3] == b"\x1f\x8b\x08": + with gzip.open(io.BytesIO(data), 'rb') as output: + return cls.deserialize_from(output) + + return cls.deserialize_from(io.BytesIO(data)) def print_verbose(self, indent=0): """Pretty prints the envelope.""" diff --git a/tests/cmake.py b/tests/cmake.py index 35c48a8b5..ad1c1fa99 100644 --- a/tests/cmake.py +++ b/tests/cmake.py @@ -138,6 +138,9 @@ def cmake(cwd, targets, options=None): if "tsan" in os.environ.get("RUN_ANALYZER", ""): configcmd.append("-DWITH_TSAN_OPTION=ON") + # TODO: connect with an env variable + configcmd.append("-DSENTRY_TRANSPORT_COMPRESSION=ON") + # we have to set `-Werror` for this cmake invocation only, otherwise # completely unrelated things will break cflags = [] diff --git a/tests/requirements.txt b/tests/requirements.txt index ae5bbcfa3..cbc8051ec 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,4 +1,5 @@ black==24.3.0 -pytest==8.0.1 +pytest==8.1.1 pytest-httpserver==1.0.10 msgpack==1.0.8 +pytest-xdist==3.5.0 \ No newline at end of file diff --git a/tests/unit/test_envelopes.c b/tests/unit/test_envelopes.c index 5aa1170d5..803fb4730 100644 --- a/tests/unit/test_envelopes.c +++ b/tests/unit/test_envelopes.c @@ -33,10 +33,12 @@ SENTRY_TEST(basic_http_request_preparation_for_event) TEST_CHECK_STRING_EQUAL(req->method, "POST"); TEST_CHECK_STRING_EQUAL( req->url, "https://sentry.invalid:443/api/42/envelope/"); +#ifndef SENTRY_TRANSPORT_COMPRESSION TEST_CHECK_STRING_EQUAL(req->body, "{\"event_id\":\"c993afb6-b4ac-48a6-b61b-2558e601d65d\"}\n" "{\"type\":\"event\",\"length\":51}\n" "{\"event_id\":\"c993afb6-b4ac-48a6-b61b-2558e601d65d\"}"); +#endif sentry__prepared_http_request_free(req); sentry_envelope_free(envelope); @@ -62,13 +64,14 @@ SENTRY_TEST(basic_http_request_preparation_for_transaction) TEST_CHECK_STRING_EQUAL(req->method, "POST"); TEST_CHECK_STRING_EQUAL( req->url, "https://sentry.invalid:443/api/42/envelope/"); +#ifndef SENTRY_TRANSPORT_COMPRESSION TEST_CHECK_STRING_EQUAL(req->body, "{\"event_id\":\"c993afb6-b4ac-48a6-b61b-2558e601d65d\",\"sent_at\":" "\"2021-12-16T05:53:59.343Z\"}\n" "{\"type\":\"transaction\",\"length\":72}\n" "{\"event_id\":\"c993afb6-b4ac-48a6-b61b-2558e601d65d\",\"type\":" "\"transaction\"}"); - +#endif sentry__prepared_http_request_free(req); sentry_envelope_free(envelope); @@ -91,12 +94,14 @@ SENTRY_TEST(basic_http_request_preparation_for_user_feedback) TEST_CHECK_STRING_EQUAL(req->method, "POST"); TEST_CHECK_STRING_EQUAL( req->url, "https://sentry.invalid:443/api/42/envelope/"); +#ifndef SENTRY_TRANSPORT_COMPRESSION TEST_CHECK_STRING_EQUAL(req->body, "{\"event_id\":\"c993afb6-b4ac-48a6-b61b-2558e601d65d\"}\n" "{\"type\":\"user_report\",\"length\":117}\n" "{\"event_id\":\"c993afb6-b4ac-48a6-b61b-2558e601d65d\",\"name\":" "\"some-name\",\"email\":\"some-email\",\"comments\":" "\"some-comment\"}"); +#endif sentry__prepared_http_request_free(req); sentry_value_decref(user_feedback); sentry_envelope_free(envelope); @@ -124,12 +129,14 @@ SENTRY_TEST(basic_http_request_preparation_for_event_with_attachment) TEST_CHECK_STRING_EQUAL(req->method, "POST"); TEST_CHECK_STRING_EQUAL( req->url, "https://sentry.invalid:443/api/42/envelope/"); +#ifndef SENTRY_TRANSPORT_COMPRESSION TEST_CHECK_STRING_EQUAL(req->body, "{\"event_id\":\"c993afb6-b4ac-48a6-b61b-2558e601d65d\"}\n" "{\"type\":\"event\",\"length\":51}\n" "{\"event_id\":\"c993afb6-b4ac-48a6-b61b-2558e601d65d\"}\n" "{\"type\":\"attachment\",\"length\":12}\n" "Hello World!"); +#endif sentry__prepared_http_request_free(req); sentry_envelope_free(envelope); @@ -153,12 +160,14 @@ SENTRY_TEST(basic_http_request_preparation_for_minidump) TEST_CHECK_STRING_EQUAL(req->method, "POST"); TEST_CHECK_STRING_EQUAL( req->url, "https://sentry.invalid:443/api/42/envelope/"); +#ifndef SENTRY_TRANSPORT_COMPRESSION TEST_CHECK_STRING_EQUAL(req->body, "{}\n" "{\"type\":\"minidump\",\"length\":4}\n" "MDMP\n" "{\"type\":\"attachment\",\"length\":12}\n" "Hello World!"); +#endif sentry__prepared_http_request_free(req); sentry_envelope_free(envelope); From 787555be936a3b2f4d8fa6430bf6eb48b9f54e06 Mon Sep 17 00:00:00 2001 From: Mischan Toosarani-Hausberger Date: Sat, 23 Mar 2024 18:32:57 +0100 Subject: [PATCH 02/17] Add transport compression to CI --- .github/workflows/ci.yml | 23 ++++++++++++++++++++++- tests/cmake.py | 4 ++-- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 38cb54361..996269245 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -53,6 +53,13 @@ jobs: CXX: clang++-15 ERROR_ON_WARNINGS: 1 RUN_ANALYZER: asan,llvm-cov + - name: Linux (clang + transport-compression + asan + llvm-cov) + os: ubuntu-22.04 + CC: clang-15 + CXX: clang++-15 + ERROR_ON_WARNINGS: 1 + TRANSPORT_COMPRESSION: 1 + RUN_ANALYZER: asan,llvm-cov - name: Linux (clang + kcov) os: ubuntu-22.04 CC: clang-15 @@ -78,6 +85,14 @@ jobs: ERROR_ON_WARNINGS: 1 SYSTEM_VERSION_COMPAT: 0 RUN_ANALYZER: asan,llvm-cov + - name: macOS (clang + transport-compression + asan + llvm-cov) + os: macOs-11 + CC: clang + CXX: clang++ + ERROR_ON_WARNINGS: 1 + SYSTEM_VERSION_COMPAT: 0 + TRANSPORT_COMPRESSION: 1 + RUN_ANALYZER: asan,llvm-cov - name: Windows (old VS, 32-bit) os: windows-2019 TEST_X86: 1 @@ -88,6 +103,12 @@ jobs: TEST_MINGW: 1 MINGW_PKG_PREFIX: x86_64-w64-mingw32 MINGW_ASM_MASM_COMPILER: llvm-ml;-m64 + - name: LLVM-Mingw (transport-compression) + os: windows-latest + TEST_MINGW: 1 + MINGW_PKG_PREFIX: x86_64-w64-mingw32 + MINGW_ASM_MASM_COMPILER: llvm-ml;-m64 + TRANSPORT_COMPRESSION: 1 # The Android emulator is currently only available on macos, see: # https://docs.microsoft.com/en-us/azure/devops/pipelines/ecosystems/android?view=azure-devops#test-on-the-android-emulator - name: Android (old API/NDK) @@ -114,7 +135,7 @@ jobs: ANDROID_ARCH: ${{ matrix.ANDROID_ARCH }} CMAKE_DEFINES: ${{ matrix.CMAKE_DEFINES }} SYSTEM_VERSION_COMPAT: ${{ matrix.SYSTEM_VERSION_COMPAT }} - + TRANSPORT_COMPRESSION: ${{ matrix.TRANSPORT_COMPRESSION }} steps: - uses: actions/checkout@v3 with: diff --git a/tests/cmake.py b/tests/cmake.py index ad1c1fa99..eb75dd1aa 100644 --- a/tests/cmake.py +++ b/tests/cmake.py @@ -138,8 +138,8 @@ def cmake(cwd, targets, options=None): if "tsan" in os.environ.get("RUN_ANALYZER", ""): configcmd.append("-DWITH_TSAN_OPTION=ON") - # TODO: connect with an env variable - configcmd.append("-DSENTRY_TRANSPORT_COMPRESSION=ON") + if os.environ.get("TRANSPORT_COMPRESSION"): + configcmd.append("-DSENTRY_TRANSPORT_COMPRESSION=ON") # we have to set `-Werror` for this cmake invocation only, otherwise # completely unrelated things will break From 92178222a946cc39e5feeee1b7b5ae35d66dfeaa Mon Sep 17 00:00:00 2001 From: Mischan Toosarani-Hausberger Date: Sat, 23 Mar 2024 18:36:12 +0100 Subject: [PATCH 03/17] format happiness --- src/sentry_transport.c | 4 ++-- tests/__init__.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/sentry_transport.c b/src/sentry_transport.c index 72b274d85..1912cc910 100644 --- a/src/sentry_transport.c +++ b/src/sentry_transport.c @@ -169,7 +169,7 @@ gzipped_with_compression(const char *body, const size_t body_len, z_stream stream; memset(&stream, 0, sizeof(stream)); - stream.next_in = (unsigned char*)body; + stream.next_in = (unsigned char *)body; stream.avail_in = body_len; int err = deflateInit2(&stream, Z_DEFAULT_COMPRESSION, Z_DEFLATED, @@ -187,7 +187,7 @@ gzipped_with_compression(const char *body, const size_t body_len, } while (err == Z_OK) { - stream.next_out = (unsigned char*)(buffer + stream.total_out); + stream.next_out = (unsigned char *)(buffer + stream.total_out); stream.avail_out = len - stream.total_out; err = deflate(&stream, Z_FINISH); } diff --git a/tests/__init__.py b/tests/__init__.py index 83c85d243..61380f90b 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -170,7 +170,7 @@ def deserialize( # check if the data is gzip encoded and extract it before deserialization. # 0x1f8b: gzip-magic, 0x08: `DEFLATE` compression method. if data[:3] == b"\x1f\x8b\x08": - with gzip.open(io.BytesIO(data), 'rb') as output: + with gzip.open(io.BytesIO(data), "rb") as output: return cls.deserialize_from(output) return cls.deserialize_from(io.BytesIO(data)) From 57622e7045e81c0636036bc61729f29eafa43c8b Mon Sep 17 00:00:00 2001 From: Mischan Toosarani-Hausberger Date: Sun, 24 Mar 2024 14:26:31 +0100 Subject: [PATCH 04/17] Add msvc build config with a zlib compilation step --- .github/workflows/ci.yml | 8 ++++++++ scripts/install-zlib-msvc.ps1 | 27 +++++++++++++++++++++++++++ 2 files changed, 35 insertions(+) create mode 100644 scripts/install-zlib-msvc.ps1 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 996269245..d77871d9b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -98,6 +98,9 @@ jobs: TEST_X86: 1 - name: Windows (latest) os: windows-latest + - name: Windows (latest + transport-compression) + os: windows-latest + TRANSPORT_COMPRESSION: 1 - name: LLVM-Mingw os: windows-latest TEST_MINGW: 1 @@ -179,6 +182,11 @@ jobs: MINGW_ASM_MASM_COMPILER: ${{ matrix.MINGW_ASM_MASM_COMPILER }} run: . "scripts\install-llvm-mingw.ps1" + - name: Install MSVC zlib for transport compression tests + if: ${{ runner.os == 'Windows' && env['TRANSPORT_COMPRESSION'] }} + shell: powershell + run: . "scripts\install-zlib-msvc.ps1" + - name: Installing Android SDK Dependencies if: ${{ env['ANDROID_API'] }} run: | diff --git a/scripts/install-zlib-msvc.ps1 b/scripts/install-zlib-msvc.ps1 new file mode 100644 index 000000000..3d23da029 --- /dev/null +++ b/scripts/install-zlib-msvc.ps1 @@ -0,0 +1,27 @@ +$ZLIB_RELEASE = "1.3.1"; +$ZLIB_RELEASE_ASSET = "zlib131.zip" +$ZLIB_DL_URL = "https://github.com/madler/zlib/releases/download/v${ZLIB_RELEASE}/${ZLIB_RELEASE_ASSET}" +$ZLIB_DL_SHA512 = "1f171880153b0120e1364baaf7d0a17f65086eff279f8f8c8538e5950097d1feee37cc173181676ba1e2aeb4565ba68749c814cd3e25bfb06271bea02feb7d94" +$ZLIB_DL_PATH = "$env:GITHUB_WORKSPACE\dl\${ZLIB_RELEASE_ASSET}" +$CurlArguments = '-s', '-Lf', '-o', "${ZLIB_DL_PATH}", "${ZLIB_DL_URL}" +& curl.exe @CurlArguments +$zlib_zip_hash = Get-FileHash -LiteralPath "${ZLIB_DL_PATH}" -Algorithm SHA512 +if ($zlib_zip_hash.Hash -eq $ZLIB_DL_SHA512) { + Write-Host "Successfully downloaded ${ZLIB_RELEASE_ASSET}" +} +Else { + Write-Error "The downloaded ${ZLIB_RELEASE_ASSET} hash '$($zlib_zip_hash.Hash)' does not match the expected hash: '$ZLIB_DL_SHA512'" +} + +Write-Host "Extracting zlib source..." +$ZLIB_SOURCE_PATH = "$env:GITHUB_WORKSPACE\buildtools\zlib-${ZLIB_RELEASE}" +Expand-Archive -LiteralPath "${ZLIB_DL_PATH}" -DestinationPath "$env:GITHUB_WORKSPACE\buildtools" + +Write-Host "Building zlib source..." +$ZLIB_BUILD_PATH = "$env:GITHUB_WORKSPACE\buildtools\zlib_build" +cmake.exe -B "${ZLIB_BUILD_PATH}" -S "${ZLIB_SOURCE_PATH}" +cmake.exe --build "${ZLIB_BUILD_PATH}" --target zlibstatic +Copy-Item "${ZLIB_SOURCE_PATH}\zlib.h" "${ZLIB_BUILD_PATH}" + +# Add CMAKE_DEFINES +Write-Output "CMAKE_DEFINES=-DZLIB_LIBRARY=${ZLIB_BUILD_PATH}\libzlibstatic.a -DZLIB_INCLUDE_DIR=${ZLIB_BUILD_PATH}" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append \ No newline at end of file From 017d573e9428404b4696bd7568e18c5380b3930f Mon Sep 17 00:00:00 2001 From: Mischan Toosarani-Hausberger Date: Sun, 24 Mar 2024 14:34:57 +0100 Subject: [PATCH 05/17] exclude mingw from msvc zlib build step --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d77871d9b..92360a559 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -183,7 +183,7 @@ jobs: run: . "scripts\install-llvm-mingw.ps1" - name: Install MSVC zlib for transport compression tests - if: ${{ runner.os == 'Windows' && env['TRANSPORT_COMPRESSION'] }} + if: ${{ runner.os == 'Windows' && env['TRANSPORT_COMPRESSION'] && !env['TEST_MINGW'] }} shell: powershell run: . "scripts\install-zlib-msvc.ps1" From b52abe49b600d517088c3bf516afb6f8d379f8da Mon Sep 17 00:00:00 2001 From: Mischan Toosarani-Hausberger Date: Sun, 24 Mar 2024 14:40:42 +0100 Subject: [PATCH 06/17] ensure the download directory exists. --- scripts/install-zlib-msvc.ps1 | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/install-zlib-msvc.ps1 b/scripts/install-zlib-msvc.ps1 index 3d23da029..2208595f8 100644 --- a/scripts/install-zlib-msvc.ps1 +++ b/scripts/install-zlib-msvc.ps1 @@ -1,8 +1,11 @@ +$DL_BASEDIR = "$env:GITHUB_WORKSPACE\dl" +if (!(Test-Path -Path "$DL_BASEDIR")) { New-Item -ItemType Directory -Force -Path "$DL_BASEDIR" } + $ZLIB_RELEASE = "1.3.1"; $ZLIB_RELEASE_ASSET = "zlib131.zip" $ZLIB_DL_URL = "https://github.com/madler/zlib/releases/download/v${ZLIB_RELEASE}/${ZLIB_RELEASE_ASSET}" $ZLIB_DL_SHA512 = "1f171880153b0120e1364baaf7d0a17f65086eff279f8f8c8538e5950097d1feee37cc173181676ba1e2aeb4565ba68749c814cd3e25bfb06271bea02feb7d94" -$ZLIB_DL_PATH = "$env:GITHUB_WORKSPACE\dl\${ZLIB_RELEASE_ASSET}" +$ZLIB_DL_PATH = "${DL_BASEDIR}\${ZLIB_RELEASE_ASSET}" $CurlArguments = '-s', '-Lf', '-o', "${ZLIB_DL_PATH}", "${ZLIB_DL_URL}" & curl.exe @CurlArguments $zlib_zip_hash = Get-FileHash -LiteralPath "${ZLIB_DL_PATH}" -Algorithm SHA512 From 79166ca5017a96413c78a7929bf32675831d2c46 Mon Sep 17 00:00:00 2001 From: Mischan Toosarani-Hausberger Date: Sun, 24 Mar 2024 15:37:16 +0100 Subject: [PATCH 07/17] further build fixes on msvc --- scripts/install-zlib-msvc.ps1 | 2 +- src/sentry_transport.c | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/install-zlib-msvc.ps1 b/scripts/install-zlib-msvc.ps1 index 2208595f8..69ab6b907 100644 --- a/scripts/install-zlib-msvc.ps1 +++ b/scripts/install-zlib-msvc.ps1 @@ -27,4 +27,4 @@ cmake.exe --build "${ZLIB_BUILD_PATH}" --target zlibstatic Copy-Item "${ZLIB_SOURCE_PATH}\zlib.h" "${ZLIB_BUILD_PATH}" # Add CMAKE_DEFINES -Write-Output "CMAKE_DEFINES=-DZLIB_LIBRARY=${ZLIB_BUILD_PATH}\libzlibstatic.a -DZLIB_INCLUDE_DIR=${ZLIB_BUILD_PATH}" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append \ No newline at end of file +Write-Output "CMAKE_DEFINES=-DZLIB_LIBRARY=${ZLIB_BUILD_PATH}\zlibstatic.lib -DZLIB_INCLUDE_DIR=${ZLIB_BUILD_PATH}" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append \ No newline at end of file diff --git a/src/sentry_transport.c b/src/sentry_transport.c index 1912cc910..a01bb87d5 100644 --- a/src/sentry_transport.c +++ b/src/sentry_transport.c @@ -170,7 +170,7 @@ gzipped_with_compression(const char *body, const size_t body_len, z_stream stream; memset(&stream, 0, sizeof(stream)); stream.next_in = (unsigned char *)body; - stream.avail_in = body_len; + stream.avail_in = (unsigned int)body_len; int err = deflateInit2(&stream, Z_DEFAULT_COMPRESSION, Z_DEFLATED, MAX_WBITS + 16, 9, Z_DEFAULT_STRATEGY); @@ -179,7 +179,7 @@ gzipped_with_compression(const char *body, const size_t body_len, return false; } - size_t len = compressBound(body_len); + size_t len = compressBound((unsigned long)body_len); char *buffer = sentry_malloc(len); if (!buffer) { deflateEnd(&stream); @@ -188,7 +188,7 @@ gzipped_with_compression(const char *body, const size_t body_len, while (err == Z_OK) { stream.next_out = (unsigned char *)(buffer + stream.total_out); - stream.avail_out = len - stream.total_out; + stream.avail_out = (unsigned int)(len - stream.total_out); err = deflate(&stream, Z_FINISH); } From a7d7607026478292a6ce2be86a9955ddc8a835f7 Mon Sep 17 00:00:00 2001 From: Mischan Toosarani-Hausberger Date: Sun, 24 Mar 2024 15:43:35 +0100 Subject: [PATCH 08/17] yet another gha mscv build adaptation --- scripts/install-zlib-msvc.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/install-zlib-msvc.ps1 b/scripts/install-zlib-msvc.ps1 index 69ab6b907..09aab6fc0 100644 --- a/scripts/install-zlib-msvc.ps1 +++ b/scripts/install-zlib-msvc.ps1 @@ -27,4 +27,4 @@ cmake.exe --build "${ZLIB_BUILD_PATH}" --target zlibstatic Copy-Item "${ZLIB_SOURCE_PATH}\zlib.h" "${ZLIB_BUILD_PATH}" # Add CMAKE_DEFINES -Write-Output "CMAKE_DEFINES=-DZLIB_LIBRARY=${ZLIB_BUILD_PATH}\zlibstatic.lib -DZLIB_INCLUDE_DIR=${ZLIB_BUILD_PATH}" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append \ No newline at end of file +Write-Output "CMAKE_DEFINES=-DZLIB_LIBRARY=${ZLIB_BUILD_PATH}\Debug\zlibstaticd.lib -DZLIB_INCLUDE_DIR=${ZLIB_BUILD_PATH}" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append \ No newline at end of file From 1498331cde7e963fc2fdea1d80b0a909dabcc8e5 Mon Sep 17 00:00:00 2001 From: Mischan Toosarani-Hausberger Date: Sun, 24 Mar 2024 17:20:47 +0100 Subject: [PATCH 09/17] merge zlib setup for msvc and mingw --- .github/workflows/ci.yml | 6 ++-- scripts/install-llvm-mingw.ps1 | 30 ------------------- ...install-zlib-msvc.ps1 => install-zlib.ps1} | 16 ++++++++-- 3 files changed, 16 insertions(+), 36 deletions(-) rename scripts/{install-zlib-msvc.ps1 => install-zlib.ps1} (56%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 92360a559..a5bbcaa75 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -182,10 +182,10 @@ jobs: MINGW_ASM_MASM_COMPILER: ${{ matrix.MINGW_ASM_MASM_COMPILER }} run: . "scripts\install-llvm-mingw.ps1" - - name: Install MSVC zlib for transport compression tests - if: ${{ runner.os == 'Windows' && env['TRANSPORT_COMPRESSION'] && !env['TEST_MINGW'] }} + - name: Set up zlib for Windows + if: ${{ runner.os == 'Windows' }} shell: powershell - run: . "scripts\install-zlib-msvc.ps1" + run: . "scripts\install-zlib.ps1" - name: Installing Android SDK Dependencies if: ${{ env['ANDROID_API'] }} diff --git a/scripts/install-llvm-mingw.ps1 b/scripts/install-llvm-mingw.ps1 index b827d803b..827f9e390 100755 --- a/scripts/install-llvm-mingw.ps1 +++ b/scripts/install-llvm-mingw.ps1 @@ -52,33 +52,3 @@ Expand-Archive -LiteralPath "${NINJA_DL_PATH}" -DestinationPath "${NINJA_INSTALL # Export the NINJA executable path Write-Output "NINJA_INSTALL_PATH=${NINJA_INSTALL_PATH}" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append Write-Output "PATH=${NINJA_INSTALL_PATH}; $env:PATH" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append -$env:PATH = "${NINJA_INSTALL_PATH}; $env:PATH" - -# Download zlib -$ZLIB_RELEASE = "1.3.1"; -$ZLIB_RELEASE_ASSET = "zlib131.zip" -$ZLIB_DL_URL = "https://github.com/madler/zlib/releases/download/v${ZLIB_RELEASE}/${ZLIB_RELEASE_ASSET}" -$ZLIB_DL_SHA512 = "1f171880153b0120e1364baaf7d0a17f65086eff279f8f8c8538e5950097d1feee37cc173181676ba1e2aeb4565ba68749c814cd3e25bfb06271bea02feb7d94" -$ZLIB_DL_PATH = "${DL_BASEDIR}\${ZLIB_RELEASE_ASSET}" -$CurlArguments = '-s', '-Lf', '-o', "${ZLIB_DL_PATH}", "${ZLIB_DL_URL}" -& curl.exe @CurlArguments -$zlib_zip_hash = Get-FileHash -LiteralPath "${ZLIB_DL_PATH}" -Algorithm SHA512 -if ($zlib_zip_hash.Hash -eq $ZLIB_DL_SHA512) { - Write-Host "Successfully downloaded ${ZLIB_RELEASE_ASSET}" -} -Else { - Write-Error "The downloaded ${ZLIB_RELEASE_ASSET} hash '$($zlib_zip_hash.Hash)' does not match the expected hash: '$ZLIB_DL_SHA512'" -} - -Write-Host "Extracting zlib source..." -$ZLIB_SOURCE_PATH = "$env:GITHUB_WORKSPACE\buildtools\zlib-${ZLIB_RELEASE}" -Expand-Archive -LiteralPath "${ZLIB_DL_PATH}" -DestinationPath "$env:GITHUB_WORKSPACE\buildtools" - -Write-Host "Building zlib source..." -$ZLIB_BUILD_PATH = "$env:GITHUB_WORKSPACE\buildtools\zlib_build" -cmake.exe -B "${ZLIB_BUILD_PATH}" -S "${ZLIB_SOURCE_PATH}" -DCMAKE_C_COMPILER="${env:MINGW_PKG_PREFIX}-gcc" -DCMAKE_CXX_COMPILER="${env:MINGW_PKG_PREFIX}-g++" -DCMAKE_RC_COMPILER="${env:MINGW_PKG_PREFIX}-windres" -DCMAKE_ASM_MASM_COMPILER="${env:MINGW_ASM_MASM_COMPILER}" -GNinja -cmake.exe --build "${ZLIB_BUILD_PATH}" --target zlibstatic -Copy-Item "${ZLIB_SOURCE_PATH}\zlib.h" "${ZLIB_BUILD_PATH}" - -# Add CMAKE_DEFINES -Write-Output "CMAKE_DEFINES=-DZLIB_LIBRARY=${ZLIB_BUILD_PATH}\libzlibstatic.a -DZLIB_INCLUDE_DIR=${ZLIB_BUILD_PATH} -DCMAKE_C_COMPILER=${env:MINGW_PKG_PREFIX}-gcc -DCMAKE_CXX_COMPILER=${env:MINGW_PKG_PREFIX}-g++ -DCMAKE_RC_COMPILER=${env:MINGW_PKG_PREFIX}-windres -DCMAKE_ASM_MASM_COMPILER=${env:MINGW_ASM_MASM_COMPILER} -GNinja" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append diff --git a/scripts/install-zlib-msvc.ps1 b/scripts/install-zlib.ps1 similarity index 56% rename from scripts/install-zlib-msvc.ps1 rename to scripts/install-zlib.ps1 index 09aab6fc0..0f9ed7b67 100644 --- a/scripts/install-zlib-msvc.ps1 +++ b/scripts/install-zlib.ps1 @@ -22,9 +22,19 @@ Expand-Archive -LiteralPath "${ZLIB_DL_PATH}" -DestinationPath "$env:GITHUB_WORK Write-Host "Building zlib source..." $ZLIB_BUILD_PATH = "$env:GITHUB_WORKSPACE\buildtools\zlib_build" -cmake.exe -B "${ZLIB_BUILD_PATH}" -S "${ZLIB_SOURCE_PATH}" +if ($env:TEST_MINGW -eq 1) { + cmake.exe -B "${ZLIB_BUILD_PATH}" -S "${ZLIB_SOURCE_PATH}" -DCMAKE_C_COMPILER="${env:MINGW_PKG_PREFIX}-gcc" -DCMAKE_CXX_COMPILER="${env:MINGW_PKG_PREFIX}-g++" -DCMAKE_RC_COMPILER="${env:MINGW_PKG_PREFIX}-windres" -DCMAKE_ASM_MASM_COMPILER="${env:MINGW_ASM_MASM_COMPILER}" -GNinja +} +Else { + cmake.exe -B "${ZLIB_BUILD_PATH}" -S "${ZLIB_SOURCE_PATH}" +} cmake.exe --build "${ZLIB_BUILD_PATH}" --target zlibstatic Copy-Item "${ZLIB_SOURCE_PATH}\zlib.h" "${ZLIB_BUILD_PATH}" -# Add CMAKE_DEFINES -Write-Output "CMAKE_DEFINES=-DZLIB_LIBRARY=${ZLIB_BUILD_PATH}\Debug\zlibstaticd.lib -DZLIB_INCLUDE_DIR=${ZLIB_BUILD_PATH}" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append \ No newline at end of file +# Add CMAKE_DEFINES to the runner env. +if ($env:TEST_MINGW -eq 1) { + Write-Output "CMAKE_DEFINES=-DZLIB_LIBRARY=${ZLIB_BUILD_PATH}\libzlibstatic.a -DZLIB_INCLUDE_DIR=${ZLIB_BUILD_PATH} -DCMAKE_C_COMPILER=${env:MINGW_PKG_PREFIX}-gcc -DCMAKE_CXX_COMPILER=${env:MINGW_PKG_PREFIX}-g++ -DCMAKE_RC_COMPILER=${env:MINGW_PKG_PREFIX}-windres -DCMAKE_ASM_MASM_COMPILER=${env:MINGW_ASM_MASM_COMPILER} -GNinja" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append +} +Else { + Write-Output "CMAKE_DEFINES=-DZLIB_LIBRARY=${ZLIB_BUILD_PATH}\Debug\zlibstaticd.lib -DZLIB_INCLUDE_DIR=${ZLIB_BUILD_PATH}" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append +} From 9e50bb384786b88ccbc47f8c523743b6bcd1fb3c Mon Sep 17 00:00:00 2001 From: Mischan Toosarani-Hausberger Date: Sun, 24 Mar 2024 18:02:07 +0100 Subject: [PATCH 10/17] export CMAKE_DEFINES from llvm dep script... ...so that we can later reuse them an only append the platform-specific zlib CMAKE_DEFINES. --- scripts/install-llvm-mingw.ps1 | 8 ++++++-- scripts/install-zlib.ps1 | 10 ++++++---- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/scripts/install-llvm-mingw.ps1 b/scripts/install-llvm-mingw.ps1 index 827f9e390..c4f97da07 100755 --- a/scripts/install-llvm-mingw.ps1 +++ b/scripts/install-llvm-mingw.ps1 @@ -49,6 +49,10 @@ Write-Host "Extracting Ninja-Build..." $NINJA_INSTALL_PATH = "$env:GITHUB_WORKSPACE\buildtools\ninja" New-Item -ItemType Directory -Force -Path "${NINJA_INSTALL_PATH}" Expand-Archive -LiteralPath "${NINJA_DL_PATH}" -DestinationPath "${NINJA_INSTALL_PATH}" + # Export the NINJA executable path -Write-Output "NINJA_INSTALL_PATH=${NINJA_INSTALL_PATH}" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append -Write-Output "PATH=${NINJA_INSTALL_PATH}; $env:PATH" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append +"NINJA_INSTALL_PATH=${NINJA_INSTALL_PATH}" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append +"PATH=${NINJA_INSTALL_PATH}; $env:PATH" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append + +# export CMAKE_DEFINES to the runner environment +"CMAKE_DEFINES=-DCMAKE_C_COMPILER=${env:MINGW_PKG_PREFIX}-gcc -DCMAKE_CXX_COMPILER=${env:MINGW_PKG_PREFIX}-g++ -DCMAKE_RC_COMPILER=${env:MINGW_PKG_PREFIX}-windres -DCMAKE_ASM_MASM_COMPILER=${env:MINGW_ASM_MASM_COMPILER} -GNinja" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append \ No newline at end of file diff --git a/scripts/install-zlib.ps1 b/scripts/install-zlib.ps1 index 0f9ed7b67..cba5c31e9 100644 --- a/scripts/install-zlib.ps1 +++ b/scripts/install-zlib.ps1 @@ -1,3 +1,4 @@ +Set-PSDebug -Trace 1 $DL_BASEDIR = "$env:GITHUB_WORKSPACE\dl" if (!(Test-Path -Path "$DL_BASEDIR")) { New-Item -ItemType Directory -Force -Path "$DL_BASEDIR" } @@ -23,7 +24,7 @@ Expand-Archive -LiteralPath "${ZLIB_DL_PATH}" -DestinationPath "$env:GITHUB_WORK Write-Host "Building zlib source..." $ZLIB_BUILD_PATH = "$env:GITHUB_WORKSPACE\buildtools\zlib_build" if ($env:TEST_MINGW -eq 1) { - cmake.exe -B "${ZLIB_BUILD_PATH}" -S "${ZLIB_SOURCE_PATH}" -DCMAKE_C_COMPILER="${env:MINGW_PKG_PREFIX}-gcc" -DCMAKE_CXX_COMPILER="${env:MINGW_PKG_PREFIX}-g++" -DCMAKE_RC_COMPILER="${env:MINGW_PKG_PREFIX}-windres" -DCMAKE_ASM_MASM_COMPILER="${env:MINGW_ASM_MASM_COMPILER}" -GNinja + cmake.exe -B "${ZLIB_BUILD_PATH}" -S "${ZLIB_SOURCE_PATH}" -GNinja } Else { cmake.exe -B "${ZLIB_BUILD_PATH}" -S "${ZLIB_SOURCE_PATH}" @@ -31,10 +32,11 @@ Else { cmake.exe --build "${ZLIB_BUILD_PATH}" --target zlibstatic Copy-Item "${ZLIB_SOURCE_PATH}\zlib.h" "${ZLIB_BUILD_PATH}" -# Add CMAKE_DEFINES to the runner env. +# Append zlib CMAKE_DEFINES to the runner env. if ($env:TEST_MINGW -eq 1) { - Write-Output "CMAKE_DEFINES=-DZLIB_LIBRARY=${ZLIB_BUILD_PATH}\libzlibstatic.a -DZLIB_INCLUDE_DIR=${ZLIB_BUILD_PATH} -DCMAKE_C_COMPILER=${env:MINGW_PKG_PREFIX}-gcc -DCMAKE_CXX_COMPILER=${env:MINGW_PKG_PREFIX}-g++ -DCMAKE_RC_COMPILER=${env:MINGW_PKG_PREFIX}-windres -DCMAKE_ASM_MASM_COMPILER=${env:MINGW_ASM_MASM_COMPILER} -GNinja" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append + $NEW_CMAKE_DEFINES="${env:CMAKE_DEFINES};CMAKE_DEFINES=-DZLIB_LIBRARY=${ZLIB_BUILD_PATH}\libzlibstatic.a -DZLIB_INCLUDE_DIR=${ZLIB_BUILD_PATH} -GNinja" } Else { - Write-Output "CMAKE_DEFINES=-DZLIB_LIBRARY=${ZLIB_BUILD_PATH}\Debug\zlibstaticd.lib -DZLIB_INCLUDE_DIR=${ZLIB_BUILD_PATH}" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append + $NEW_CMAKE_DEFINES="${env:CMAKE_DEFINES};CMAKE_DEFINES=-DZLIB_LIBRARY=${ZLIB_BUILD_PATH}\Debug\zlibstaticd.lib -DZLIB_INCLUDE_DIR=${ZLIB_BUILD_PATH}" } +"$NEW_CMAKE_DEFINES" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append \ No newline at end of file From d14c0c25ca01958efad5842a24bd6e4bbbbfed38 Mon Sep 17 00:00:00 2001 From: Mischan Toosarani-Hausberger Date: Sun, 24 Mar 2024 18:15:12 +0100 Subject: [PATCH 11/17] reintroduce Write-Output --- scripts/install-llvm-mingw.ps1 | 6 +++--- scripts/install-zlib.ps1 | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/scripts/install-llvm-mingw.ps1 b/scripts/install-llvm-mingw.ps1 index c4f97da07..5610788be 100755 --- a/scripts/install-llvm-mingw.ps1 +++ b/scripts/install-llvm-mingw.ps1 @@ -51,8 +51,8 @@ New-Item -ItemType Directory -Force -Path "${NINJA_INSTALL_PATH}" Expand-Archive -LiteralPath "${NINJA_DL_PATH}" -DestinationPath "${NINJA_INSTALL_PATH}" # Export the NINJA executable path -"NINJA_INSTALL_PATH=${NINJA_INSTALL_PATH}" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append -"PATH=${NINJA_INSTALL_PATH}; $env:PATH" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append +Write-Output "NINJA_INSTALL_PATH=${NINJA_INSTALL_PATH}" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append +Write-Output "PATH=${NINJA_INSTALL_PATH}; $env:PATH" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append # export CMAKE_DEFINES to the runner environment -"CMAKE_DEFINES=-DCMAKE_C_COMPILER=${env:MINGW_PKG_PREFIX}-gcc -DCMAKE_CXX_COMPILER=${env:MINGW_PKG_PREFIX}-g++ -DCMAKE_RC_COMPILER=${env:MINGW_PKG_PREFIX}-windres -DCMAKE_ASM_MASM_COMPILER=${env:MINGW_ASM_MASM_COMPILER} -GNinja" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append \ No newline at end of file +Write-Output "CMAKE_DEFINES=-DCMAKE_C_COMPILER=${env:MINGW_PKG_PREFIX}-gcc -DCMAKE_CXX_COMPILER=${env:MINGW_PKG_PREFIX}-g++ -DCMAKE_RC_COMPILER=${env:MINGW_PKG_PREFIX}-windres -DCMAKE_ASM_MASM_COMPILER=${env:MINGW_ASM_MASM_COMPILER} -GNinja" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append \ No newline at end of file diff --git a/scripts/install-zlib.ps1 b/scripts/install-zlib.ps1 index cba5c31e9..2b6ad189b 100644 --- a/scripts/install-zlib.ps1 +++ b/scripts/install-zlib.ps1 @@ -39,4 +39,5 @@ if ($env:TEST_MINGW -eq 1) { Else { $NEW_CMAKE_DEFINES="${env:CMAKE_DEFINES};CMAKE_DEFINES=-DZLIB_LIBRARY=${ZLIB_BUILD_PATH}\Debug\zlibstaticd.lib -DZLIB_INCLUDE_DIR=${ZLIB_BUILD_PATH}" } -"$NEW_CMAKE_DEFINES" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append \ No newline at end of file +Write-Host $NEW_CMAKE_DEFINES +Write-Output $NEW_CMAKE_DEFINES | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append \ No newline at end of file From 1d175a9673d770c747c92f56178f243b4e81f649 Mon Sep 17 00:00:00 2001 From: Mischan Toosarani-Hausberger Date: Sun, 24 Mar 2024 18:18:20 +0100 Subject: [PATCH 12/17] actually assign new CMAKE_DEFINES before writing them to GITHUB_ENV --- scripts/install-zlib.ps1 | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/install-zlib.ps1 b/scripts/install-zlib.ps1 index 2b6ad189b..b95f10572 100644 --- a/scripts/install-zlib.ps1 +++ b/scripts/install-zlib.ps1 @@ -39,5 +39,4 @@ if ($env:TEST_MINGW -eq 1) { Else { $NEW_CMAKE_DEFINES="${env:CMAKE_DEFINES};CMAKE_DEFINES=-DZLIB_LIBRARY=${ZLIB_BUILD_PATH}\Debug\zlibstaticd.lib -DZLIB_INCLUDE_DIR=${ZLIB_BUILD_PATH}" } -Write-Host $NEW_CMAKE_DEFINES -Write-Output $NEW_CMAKE_DEFINES | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append \ No newline at end of file +"CMAKE_DEFINES=${NEW_CMAKE_DEFINES}" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append \ No newline at end of file From b3c777f12089ee6fad68fc9580e3e23a51bff579 Mon Sep 17 00:00:00 2001 From: Mischan Toosarani-Hausberger Date: Sun, 24 Mar 2024 18:25:03 +0100 Subject: [PATCH 13/17] get rid of PATH-like prepend. --- scripts/install-llvm-mingw.ps1 | 12 ++++++------ scripts/install-zlib.ps1 | 7 +++---- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/scripts/install-llvm-mingw.ps1 b/scripts/install-llvm-mingw.ps1 index 5610788be..0ca83cb29 100755 --- a/scripts/install-llvm-mingw.ps1 +++ b/scripts/install-llvm-mingw.ps1 @@ -26,10 +26,10 @@ New-Item -ItemType Directory -Force -Path "${LLVM_MINGW_INSTALL_PATH}" Expand-Archive -LiteralPath "${LLVM_MINGW_DL_PATH}" -DestinationPath "${LLVM_MINGW_INSTALL_PATH}" # Export the LLVM-mingw install path $LLVM_MINGW_INSTALL_PATH = "${LLVM_MINGW_INSTALL_PATH}\${LLVM_MINGW_PKG}" -Write-Output "LLVM_MINGW_INSTALL_PATH=${LLVM_MINGW_INSTALL_PATH}" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append +"LLVM_MINGW_INSTALL_PATH=${LLVM_MINGW_INSTALL_PATH}" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append # Prepend bin path to the system PATH -Write-Output "Path to LLVM-mingw bin folder: ${LLVM_MINGW_INSTALL_PATH}\bin" -Write-Output "${LLVM_MINGW_INSTALL_PATH}\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append +Write-Host "Path to LLVM-mingw bin folder: ${LLVM_MINGW_INSTALL_PATH}\bin" +"${LLVM_MINGW_INSTALL_PATH}\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append # Download ninja-build $NINJA_DL_URL = "https://github.com/ninja-build/ninja/releases/download/v1.11.1/ninja-win.zip" @@ -51,8 +51,8 @@ New-Item -ItemType Directory -Force -Path "${NINJA_INSTALL_PATH}" Expand-Archive -LiteralPath "${NINJA_DL_PATH}" -DestinationPath "${NINJA_INSTALL_PATH}" # Export the NINJA executable path -Write-Output "NINJA_INSTALL_PATH=${NINJA_INSTALL_PATH}" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append -Write-Output "PATH=${NINJA_INSTALL_PATH}; $env:PATH" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append +"NINJA_INSTALL_PATH=${NINJA_INSTALL_PATH}" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append +"PATH=${NINJA_INSTALL_PATH}; $env:PATH" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append # export CMAKE_DEFINES to the runner environment -Write-Output "CMAKE_DEFINES=-DCMAKE_C_COMPILER=${env:MINGW_PKG_PREFIX}-gcc -DCMAKE_CXX_COMPILER=${env:MINGW_PKG_PREFIX}-g++ -DCMAKE_RC_COMPILER=${env:MINGW_PKG_PREFIX}-windres -DCMAKE_ASM_MASM_COMPILER=${env:MINGW_ASM_MASM_COMPILER} -GNinja" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append \ No newline at end of file +"CMAKE_DEFINES=-DCMAKE_C_COMPILER=${env:MINGW_PKG_PREFIX}-gcc -DCMAKE_CXX_COMPILER=${env:MINGW_PKG_PREFIX}-g++ -DCMAKE_RC_COMPILER=${env:MINGW_PKG_PREFIX}-windres -DCMAKE_ASM_MASM_COMPILER=${env:MINGW_ASM_MASM_COMPILER} -GNinja" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append \ No newline at end of file diff --git a/scripts/install-zlib.ps1 b/scripts/install-zlib.ps1 index b95f10572..b8bc6b4ff 100644 --- a/scripts/install-zlib.ps1 +++ b/scripts/install-zlib.ps1 @@ -1,4 +1,3 @@ -Set-PSDebug -Trace 1 $DL_BASEDIR = "$env:GITHUB_WORKSPACE\dl" if (!(Test-Path -Path "$DL_BASEDIR")) { New-Item -ItemType Directory -Force -Path "$DL_BASEDIR" } @@ -34,9 +33,9 @@ Copy-Item "${ZLIB_SOURCE_PATH}\zlib.h" "${ZLIB_BUILD_PATH}" # Append zlib CMAKE_DEFINES to the runner env. if ($env:TEST_MINGW -eq 1) { - $NEW_CMAKE_DEFINES="${env:CMAKE_DEFINES};CMAKE_DEFINES=-DZLIB_LIBRARY=${ZLIB_BUILD_PATH}\libzlibstatic.a -DZLIB_INCLUDE_DIR=${ZLIB_BUILD_PATH} -GNinja" + $NEW_CMAKE_DEFINES="CMAKE_DEFINES=${env:CMAKE_DEFINES} -DZLIB_LIBRARY=${ZLIB_BUILD_PATH}\libzlibstatic.a -DZLIB_INCLUDE_DIR=${ZLIB_BUILD_PATH} -GNinja" } Else { - $NEW_CMAKE_DEFINES="${env:CMAKE_DEFINES};CMAKE_DEFINES=-DZLIB_LIBRARY=${ZLIB_BUILD_PATH}\Debug\zlibstaticd.lib -DZLIB_INCLUDE_DIR=${ZLIB_BUILD_PATH}" + $NEW_CMAKE_DEFINES="CMAKE_DEFINES=-DZLIB_LIBRARY=${ZLIB_BUILD_PATH}\Debug\zlibstaticd.lib -DZLIB_INCLUDE_DIR=${ZLIB_BUILD_PATH}" } -"CMAKE_DEFINES=${NEW_CMAKE_DEFINES}" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append \ No newline at end of file +$NEW_CMAKE_DEFINES | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append \ No newline at end of file From 6502d38ee3437cdec045baac5493e2c8a27e8e4e Mon Sep 17 00:00:00 2001 From: Mischan Toosarani-Hausberger Date: Sun, 24 Mar 2024 18:30:35 +0100 Subject: [PATCH 14/17] get rid of duplicate Ninja generator spec --- scripts/install-llvm-mingw.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/install-llvm-mingw.ps1 b/scripts/install-llvm-mingw.ps1 index 0ca83cb29..2af029272 100755 --- a/scripts/install-llvm-mingw.ps1 +++ b/scripts/install-llvm-mingw.ps1 @@ -55,4 +55,4 @@ Expand-Archive -LiteralPath "${NINJA_DL_PATH}" -DestinationPath "${NINJA_INSTALL "PATH=${NINJA_INSTALL_PATH}; $env:PATH" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append # export CMAKE_DEFINES to the runner environment -"CMAKE_DEFINES=-DCMAKE_C_COMPILER=${env:MINGW_PKG_PREFIX}-gcc -DCMAKE_CXX_COMPILER=${env:MINGW_PKG_PREFIX}-g++ -DCMAKE_RC_COMPILER=${env:MINGW_PKG_PREFIX}-windres -DCMAKE_ASM_MASM_COMPILER=${env:MINGW_ASM_MASM_COMPILER} -GNinja" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append \ No newline at end of file +"CMAKE_DEFINES=-DCMAKE_C_COMPILER=${env:MINGW_PKG_PREFIX}-gcc -DCMAKE_CXX_COMPILER=${env:MINGW_PKG_PREFIX}-g++ -DCMAKE_RC_COMPILER=${env:MINGW_PKG_PREFIX}-windres -DCMAKE_ASM_MASM_COMPILER=${env:MINGW_ASM_MASM_COMPILER}" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append \ No newline at end of file From c7fd3712deef4035fb5d5eb3fb7bd6f75714d8fe Mon Sep 17 00:00:00 2001 From: Mischan Toosarani-Hausberger Date: Sun, 24 Mar 2024 18:40:39 +0100 Subject: [PATCH 15/17] fix CHANGELOG.md --- CHANGELOG.md | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index feda93927..6fb6aac04 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,10 +1,19 @@ # Changelog -## 0.7.1 +## Unreleased **Features**: - Add optional Gzip transport compression via build option `SENTRY_TRANSPORT_COMPRESSION`. Requires system `zlib`. ([#954](https://github.com/getsentry/sentry-native/pull/954)) + +**Thank you**: + +- [@Strive-Sun](https://github.com/Strive-Sun) + +## 0.7.1 + +**Features**: + - Add user feedback capability to the Native SDK ([#966](https://github.com/getsentry/sentry-native/pull/966)) **Internal**: @@ -15,10 +24,6 @@ - Add usage of the breadcrumb `data` property to the example. [#951](https://github.com/getsentry/sentry-native/pull/951) -**Thank you**: - -- [@Strive-Sun](https://github.com/Strive-Sun) - ## 0.7.0 **Breaking changes**: From 340ebef9cfef0a930200c701e75bc79657e618c5 Mon Sep 17 00:00:00 2001 From: Mischan Toosarani-Hausberger Date: Fri, 29 Mar 2024 10:59:49 +0100 Subject: [PATCH 16/17] Enable transport-compression per test rather than per build --- .github/workflows/ci.yml | 26 +-------- tests/assertions.py | 12 +++- tests/cmake.py | 10 ++-- tests/test_integration_crashpad.py | 37 +++++++++---- tests/test_integration_http.py | 88 +++++++++++++++++++++++++----- 5 files changed, 115 insertions(+), 58 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a5bbcaa75..833b4269b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -53,13 +53,6 @@ jobs: CXX: clang++-15 ERROR_ON_WARNINGS: 1 RUN_ANALYZER: asan,llvm-cov - - name: Linux (clang + transport-compression + asan + llvm-cov) - os: ubuntu-22.04 - CC: clang-15 - CXX: clang++-15 - ERROR_ON_WARNINGS: 1 - TRANSPORT_COMPRESSION: 1 - RUN_ANALYZER: asan,llvm-cov - name: Linux (clang + kcov) os: ubuntu-22.04 CC: clang-15 @@ -85,33 +78,16 @@ jobs: ERROR_ON_WARNINGS: 1 SYSTEM_VERSION_COMPAT: 0 RUN_ANALYZER: asan,llvm-cov - - name: macOS (clang + transport-compression + asan + llvm-cov) - os: macOs-11 - CC: clang - CXX: clang++ - ERROR_ON_WARNINGS: 1 - SYSTEM_VERSION_COMPAT: 0 - TRANSPORT_COMPRESSION: 1 - RUN_ANALYZER: asan,llvm-cov - name: Windows (old VS, 32-bit) os: windows-2019 TEST_X86: 1 - name: Windows (latest) os: windows-latest - - name: Windows (latest + transport-compression) - os: windows-latest - TRANSPORT_COMPRESSION: 1 - name: LLVM-Mingw os: windows-latest TEST_MINGW: 1 MINGW_PKG_PREFIX: x86_64-w64-mingw32 MINGW_ASM_MASM_COMPILER: llvm-ml;-m64 - - name: LLVM-Mingw (transport-compression) - os: windows-latest - TEST_MINGW: 1 - MINGW_PKG_PREFIX: x86_64-w64-mingw32 - MINGW_ASM_MASM_COMPILER: llvm-ml;-m64 - TRANSPORT_COMPRESSION: 1 # The Android emulator is currently only available on macos, see: # https://docs.microsoft.com/en-us/azure/devops/pipelines/ecosystems/android?view=azure-devops#test-on-the-android-emulator - name: Android (old API/NDK) @@ -138,7 +114,7 @@ jobs: ANDROID_ARCH: ${{ matrix.ANDROID_ARCH }} CMAKE_DEFINES: ${{ matrix.CMAKE_DEFINES }} SYSTEM_VERSION_COMPAT: ${{ matrix.SYSTEM_VERSION_COMPAT }} - TRANSPORT_COMPRESSION: ${{ matrix.TRANSPORT_COMPRESSION }} + steps: - uses: actions/checkout@v3 with: diff --git a/tests/assertions.py b/tests/assertions.py index e3a0c11c9..9ab79c18a 100644 --- a/tests/assertions.py +++ b/tests/assertions.py @@ -4,7 +4,7 @@ import re import sys from dataclasses import dataclass -from datetime import datetime +from datetime import datetime, UTC import msgpack @@ -185,7 +185,7 @@ def assert_minidump(envelope): assert minidump.payload.bytes.startswith(b"MDMP") -def assert_timestamp(ts, now=datetime.utcnow()): +def assert_timestamp(ts, now=datetime.now(UTC)): assert ts[:11] == now.isoformat()[:11] @@ -307,3 +307,11 @@ def assert_crashpad_upload(req): and b"\n\nMDMP" in part.as_bytes() for part in msg.walk() ) + + +def assert_gzip_file_header(output): + assert output[:3] == b"\x1f\x8b\x08" + + +def assert_gzip_content_encoding(req): + assert req.content_encoding == "gzip" diff --git a/tests/cmake.py b/tests/cmake.py index eb75dd1aa..906b8afad 100644 --- a/tests/cmake.py +++ b/tests/cmake.py @@ -1,9 +1,10 @@ -import os import json -import sys +import os +import shutil import subprocess +import sys + import pytest -import shutil class CMake: @@ -138,9 +139,6 @@ def cmake(cwd, targets, options=None): if "tsan" in os.environ.get("RUN_ANALYZER", ""): configcmd.append("-DWITH_TSAN_OPTION=ON") - if os.environ.get("TRANSPORT_COMPRESSION"): - configcmd.append("-DSENTRY_TRANSPORT_COMPRESSION=ON") - # we have to set `-Werror` for this cmake invocation only, otherwise # completely unrelated things will break cflags = [] diff --git a/tests/test_integration_crashpad.py b/tests/test_integration_crashpad.py index fda314da6..28647bcc4 100644 --- a/tests/test_integration_crashpad.py +++ b/tests/test_integration_crashpad.py @@ -1,11 +1,17 @@ -import pytest import os import shutil import sys import time + +import pytest + from . import make_dsn, run, Envelope +from .assertions import ( + assert_crashpad_upload, + assert_session, + assert_gzip_file_header, +) from .conditions import has_crashpad -from .assertions import assert_crashpad_upload, assert_session pytestmark = pytest.mark.skipif(not has_crashpad, reason="tests need crashpad backend") @@ -118,13 +124,15 @@ def test_crashpad_wer_crash(cmake, httpserver, run_args): @pytest.mark.parametrize( - "run_args", + "run_args,build_args", [ # if we crash, we want a dump - ([]), + ([], {"SENTRY_TRANSPORT_COMPRESSION": "Off"}), + ([], {"SENTRY_TRANSPORT_COMPRESSION": "On"}), # if we crash and before-send doesn't discard, we want a dump pytest.param( ["before-send"], + {}, marks=pytest.mark.skipif( sys.platform == "darwin", reason="crashpad doesn't provide SetFirstChanceExceptionHandler on macOS", @@ -133,6 +141,7 @@ def test_crashpad_wer_crash(cmake, httpserver, run_args): # if on_crash() is non-discarding, a discarding before_send() is overruled, so we get a dump pytest.param( ["discarding-before-send", "on-crash"], + {}, marks=pytest.mark.skipif( sys.platform == "darwin", reason="crashpad doesn't provide SetFirstChanceExceptionHandler on macOS", @@ -140,8 +149,9 @@ def test_crashpad_wer_crash(cmake, httpserver, run_args): ), ], ) -def test_crashpad_dumping_crash(cmake, httpserver, run_args): - tmp_path = cmake(["sentry_example"], {"SENTRY_BACKEND": "crashpad"}) +def test_crashpad_dumping_crash(cmake, httpserver, run_args, build_args): + build_args.update({"SENTRY_BACKEND": "crashpad"}) + tmp_path = cmake(["sentry_example"], build_args) # make sure we are isolated from previous runs shutil.rmtree(tmp_path / ".sentry-native", ignore_errors=True) @@ -169,19 +179,24 @@ def test_crashpad_dumping_crash(cmake, httpserver, run_args): run(tmp_path, "sentry_example", ["log", "no-setup"], check=True, env=env) assert len(httpserver.log) == 2 - outputs = (httpserver.log[0][0], httpserver.log[1][0]) session, multipart = ( - (outputs[0].get_data(), outputs[1]) - if b'"type":"session"' in outputs[0].get_data() - else (outputs[1].get_data(), outputs[0]) + (httpserver.log[0][0], httpserver.log[1][0]) + if is_session_envelope(httpserver.log[0][0].get_data()) + else (httpserver.log[1][0], httpserver.log[0][0]) ) - envelope = Envelope.deserialize(session) + if build_args.get("SENTRY_TRANSPORT_COMPRESSION") == "On": + assert_gzip_file_header(session.get_data()) + envelope = Envelope.deserialize(session.get_data()) assert_session(envelope, {"status": "crashed", "errors": 1}) assert_crashpad_upload(multipart) +def is_session_envelope(data): + return b'"type":"session"' in data + + @pytest.mark.skipif( sys.platform == "darwin", reason="crashpad doesn't provide SetFirstChanceExceptionHandler on macOS", diff --git a/tests/test_integration_http.py b/tests/test_integration_http.py index 0cbd304da..1f5eb9132 100644 --- a/tests/test_integration_http.py +++ b/tests/test_integration_http.py @@ -19,6 +19,8 @@ assert_user_feedback, assert_minidump, assert_breakpad_crash, + assert_gzip_content_encoding, + assert_gzip_file_header, ) from .conditions import has_http, has_breakpad, has_files @@ -29,8 +31,16 @@ ) -def test_capture_http(cmake, httpserver): - tmp_path = cmake(["sentry_example"], {"SENTRY_BACKEND": "none"}) +@pytest.mark.parametrize( + "build_args", + [ + ({"SENTRY_TRANSPORT_COMPRESSION": "Off"}), + ({"SENTRY_TRANSPORT_COMPRESSION": "On"}), + ], +) +def test_capture_http(cmake, httpserver, build_args): + build_args.update({"SENTRY_BACKEND": "none"}) + tmp_path = cmake(["sentry_example"], build_args) httpserver.expect_oneshot_request( "/api/123456/envelope/", @@ -47,8 +57,14 @@ def test_capture_http(cmake, httpserver): ) assert len(httpserver.log) == 1 - output = httpserver.log[0][0].get_data() - envelope = Envelope.deserialize(output) + req = httpserver.log[0][0] + body = req.get_data() + + if build_args.get("SENTRY_TRANSPORT_COMPRESSION") == "On": + assert_gzip_content_encoding(req) + assert_gzip_file_header(body) + + envelope = Envelope.deserialize(body) assert_meta(envelope, "🤮🚀") assert_breadcrumb(envelope) @@ -232,8 +248,16 @@ def test_abnormal_session(cmake, httpserver): assert_session(envelope1, {"status": "abnormal", "errors": 0, "duration": 10}) -def test_inproc_crash_http(cmake, httpserver): - tmp_path = cmake(["sentry_example"], {"SENTRY_BACKEND": "inproc"}) +@pytest.mark.parametrize( + "build_args", + [ + ({"SENTRY_TRANSPORT_COMPRESSION": "Off"}), + ({"SENTRY_TRANSPORT_COMPRESSION": "On"}), + ], +) +def test_inproc_crash_http(cmake, httpserver, build_args): + build_args.update({"SENTRY_BACKEND": "inproc"}) + tmp_path = cmake(["sentry_example"], build_args) httpserver.expect_request( "/api/123456/envelope/", @@ -258,7 +282,14 @@ def test_inproc_crash_http(cmake, httpserver): ) assert len(httpserver.log) == 1 - envelope = Envelope.deserialize(httpserver.log[0][0].get_data()) + req = httpserver.log[0][0] + body = req.get_data() + + if build_args.get("SENTRY_TRANSPORT_COMPRESSION") == "On": + assert_gzip_content_encoding(req) + assert_gzip_file_header(body) + + envelope = Envelope.deserialize(body) assert_session(envelope, {"init": True, "status": "crashed", "errors": 1}) @@ -317,8 +348,16 @@ def test_inproc_dump_inflight(cmake, httpserver): @pytest.mark.skipif(not has_breakpad, reason="test needs breakpad backend") -def test_breakpad_crash_http(cmake, httpserver): - tmp_path = cmake(["sentry_example"], {"SENTRY_BACKEND": "breakpad"}) +@pytest.mark.parametrize( + "build_args", + [ + ({"SENTRY_TRANSPORT_COMPRESSION": "Off"}), + ({"SENTRY_TRANSPORT_COMPRESSION": "On"}), + ], +) +def test_breakpad_crash_http(cmake, httpserver, build_args): + build_args.update({"SENTRY_BACKEND": "breakpad"}) + tmp_path = cmake(["sentry_example"], build_args) httpserver.expect_request( "/api/123456/envelope/", @@ -343,7 +382,14 @@ def test_breakpad_crash_http(cmake, httpserver): ) assert len(httpserver.log) == 1 - envelope = Envelope.deserialize(httpserver.log[0][0].get_data()) + req = httpserver.log[0][0] + body = req.get_data() + + if build_args.get("SENTRY_TRANSPORT_COMPRESSION") == "On": + assert_gzip_content_encoding(req) + assert_gzip_file_header(body) + + envelope = Envelope.deserialize(body) assert_session(envelope, {"init": True, "status": "crashed", "errors": 1}) @@ -453,8 +499,16 @@ def delayed(req): RFC3339_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ" -def test_transaction_only(cmake, httpserver): - tmp_path = cmake(["sentry_example"], {"SENTRY_BACKEND": "none"}) +@pytest.mark.parametrize( + "build_args", + [ + ({"SENTRY_TRANSPORT_COMPRESSION": "Off"}), + ({"SENTRY_TRANSPORT_COMPRESSION": "On"}), + ], +) +def test_transaction_only(cmake, httpserver, build_args): + build_args.update({"SENTRY_BACKEND": "none"}) + tmp_path = cmake(["sentry_example"], build_args) httpserver.expect_oneshot_request( "/api/123456/envelope/", @@ -471,8 +525,14 @@ def test_transaction_only(cmake, httpserver): ) assert len(httpserver.log) == 1 - output = httpserver.log[0][0].get_data() - envelope = Envelope.deserialize(output) + req = httpserver.log[0][0] + body = req.get_data() + + if build_args.get("SENTRY_TRANSPORT_COMPRESSION") == "On": + assert_gzip_content_encoding(req) + assert_gzip_file_header(body) + + envelope = Envelope.deserialize(body) # Show what the envelope looks like if the test fails. envelope.print_verbose() From 5008fef54866b0bd953371783e68f59a6ed2a50d Mon Sep 17 00:00:00 2001 From: Mischan Toosarani-Hausberger Date: Fri, 29 Mar 2024 11:47:16 +0100 Subject: [PATCH 17/17] Since we now also run in MSVC 32-bit, create x86 build of zlib for MSVC --- scripts/install-zlib.ps1 | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/install-zlib.ps1 b/scripts/install-zlib.ps1 index b8bc6b4ff..c9b543638 100644 --- a/scripts/install-zlib.ps1 +++ b/scripts/install-zlib.ps1 @@ -25,6 +25,9 @@ $ZLIB_BUILD_PATH = "$env:GITHUB_WORKSPACE\buildtools\zlib_build" if ($env:TEST_MINGW -eq 1) { cmake.exe -B "${ZLIB_BUILD_PATH}" -S "${ZLIB_SOURCE_PATH}" -GNinja } +Elseif ($env:TEST_X86 -eq 1) { + cmake.exe -B "${ZLIB_BUILD_PATH}" -S "${ZLIB_SOURCE_PATH}" -AWin32 +} Else { cmake.exe -B "${ZLIB_BUILD_PATH}" -S "${ZLIB_SOURCE_PATH}" }