From b02d18e93f4fb1c675b88498281ced970898130a Mon Sep 17 00:00:00 2001 From: Lennart Dohmann Date: Thu, 8 Feb 2024 08:33:02 +0100 Subject: [PATCH 1/4] Implement scan for stream and tests --- .../Vaas/Messages/VerdictRequestForStream.cs | 33 +++++++++ dotnet/Vaas/src/Vaas/Vaas.cs | 71 ++++++++++++++++++ .../Vaas/test/Vaas.Test/IntegrationTests.cs | 74 +++++++++++++++++++ 3 files changed, 178 insertions(+) create mode 100644 dotnet/Vaas/src/Vaas/Messages/VerdictRequestForStream.cs diff --git a/dotnet/Vaas/src/Vaas/Messages/VerdictRequestForStream.cs b/dotnet/Vaas/src/Vaas/Messages/VerdictRequestForStream.cs new file mode 100644 index 00000000..e07af587 --- /dev/null +++ b/dotnet/Vaas/src/Vaas/Messages/VerdictRequestForStream.cs @@ -0,0 +1,33 @@ +using System.Collections.Generic; +using System.Text.Json.Serialization; + +namespace Vaas.Messages; + +public class VerdictRequestForStream +{ + [JsonPropertyName("kind")] + public string Kind => "VerdictRequestForStream"; + + [JsonPropertyName("guid")] + public string Guid { get; } + + [JsonPropertyName("session_id")] + public string SessionId { get; } + + [JsonPropertyName("verdict_request_attributes")] + public Dictionary? VerdictRequestAttributes { get; set; } + + [JsonPropertyName("use_cache")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public bool? UseCache { get; init; } + + [JsonPropertyName("use_hash_lookup")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public bool? UseHashLookup { get; init; } + + public VerdictRequestForStream(string sessionId) + { + SessionId = sessionId; + Guid = System.Guid.NewGuid().ToString(); + } +} \ No newline at end of file diff --git a/dotnet/Vaas/src/Vaas/Vaas.cs b/dotnet/Vaas/src/Vaas/Vaas.cs index aad03e92..dc6f197d 100644 --- a/dotnet/Vaas/src/Vaas/Vaas.cs +++ b/dotnet/Vaas/src/Vaas/Vaas.cs @@ -183,6 +183,69 @@ public async Task ForUrlAsync(Uri uri, CancellationToken cancellati return new VaasVerdict(verdictResponse); } + public async Task ForStreamAsync( + Stream stream, + long contentLength, + CancellationToken cancellationToken, + Dictionary? verdictRequestAttributes = null + ) + { + if (stream == null || contentLength <= 0) + throw new VaasClientException("Stream or content length was null."); + + var verdictResponse = await ForStreamRequestAsync( + new VerdictRequestForStream(SessionId ?? throw new InvalidOperationException()) + { + UseCache = _options.UseCache, + UseHashLookup = _options.UseHashLookup, + VerdictRequestAttributes = verdictRequestAttributes + }); + if (!verdictResponse.IsValid) + throw new JsonException("VerdictResponse is not valid"); + if (verdictResponse.Verdict != Verdict.Unknown) + throw new VaasServerException("Server returned verdict without receiving content."); + + if ( + string.IsNullOrWhiteSpace(verdictResponse.Url) + || string.IsNullOrWhiteSpace(verdictResponse.UploadToken) + ) + { + throw new JsonException( + "VerdictResponse missing URL or UploadToken for stream upload." + ); + } + + var response = WaitForResponseAsync(verdictResponse.Guid); + await UploadStream(stream, contentLength, verdictResponse.Url, verdictResponse.UploadToken, cancellationToken); + + return new VaasVerdict(await response); + } + + private async Task UploadStream(Stream stream, long contentLength, string url, string token, CancellationToken cancellationToken) + { + using var requestContent = new StreamContent(stream, (int)contentLength); + using var requestMessage = new HttpRequestMessage(HttpMethod.Put, url); + requestMessage.Content = requestContent; + requestMessage.Headers.Authorization = new AuthenticationHeaderValue(token); + + var response = await _uploadHttpClient.SendAsync(requestMessage, cancellationToken); + if (!response.IsSuccessStatusCode) + { + var responseBody = await response.Content.ReadAsStringAsync(cancellationToken); + ProblemDetails? problemDetails; + try + { + problemDetails = JsonSerializer.Deserialize(responseBody); + } + catch (JsonException) + { + throw new VaasServerException("Server did not return proper ProblemDetails"); + } + + throw ProblemDetailsToException(problemDetails); + } + } + public async Task ForSha256Async(ChecksumSha256 sha256, CancellationToken cancellationToken, ForSha256Options? options = null) { var url = _options.Url; @@ -312,6 +375,14 @@ private async Task ForRequestAsync(VerdictRequest verdictReques return await WaitForResponseAsync(verdictRequest.Guid); } + + private async Task ForStreamRequestAsync(VerdictRequestForStream verdictRequest) + { + var jsonString = JsonSerializer.Serialize(verdictRequest); + AuthenticatedClient.Send(jsonString); + + return await WaitForResponseAsync(verdictRequest.Guid); + } private async Task ForUrlRequestAsync(VerdictRequestForUrl verdictRequestForUrl) { diff --git a/dotnet/Vaas/test/Vaas.Test/IntegrationTests.cs b/dotnet/Vaas/test/Vaas.Test/IntegrationTests.cs index c5302d4b..b8c3fe0f 100644 --- a/dotnet/Vaas/test/Vaas.Test/IntegrationTests.cs +++ b/dotnet/Vaas/test/Vaas.Test/IntegrationTests.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.IO; +using System.Net.Http; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Configuration; @@ -19,6 +20,7 @@ public class IntegrationTests "wss://gateway.production.vaas.gdatasecurity.de")); private readonly ITestOutputHelper _output; + private readonly HttpClient _httpClient = new(); public IntegrationTests(ITestOutputHelper output) { @@ -178,6 +180,78 @@ public async Task ForUrl_WithUrlWithStatusCode4xx_ThrowsVaasClientException() "Call failed with status code 404 (Not Found): GET https://upload.production.vaas.gdatasecurity.de/nocontenthere", e.Message); } + + [Fact] + public async Task ForStream_WithEicarString_ReturnsMalicious() + { + // Arrange + var vaas = await AuthenticateWithCredentials(); + var targetStream = new MemoryStream(); + var eicarBytes = System.Text.Encoding.UTF8.GetBytes("X5O!P%@AP[4\\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-FILE!$H+H*"); + targetStream.Write(eicarBytes, 0, eicarBytes.Length); + targetStream.Position = 0; + var contentLength = targetStream.Length; + + // Act + var verdict = await vaas.ForStreamAsync(targetStream, contentLength, CancellationToken.None); + + // Assert + Assert.Equal(Verdict.Malicious, verdict.Verdict); + } + + [Fact] + public async Task ForStream_WithCleanString_ReturnsClean() + { + // Arrange + var vaas = await AuthenticateWithCredentials(); + var targetStream = new MemoryStream(); + var cleanBytes = System.Text.Encoding.UTF8.GetBytes("This is a clean file"); + targetStream.Write(cleanBytes, 0, cleanBytes.Length); + targetStream.Position = 0; + var contentLength = targetStream.Length; + + // Act + var verdict = await vaas.ForStreamAsync(targetStream, contentLength, CancellationToken.None); + + // Assert + Assert.Equal(Verdict.Clean, verdict.Verdict); + } + + [Fact] + public async Task ForStream_WithCleanUrl_ReturnsClean() + { + // Arrange + var vaas = await AuthenticateWithCredentials(); + var url = new Uri("https://raw.githubusercontent.com/GDATASoftwareAG/vaas/main/Readme.md"); + var response = await _httpClient.SendAsync(new HttpRequestMessage(HttpMethod.Get, url), CancellationToken.None); + var targetStream = await response.Content.ReadAsStreamAsync(); + var contentLength = response.Content.Headers.ContentLength; + Assert.NotNull(contentLength); + + // Act + var verdict = await vaas.ForStreamAsync(targetStream, (long)contentLength, CancellationToken.None); + + // Assert + Assert.Equal(Verdict.Clean, verdict.Verdict); + } + + [Fact] + public async Task ForStream_WithEicarUrl_ReturnsEicar() + { + // Arrange + var vaas = await AuthenticateWithCredentials(); + var url = new Uri("https://secure.eicar.org/eicar.com.txt"); + var response = await _httpClient.SendAsync(new HttpRequestMessage(HttpMethod.Get, url), CancellationToken.None); + var targetStream = await response.Content.ReadAsStreamAsync(); + var contentLength = response.Content.Headers.ContentLength; + Assert.NotNull(contentLength); + + // Act + var verdict = await vaas.ForStreamAsync(targetStream, (long)contentLength, CancellationToken.None); + + // Assert + Assert.Equal(Verdict.Malicious, verdict.Verdict); + } private async Task AuthenticateWithCredentials() { From d975d583ee41bfa549c8591ab3a47741e21899a6 Mon Sep 17 00:00:00 2001 From: Lennart Dohmann Date: Thu, 8 Feb 2024 08:40:58 +0100 Subject: [PATCH 2/4] Let dotnet handle contentLength --- dotnet/Vaas/src/Vaas/Vaas.cs | 11 +++++------ dotnet/Vaas/test/Vaas.Test/IntegrationTests.cs | 14 ++++---------- 2 files changed, 9 insertions(+), 16 deletions(-) diff --git a/dotnet/Vaas/src/Vaas/Vaas.cs b/dotnet/Vaas/src/Vaas/Vaas.cs index dc6f197d..b20054c8 100644 --- a/dotnet/Vaas/src/Vaas/Vaas.cs +++ b/dotnet/Vaas/src/Vaas/Vaas.cs @@ -185,13 +185,12 @@ public async Task ForUrlAsync(Uri uri, CancellationToken cancellati public async Task ForStreamAsync( Stream stream, - long contentLength, CancellationToken cancellationToken, Dictionary? verdictRequestAttributes = null ) { - if (stream == null || contentLength <= 0) - throw new VaasClientException("Stream or content length was null."); + if (stream == null) + throw new VaasClientException("Stream was null."); var verdictResponse = await ForStreamRequestAsync( new VerdictRequestForStream(SessionId ?? throw new InvalidOperationException()) @@ -216,14 +215,14 @@ public async Task ForStreamAsync( } var response = WaitForResponseAsync(verdictResponse.Guid); - await UploadStream(stream, contentLength, verdictResponse.Url, verdictResponse.UploadToken, cancellationToken); + await UploadStream(stream, verdictResponse.Url, verdictResponse.UploadToken, cancellationToken); return new VaasVerdict(await response); } - private async Task UploadStream(Stream stream, long contentLength, string url, string token, CancellationToken cancellationToken) + private async Task UploadStream(Stream stream, string url, string token, CancellationToken cancellationToken) { - using var requestContent = new StreamContent(stream, (int)contentLength); + using var requestContent = new StreamContent(stream); using var requestMessage = new HttpRequestMessage(HttpMethod.Put, url); requestMessage.Content = requestContent; requestMessage.Headers.Authorization = new AuthenticationHeaderValue(token); diff --git a/dotnet/Vaas/test/Vaas.Test/IntegrationTests.cs b/dotnet/Vaas/test/Vaas.Test/IntegrationTests.cs index b8c3fe0f..83c26811 100644 --- a/dotnet/Vaas/test/Vaas.Test/IntegrationTests.cs +++ b/dotnet/Vaas/test/Vaas.Test/IntegrationTests.cs @@ -190,10 +190,9 @@ public async Task ForStream_WithEicarString_ReturnsMalicious() var eicarBytes = System.Text.Encoding.UTF8.GetBytes("X5O!P%@AP[4\\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-FILE!$H+H*"); targetStream.Write(eicarBytes, 0, eicarBytes.Length); targetStream.Position = 0; - var contentLength = targetStream.Length; // Act - var verdict = await vaas.ForStreamAsync(targetStream, contentLength, CancellationToken.None); + var verdict = await vaas.ForStreamAsync(targetStream, CancellationToken.None); // Assert Assert.Equal(Verdict.Malicious, verdict.Verdict); @@ -208,10 +207,9 @@ public async Task ForStream_WithCleanString_ReturnsClean() var cleanBytes = System.Text.Encoding.UTF8.GetBytes("This is a clean file"); targetStream.Write(cleanBytes, 0, cleanBytes.Length); targetStream.Position = 0; - var contentLength = targetStream.Length; // Act - var verdict = await vaas.ForStreamAsync(targetStream, contentLength, CancellationToken.None); + var verdict = await vaas.ForStreamAsync(targetStream, CancellationToken.None); // Assert Assert.Equal(Verdict.Clean, verdict.Verdict); @@ -225,11 +223,9 @@ public async Task ForStream_WithCleanUrl_ReturnsClean() var url = new Uri("https://raw.githubusercontent.com/GDATASoftwareAG/vaas/main/Readme.md"); var response = await _httpClient.SendAsync(new HttpRequestMessage(HttpMethod.Get, url), CancellationToken.None); var targetStream = await response.Content.ReadAsStreamAsync(); - var contentLength = response.Content.Headers.ContentLength; - Assert.NotNull(contentLength); // Act - var verdict = await vaas.ForStreamAsync(targetStream, (long)contentLength, CancellationToken.None); + var verdict = await vaas.ForStreamAsync(targetStream, CancellationToken.None); // Assert Assert.Equal(Verdict.Clean, verdict.Verdict); @@ -243,11 +239,9 @@ public async Task ForStream_WithEicarUrl_ReturnsEicar() var url = new Uri("https://secure.eicar.org/eicar.com.txt"); var response = await _httpClient.SendAsync(new HttpRequestMessage(HttpMethod.Get, url), CancellationToken.None); var targetStream = await response.Content.ReadAsStreamAsync(); - var contentLength = response.Content.Headers.ContentLength; - Assert.NotNull(contentLength); // Act - var verdict = await vaas.ForStreamAsync(targetStream, (long)contentLength, CancellationToken.None); + var verdict = await vaas.ForStreamAsync(targetStream, CancellationToken.None); // Assert Assert.Equal(Verdict.Malicious, verdict.Verdict); From a4f27c4901bfda8d092849bf83f71f8397b8ecf6 Mon Sep 17 00:00:00 2001 From: Lennart Dohmann Date: Fri, 16 Feb 2024 11:26:10 +0100 Subject: [PATCH 3/4] Update .NET Core version to 8.0 --- .github/workflows/ci-dotnet.yaml | 35 +++++++++++++++++---- dotnet/Vaas/src/Vaas/Vaas.csproj | 2 +- dotnet/Vaas/test/TestFiles/TestFiles.csproj | 2 +- dotnet/Vaas/test/Vaas.Test/Vaas.Test.csproj | 2 +- dotnet/Vaas/test/Vaas.Test/VaasTest.cs | 4 +-- 5 files changed, 34 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci-dotnet.yaml b/.github/workflows/ci-dotnet.yaml index a5ac66c3..1465227d 100644 --- a/.github/workflows/ci-dotnet.yaml +++ b/.github/workflows/ci-dotnet.yaml @@ -22,16 +22,17 @@ on: options: - production - staging + - develop default: "production" env: CLIENT_ID: ${{ secrets.CLIENT_ID }} - CLIENT_SECRET: ${{ (inputs.environment == 'production' || inputs.environment == null || startsWith(github.ref, 'refs/tags/cs')) && secrets.CLIENT_SECRET || secrets.STAGING_CLIENT_SECRET }} - VAAS_URL: ${{ (inputs.environment == 'production' || inputs.environment == null || startsWith(github.ref, 'refs/tags/cs')) && 'wss://gateway.production.vaas.gdatasecurity.de' || 'wss://gateway.staging.vaas.gdatasecurity.de' }} - TOKEN_URL: ${{ (inputs.environment == 'production' || inputs.environment == null || startsWith(github.ref, 'refs/tags/cs')) && 'https://account.gdata.de/realms/vaas-production/protocol/openid-connect/token' || 'https://account-staging.gdata.de/realms/vaas-staging/protocol/openid-connect/token' }} + CLIENT_SECRET: ${{secrets.CLIENT_SECRET}} + VAAS_URL: 'wss://gateway.production.vaas.gdatasecurity.de' + TOKEN_URL: 'https://account.gdata.de/realms/vaas-production/protocol/openid-connect/token' VAAS_CLIENT_ID: ${{ secrets.VAAS_CLIENT_ID }} VAAS_USER_NAME: ${{ secrets.VAAS_USER_NAME }} - VAAS_PASSWORD: ${{ (inputs.environment == 'production' || inputs.environment == null || startsWith(github.ref, 'refs/tags/cs')) && secrets.VAAS_PASSWORD || secrets.STAGING_VAAS_PASSWORD }} + VAAS_PASSWORD: ${{secrets.VAAS_PASSWORD}} jobs: build-dotnet: @@ -39,13 +40,35 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - dotnet-version: ["6.0.x"] + dotnet-version: ["8.0.x"] steps: - uses: actions/checkout@v3 + + - name: set staging environment + if: (inputs.environment == 'staging' || (startsWith(github.ref, 'refs/tags/java') && endsWith(github.ref, '-beta'))) + run: | + echo "CLIENT_ID=${{ secrets.STAGING_CLIENT_ID }}" >> $GITHUB_ENV + echo "CLIENT_SECRET=${{ secrets.STAGING_CLIENT_SECRET }}" >> $GITHUB_ENV + echo "VAAS_URL=wss://gateway.staging.vaas.gdatasecurity.de" >> $GITHUB_ENV + echo "TOKEN_URL=https://account-staging.gdata.de/realms/vaas-staging/protocol/openid-connect/token" >> $GITHUB_ENV + echo "VAAS_CLIENT_ID=${{ secrets.STAGING_VAAS_CLIENT_ID }}" >> $GITHUB_ENV + echo "VAAS_USER_NAME=${{ secrets.STAGING_VAAS_USER_NAME }}" >> $GITHUB_ENV + echo "VAAS_PASSWORD=${{ secrets.STAGING_VAAS_PASSWORD }}" >> $GITHUB_ENV + + - name: set develop environment + if: (inputs.environment == 'develop' || (startsWith(github.ref, 'refs/tags/java') && endsWith(github.ref, '-alpha'))) + run: | + echo "CLIENT_ID=${{ secrets.DEVELOP_CLIENT_ID }}" >> $GITHUB_ENV + echo "CLIENT_SECRET=${{ secrets.DEVELOP_CLIENT_SECRET }}" >> $GITHUB_ENV + echo "VAAS_URL=wss://gateway.develop.vaas.gdatasecurity.de" >> $GITHUB_ENV + echo "TOKEN_URL=https://account-staging.gdata.de/realms/vaas-develop/protocol/openid-connect/token" >> $GITHUB_ENV + echo "VAAS_CLIENT_ID=${{ secrets.DEVELOP_VAAS_CLIENT_ID }}" >> $GITHUB_ENV + echo "VAAS_USER_NAME=${{ secrets.DEVELOP_VAAS_USER_NAME }}" >> $GITHUB_ENV + echo "VAAS_PASSWORD=${{ secrets.DEVELOP_VAAS_PASSWORD }}" >> $GITHUB_ENV - name: Setup .NET Core SDK ${{ matrix.dotnet-version }} - uses: actions/setup-dotnet@v3 + uses: actions/setup-dotnet@v4 with: dotnet-version: ${{ matrix.dotnet-version }} diff --git a/dotnet/Vaas/src/Vaas/Vaas.csproj b/dotnet/Vaas/src/Vaas/Vaas.csproj index a5dd6b4b..191c6933 100644 --- a/dotnet/Vaas/src/Vaas/Vaas.csproj +++ b/dotnet/Vaas/src/Vaas/Vaas.csproj @@ -3,7 +3,7 @@ 0.0.0 MIT - net6.0 + net8.0 enable opensource@gdata.de G DATA CyberDefense AG diff --git a/dotnet/Vaas/test/TestFiles/TestFiles.csproj b/dotnet/Vaas/test/TestFiles/TestFiles.csproj index c3f3c87e..a2b06607 100644 --- a/dotnet/Vaas/test/TestFiles/TestFiles.csproj +++ b/dotnet/Vaas/test/TestFiles/TestFiles.csproj @@ -2,7 +2,7 @@ Exe - net6.0 + net8.0 enable enable diff --git a/dotnet/Vaas/test/Vaas.Test/Vaas.Test.csproj b/dotnet/Vaas/test/Vaas.Test/Vaas.Test.csproj index 7442f650..90fb38f0 100644 --- a/dotnet/Vaas/test/Vaas.Test/Vaas.Test.csproj +++ b/dotnet/Vaas/test/Vaas.Test/Vaas.Test.csproj @@ -1,7 +1,7 @@ - net6.0 + net8.0 false diff --git a/dotnet/Vaas/test/Vaas.Test/VaasTest.cs b/dotnet/Vaas/test/Vaas.Test/VaasTest.cs index 1c609318..412a68c8 100644 --- a/dotnet/Vaas/test/Vaas.Test/VaasTest.cs +++ b/dotnet/Vaas/test/Vaas.Test/VaasTest.cs @@ -47,10 +47,10 @@ public async Task ForSha256Async_SendsUserAgent() [Fact] public void Constructor_IfRelativeUrl_ThrowsVaasClientException() { - var e = Assert.Throws(() => + var e = Assert.Throws(() => new Vaas(_httpClient, _authenticator.Object, new VaasOptions() { Url = new Uri("/relative") })); Assert.Equal( - "Parameter \"options.Url.Host\" (string) must not be null or whitespace, was whitespace. (Parameter 'options.Url.Host')", + "Invalid URI: The format of the URI could not be determined.", e.Message); } From a9bb8595d642d95314b3280a639bea06a2b70f5f Mon Sep 17 00:00:00 2001 From: Lennart Dohmann Date: Fri, 16 Feb 2024 11:48:55 +0100 Subject: [PATCH 4/4] Fix test --- dotnet/Vaas/test/Vaas.Test/VaasTest.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dotnet/Vaas/test/Vaas.Test/VaasTest.cs b/dotnet/Vaas/test/Vaas.Test/VaasTest.cs index 412a68c8..1c609318 100644 --- a/dotnet/Vaas/test/Vaas.Test/VaasTest.cs +++ b/dotnet/Vaas/test/Vaas.Test/VaasTest.cs @@ -47,10 +47,10 @@ public async Task ForSha256Async_SendsUserAgent() [Fact] public void Constructor_IfRelativeUrl_ThrowsVaasClientException() { - var e = Assert.Throws(() => + var e = Assert.Throws(() => new Vaas(_httpClient, _authenticator.Object, new VaasOptions() { Url = new Uri("/relative") })); Assert.Equal( - "Invalid URI: The format of the URI could not be determined.", + "Parameter \"options.Url.Host\" (string) must not be null or whitespace, was whitespace. (Parameter 'options.Url.Host')", e.Message); }