From 655fbdd33af6f1934a6e67d4013a2cb56c245465 Mon Sep 17 00:00:00 2001 From: Varun Puranik Date: Fri, 7 Jun 2019 15:55:55 -0700 Subject: [PATCH] EdgeAgent: Fix log upload formatting (#1310) * Fix logs upload issues * Fix stylecop --- .../logs/LogsProvider.cs | 41 +------------------ .../requests/LogsRequestHandler.cs | 24 ++++++++++- .../requests/LogsUploadRequest.cs | 2 +- .../requests/LogsUploadRequestHandler.cs | 10 ++++- .../requests/RequestManager.cs | 13 +----- .../logs/LogsProviderTest.cs | 9 +--- 6 files changed, 37 insertions(+), 62 deletions(-) diff --git a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Core/logs/LogsProvider.cs b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Core/logs/LogsProvider.cs index 6ec788e0dbe..1cce991faa9 100644 --- a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Core/logs/LogsProvider.cs +++ b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Core/logs/LogsProvider.cs @@ -54,13 +54,6 @@ public Task GetLogsStream(IList<(string id, ModuleLogOptions logOptions)> ids, F return Task.WhenAll(streamingTasks); } - internal static bool NeedToProcessStream(ModuleLogOptions logOptions) => - logOptions.Filter.LogLevel.HasValue - || logOptions.Filter.Regex.HasValue - || logOptions.ContentEncoding != LogsContentEncoding.None - || logOptions.ContentType != LogsContentType.Text - || logOptions.OutputFraming != LogOutputFraming.None; - internal async Task GetLogsStreamInternal(string id, ModuleLogOptions logOptions, Func, Task> callback, CancellationToken cancellationToken) { Preconditions.CheckNotNull(logOptions, nameof(logOptions)); @@ -69,9 +62,7 @@ internal async Task GetLogsStreamInternal(string id, ModuleLogOptions logOptions Stream logsStream = await this.runtimeInfoProvider.GetModuleLogs(id, logOptions.Follow, logOptions.Filter.Tail, logOptions.Filter.Since, cancellationToken); Events.ReceivedStream(id); - await (NeedToProcessStream(logOptions) - ? this.logsProcessor.ProcessLogsStream(id, logsStream, logOptions, callback) - : this.WriteLogsStreamToOutput(id, callback, logsStream, cancellationToken)); + await this.logsProcessor.ProcessLogsStream(id, logsStream, logOptions, callback); } static byte[] ProcessByContentEncoding(byte[] bytes, LogsContentEncoding contentEncoding) => @@ -79,36 +70,6 @@ static byte[] ProcessByContentEncoding(byte[] bytes, LogsContentEncoding content ? Compression.CompressToGzip(bytes) : bytes; - async Task WriteLogsStreamToOutput(string id, Func, Task> callback, Stream stream, CancellationToken cancellationToken) - { - var buf = new byte[1024]; - try - { - while (true) - { - if (cancellationToken.IsCancellationRequested) - { - Events.StreamingCancelled(id); - break; - } - - int count = await stream.ReadAsync(buf, 0, buf.Length, cancellationToken); - if (count == 0) - { - Events.StreamingCompleted(id); - break; - } - - var arrSeg = new ArraySegment(buf, 0, count); - await callback(arrSeg); - } - } - catch (Exception ex) - { - Events.ErrorWhileProcessingStream(id, ex); - } - } - async Task GetProcessedLogs(string id, Stream logsStream, ModuleLogOptions logOptions) { byte[] logBytes = await this.ProcessByContentType(id, logsStream, logOptions); diff --git a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Core/requests/LogsRequestHandler.cs b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Core/requests/LogsRequestHandler.cs index 735df3cca63..a27cd76ab5e 100644 --- a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Core/requests/LogsRequestHandler.cs +++ b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Core/requests/LogsRequestHandler.cs @@ -14,6 +14,9 @@ namespace Microsoft.Azure.Devices.Edge.Agent.Core.Requests public class LogsRequestHandler : RequestHandlerBase> { const int MaxTailValue = 500; + + static readonly Version ExpectedSchemaVersion = new Version("1.0"); + readonly ILogsProvider logsProvider; readonly IRuntimeInfoProvider runtimeInfoProvider; @@ -28,6 +31,13 @@ public LogsRequestHandler(ILogsProvider logsProvider, IRuntimeInfoProvider runti protected override async Task>> HandleRequestInternal(Option payloadOption, CancellationToken cancellationToken) { LogsRequest payload = payloadOption.Expect(() => new ArgumentException("Request payload not found")); + if (ExpectedSchemaVersion.CompareMajorVersion(payload.SchemaVersion, "logs upload request schema") != 0) + { + Events.MismatchedMinorVersions(payload.SchemaVersion, ExpectedSchemaVersion); + } + + Events.ProcessingRequest(payload); + ILogsRequestToOptionsMapper requestToOptionsMapper = new LogsRequestToOptionsMapper( this.runtimeInfoProvider, payload.Encoding, @@ -70,7 +80,9 @@ static class Events enum EventIds { ReceivedModuleLogs = IdStart + 1, - ReceivedLogOptions + ReceivedLogOptions, + ProcessingRequest, + MismatchedMinorVersions } public static void ReceivedModuleLogs(byte[] moduleLogs, string id) @@ -94,6 +106,16 @@ public static void ReceivedLogOptions((string id, ModuleLogOptions logOptions) r Log.LogInformation((int)EventIds.ReceivedLogOptions, $"Received log options for {receivedLogOptions.id} with no tail value specified, setting tail value to {MaxTailValue}"); } } + + public static void ProcessingRequest(LogsRequest payload) + { + Log.LogInformation((int)EventIds.ProcessingRequest, $"Processing request to get logs for {payload.ToJson()}"); + } + + public static void MismatchedMinorVersions(string payloadSchemaVersion, Version expectedSchemaVersion) + { + Log.LogWarning((int)EventIds.MismatchedMinorVersions, $"Logs upload request schema version {payloadSchemaVersion} does not match expected schema version {expectedSchemaVersion}. Some settings may not be supported."); + } } } } diff --git a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Core/requests/LogsUploadRequest.cs b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Core/requests/LogsUploadRequest.cs index 995e6ed19c4..6da209e7fa9 100644 --- a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Core/requests/LogsUploadRequest.cs +++ b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Core/requests/LogsUploadRequest.cs @@ -50,7 +50,7 @@ public LogsUploadRequest( [DefaultValue(LogsContentType.Text)] public LogsContentType ContentType { get; } - [JsonProperty("sasUrl")] + [JsonIgnore] public string SasUrl { get; } } } diff --git a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Core/requests/LogsUploadRequestHandler.cs b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Core/requests/LogsUploadRequestHandler.cs index 38d94c2b443..d3d80fb4032 100644 --- a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Core/requests/LogsUploadRequestHandler.cs +++ b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Core/requests/LogsUploadRequestHandler.cs @@ -8,6 +8,7 @@ namespace Microsoft.Azure.Devices.Edge.Agent.Core.Requests using System.Threading; using System.Threading.Tasks; using Microsoft.Azure.Devices.Edge.Agent.Core.Logs; + using Microsoft.Azure.Devices.Edge.Storage; using Microsoft.Azure.Devices.Edge.Util; using Microsoft.Extensions.Logging; @@ -36,6 +37,7 @@ protected override async Task> HandleRequestInternal(Option bytes) // Assert Assert.NotEmpty(receivedBytes); - Assert.Equal(dockerLogsStreamBytes, receivedBytes); + Assert.Equal(string.Join(string.Empty, TestLogTexts).ToBytes(), receivedBytes); } [Fact] @@ -404,12 +404,5 @@ Task Callback(ArraySegment bytes) Assert.Equal(expectedTextLines, receivedText); } - - [Theory] - [MemberData(nameof(GetNeedToProcessStreamTestData))] - public void NeedToProcessStreamTest(ModuleLogOptions logOptions, bool expectedResult) - { - Assert.Equal(expectedResult, LogsProvider.NeedToProcessStream(logOptions)); - } } }