From 559ebd2f6c87f4a0723273a4daa4c7ffd4f0bd02 Mon Sep 17 00:00:00 2001 From: Lior Banai Date: Sat, 27 Jan 2024 09:54:46 +0200 Subject: [PATCH] Fix file alread in use --- ...iewer.ElasticCommonSchema.UnitTests.csproj | 4 + .../JsonFormatTests.cs | 89 ------------------ .../log files/CompactJsonFormat.clef | 4 - .../log files/CompactJsonFormat.gz | Bin 275 -> 0 bytes .../CompactJsonFormatSourceContextTest.clef | 2 - .../CompactJsonFormatTestColumns.clef | 4 - .../log files/JsonFileCompactFormat.clef | 81 ---------------- .../log files/JsonFormatPerFile.clef | 44 --------- .../log files/JsonFormatPerLine.clef | 2 - ...alogy.LogViewer.ElasticCommonSchema.csproj | 2 +- .../IAnalogy/OfflineDataProvider.cs | 89 ------------------ .../Parsers/EcsJsonFileParser.cs | 35 ++++--- 12 files changed, 27 insertions(+), 329 deletions(-) delete mode 100644 Analogy.LogViewer.ElasticCommonSchema.UnitTests/JsonFormatTests.cs delete mode 100644 Analogy.LogViewer.ElasticCommonSchema.UnitTests/log files/CompactJsonFormat.clef delete mode 100644 Analogy.LogViewer.ElasticCommonSchema.UnitTests/log files/CompactJsonFormat.gz delete mode 100644 Analogy.LogViewer.ElasticCommonSchema.UnitTests/log files/CompactJsonFormatSourceContextTest.clef delete mode 100644 Analogy.LogViewer.ElasticCommonSchema.UnitTests/log files/CompactJsonFormatTestColumns.clef delete mode 100644 Analogy.LogViewer.ElasticCommonSchema.UnitTests/log files/JsonFileCompactFormat.clef delete mode 100644 Analogy.LogViewer.ElasticCommonSchema.UnitTests/log files/JsonFormatPerFile.clef delete mode 100644 Analogy.LogViewer.ElasticCommonSchema.UnitTests/log files/JsonFormatPerLine.clef diff --git a/Analogy.LogViewer.ElasticCommonSchema.UnitTests/Analogy.LogViewer.ElasticCommonSchema.UnitTests.csproj b/Analogy.LogViewer.ElasticCommonSchema.UnitTests/Analogy.LogViewer.ElasticCommonSchema.UnitTests.csproj index 96ba221..7522302 100644 --- a/Analogy.LogViewer.ElasticCommonSchema.UnitTests/Analogy.LogViewer.ElasticCommonSchema.UnitTests.csproj +++ b/Analogy.LogViewer.ElasticCommonSchema.UnitTests/Analogy.LogViewer.ElasticCommonSchema.UnitTests.csproj @@ -49,4 +49,8 @@ + + + + diff --git a/Analogy.LogViewer.ElasticCommonSchema.UnitTests/JsonFormatTests.cs b/Analogy.LogViewer.ElasticCommonSchema.UnitTests/JsonFormatTests.cs deleted file mode 100644 index b45b4f8..0000000 --- a/Analogy.LogViewer.ElasticCommonSchema.UnitTests/JsonFormatTests.cs +++ /dev/null @@ -1,89 +0,0 @@ -using Microsoft.VisualStudio.TestTools.UnitTesting; -using System; -using System.IO; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; - -namespace Analogy.LogViewer.Elastic.CommonSchema.Serilog.UnitTests -{ - [TestClass] - public class JsonFormatTests - { - private string Folder { get; } = Environment.CurrentDirectory; - - [TestMethod] - public async Task ECSFullFileTest() - { - //EcsJsonFileParser parser = new EcsJsonFileParser(new CompactJsonFormatMessageFields()); - //CancellationTokenSource cts = new CancellationTokenSource(); - //string fileName = Path.Combine(Folder, "log files", "JsonFileCompactFormat.clef"); - //MessageHandlerForTesting forTesting = new MessageHandlerForTesting(); - //var messages = (await parser.Process(fileName, cts.Token, forTesting)).ToList(); - //Assert.IsTrue(messages.Count == 2); - //Assert.IsTrue(messages[0].MachineName == "MY-MACHINE"); - //Assert.IsTrue(messages[1].Text.StartsWith("An unknown error occurred")); - //Assert.IsTrue((messages[1].Module == "My process")); - } - - //[TestMethod] - //public async Task JsonFilePerLineTest() - //{ - // var p = new JsonFormatterParser(new JsonFormatMessageFields()); - // CancellationTokenSource cts = new CancellationTokenSource(); - // string fileName = Path.Combine(Folder, "log files", "JsonFormatPerLine.clef"); - // MessageHandlerForTesting forTesting = new MessageHandlerForTesting(); - // var messages = (await p.Process(fileName, cts.Token, forTesting)).ToList(); - // Assert.IsTrue(messages.Count == 2); - // //Assert.IsTrue(messages[0].Text == "Hello, { Name: \"nblumhardt\", Tags: [1, 2, 3] }, 0000007b at 06/07/2016 06:44:57","got"+ messages[0].Text); - // Assert.IsTrue(messages[0].User == "{ Name: \"nblumhardt\", Tags: [1, 2, 3] }"); - //} - - //[TestMethod] - //public async Task JsonFilePerFileTest() - //{ - // EcsJsonFileParser parser = new EcsJsonFileParser(new JsonFormatMessageFields()); - // CancellationTokenSource cts = new CancellationTokenSource(); - // string fileName = Path.Combine(Folder, "log files", "JsonFormatPerFile.clef"); - // MessageHandlerForTesting forTesting = new MessageHandlerForTesting(); - // var messages = (await parser.Process(fileName, cts.Token, forTesting)).ToList(); - // Assert.IsTrue(messages.Count == 2); - // //Assert.IsTrue(messages[0].Text == "Hello, { Name: \"nblumhardt\", Tags: [1, 2, 3] }, 0000007b at 06/07/2016 06:44:57","got"+ messages[0].Text); - // Assert.IsTrue(messages[0].User == "{ Name: \"nblumhardt\", Tags: [1, 2, 3] }"); - //} - - //[TestMethod] - //[DataRow("JsonFileCompactFormat.clef", FileFormat.CompactJsonFormatPerFile)] - //[DataRow("JsonFormatPerLine.clef", FileFormat.JsonFormatPerLine)] - //public void CompactJsonFormatTestAutomaticDetection(string fileName, FileFormat format) - //{ - // string file = Path.Combine(Folder, "log files", fileName); - // var type = OfflineDataProvider.TryDetectFormat(file); - // Assert.IsTrue(type == format); - //} - - //[TestMethod] - //public async Task JsonFilePerLineDateTimeWithOffsetTest() - //{ - // var p = new JsonFormatterParser(new JsonFormatMessageFields()); - // CancellationTokenSource cts = new CancellationTokenSource(); - // string fileName = Path.Combine(Folder, "log files", "JsonFormatPerLine.clef"); - // MessageHandlerForTesting forTesting = new MessageHandlerForTesting(); - // var messages = (await p.Process(fileName, cts.Token, forTesting)).ToList(); - // Assert.IsTrue(messages.Count == 2); - //} - - //[TestMethod] - //public async Task ECSParserTest() - //{ - // JsonFormatterParser parser = new JsonFormatterParser(new ECSFormatMessageFields()); - // CancellationTokenSource cts = new CancellationTokenSource(); - // string fileName = @"C:\MVD2\Logs\ECS\ILIUM.log"; - // MessageHandlerForTesting forTesting = new MessageHandlerForTesting(); - // var messages = (await parser.Process(fileName, cts.Token, forTesting)).ToList(); - // Assert.IsTrue(messages.Count == 2); - // //Assert.IsTrue(messages[0].Text == "Hello, { Name: \"nblumhardt\", Tags: [1, 2, 3] }, 0000007b at 06/07/2016 06:44:57","got"+ messages[0].Text); - // Assert.IsTrue(messages[0].User == "{ Name: \"nblumhardt\", Tags: [1, 2, 3] }"); - //} - } -} \ No newline at end of file diff --git a/Analogy.LogViewer.ElasticCommonSchema.UnitTests/log files/CompactJsonFormat.clef b/Analogy.LogViewer.ElasticCommonSchema.UnitTests/log files/CompactJsonFormat.clef deleted file mode 100644 index 37fe231..0000000 --- a/Analogy.LogViewer.ElasticCommonSchema.UnitTests/log files/CompactJsonFormat.clef +++ /dev/null @@ -1,4 +0,0 @@ -{"@t":"2016-10-12T04:46:58.0554314Z","@mt":"Hello, {@User}","User":{"Name":"nblumhardt","Id":101}} -{"@t":"2016-10-12T04:46:58.0684369Z","@mt":"Number {N:x8}","@r":["0000002a"],"N":42} -{"@t":"2016-10-12T04:46:58.0724384Z","@mt":"Tags are {Tags}","@l":"Warning","Tags":["test","orange"]} -{"@t":"2016-10-12T04:46:58.0904378Z","@mt":"Something failed","@l":"Error", "@x":"System.DivideByZer..."} \ No newline at end of file diff --git a/Analogy.LogViewer.ElasticCommonSchema.UnitTests/log files/CompactJsonFormat.gz b/Analogy.LogViewer.ElasticCommonSchema.UnitTests/log files/CompactJsonFormat.gz deleted file mode 100644 index 7f85be101218bd3d725adb4edba356d929a8addc..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 275 zcmV+u0qp)CiwFotoNHbJ07GnLW<_{mZE$R5F)m|lWo7_{jzMd~Fbsw7gZ&4ia~kZ# zNmDaM7#nPtT-Gt@=)omg8i<`?yQ9VUzt1j=?zHM6=}Elrd1xI7C@8BJj4oKQrd;q^ zR83B+ikFOUAcIz=r)Qj|IU~M(vuHw;QwQOplM3Tq@1}#=OVhh}d+9+i%0jrk`HxXI zysTFjMt4vhttps://github.com/Analogy-LogViewer/Analogy.LogViewer.ElasticCommonSchema AnalogyECS128x128.png - 6.0.0 + 6.0.1 Analogy.LogViewer README.md diff --git a/Analogy.LogViewer.ElasticCommonSchema/IAnalogy/OfflineDataProvider.cs b/Analogy.LogViewer.ElasticCommonSchema/IAnalogy/OfflineDataProvider.cs index fad75d1..5638c8e 100644 --- a/Analogy.LogViewer.ElasticCommonSchema/IAnalogy/OfflineDataProvider.cs +++ b/Analogy.LogViewer.ElasticCommonSchema/IAnalogy/OfflineDataProvider.cs @@ -94,94 +94,5 @@ protected override List GetSupportedFilesInternal(DirectoryInfo dirInf return files; } - - //public static FileFormat TryDetectFormat(string fileName) - //{ - // string jsonData = string.Empty; - // if (fileName.EndsWith(".gz", StringComparison.InvariantCultureIgnoreCase)) - // { - // using (var fileStream = new FileStream(fileName, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) - // { - // using (var gzStream = new GZipStream(fileStream, CompressionMode.Decompress)) - // { - // using (var streamReader = new StreamReader(gzStream, encoding: Encoding.UTF8)) - // { - // jsonData = streamReader.ReadToEnd(); - // } - // } - // } - // } - - // if (string.IsNullOrEmpty(jsonData)) - // { - // jsonData = SafeReadAllLines(fileName); - // } - - // var format = TryParseAsFile(jsonData); - // if (format == FileFormat.Unknown) - // { - // format = TryParsePerLine(jsonData); - // } - - // return format; - //} - //private static FileFormat TryParsePerLine(string jsonData) - //{ - // try - // { - // IMessageFields fields = new JsonFormatMessageFields(); - // if (jsonData.Contains(fields.Timestamp) && jsonData.Contains(fields.MessageTemplate)) - // { - // return FileFormat.JsonFormatPerLine; - // } - - // fields = new CompactJsonFormatMessageFields(); - // if (jsonData.Contains(fields.Timestamp) && jsonData.Contains(fields.MessageTemplate)) - // { - // return FileFormat.CompactJsonFormatPerLine; - // } - - // return FileFormat.Unknown; - // } - // catch (Exception) - // { - // return FileFormat.Unknown; - // } - //} - - //private static FileFormat TryParseAsFile(string jsonData) - //{ - // try - // { - // var jsonObject = JsonConvert.DeserializeObject(jsonData); - // IMessageFields fields = new JsonFormatMessageFields(); - // if (jsonData.Contains(fields.Timestamp) && jsonData.Contains(fields.MessageTemplate)) - // { - // return FileFormat.JsonFormatFile; - // } - - // fields = new CompactJsonFormatMessageFields(); - // if (jsonData.Contains(fields.Timestamp) && jsonData.Contains(fields.MessageTemplate)) - // { - // return FileFormat.CompactJsonFormatPerFile; - // } - - // return FileFormat.Unknown; - // } - // catch (Exception) - // { - // return FileFormat.Unknown; - // } - //} - - private static string SafeReadAllLines(string path) - { - using (var stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) - using (var sr = new StreamReader(stream)) - { - string data = sr.ReadToEnd(); - return data; - } - } } } \ No newline at end of file diff --git a/Analogy.LogViewer.ElasticCommonSchema/Parsers/EcsJsonFileParser.cs b/Analogy.LogViewer.ElasticCommonSchema/Parsers/EcsJsonFileParser.cs index 49425d6..bb2a5ad 100644 --- a/Analogy.LogViewer.ElasticCommonSchema/Parsers/EcsJsonFileParser.cs +++ b/Analogy.LogViewer.ElasticCommonSchema/Parsers/EcsJsonFileParser.cs @@ -5,6 +5,7 @@ using Newtonsoft.Json; using System; using System.Collections.Generic; +using System.IO; using System.Threading; using System.Threading.Tasks; @@ -34,27 +35,34 @@ public async Task> Process(string fileName, Canc AnalogyLogMessage empty = new AnalogyLogMessage($"File is null or empty. Aborting.", AnalogyLogLevel.Critical, AnalogyLogClass.General, "Analogy", "None") { - Source = "Analogy", Module = System.Diagnostics.Process.GetCurrentProcess().ProcessName, + Source = "Analogy", + Module = System.Diagnostics.Process.GetCurrentProcess().ProcessName, }; messagesHandler.AppendMessage(empty, fileName); return new List { empty }; } - List messages = new List(); + List messages = new(); try { -#if NET - var jsonLines = await System.IO.File.ReadAllLinesAsync(fileName, token); -#else - var jsonLines = System.IO.File.ReadAllLines(fileName); -#endif - foreach (var line in jsonLines) + using (var stream = System.IO.File.Open(fileName, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) { - var message = EcsDocumentUtils.ParseLine(line, UserSettingsManager.UserSettings.Settings.ShowAllColumnsFromMetaDataField, UserSettingsManager.UserSettings.Settings.AdditionalColumnsFromMetaDataField); - messages.Add(message); - messagesHandler.AppendMessage(message, fileName); + long count = 0; + using (var reader = new StreamReader(stream)) + { + while (!reader.EndOfStream && !token.IsCancellationRequested) + { + var line = await reader.ReadLineAsync(); + var message = EcsDocumentUtils.ParseLine(line, + UserSettingsManager.UserSettings.Settings.ShowAllColumnsFromMetaDataField, + UserSettingsManager.UserSettings.Settings.AdditionalColumnsFromMetaDataField); + messages.Add(message); + messagesHandler.AppendMessage(message, fileName); + } + + return messages; + } } - return messages; } catch (Exception e) { @@ -62,7 +70,8 @@ public async Task> Process(string fileName, Canc $"Error occurred processing file {fileName}. Reason: {e.Message}", AnalogyLogLevel.Critical, AnalogyLogClass.General, "Analogy", "None") { - Source = "Analogy", Module = System.Diagnostics.Process.GetCurrentProcess().ProcessName, + Source = "Analogy", + Module = System.Diagnostics.Process.GetCurrentProcess().ProcessName, }; messagesHandler.AppendMessage(empty, fileName); return new List { empty };