Add array type support for EventPipe to TraceEvent

src/TraceEvent/EventPipe/EventPipeFormat.md

@@ -249,7 +249,7 @@ Followed by PayloadSize bytes of payload and no alignment.
 
 ### Metadata event encoding
 
-For event blobs within a MetadataBlock, the payload describes a type of event. This payload format hasn't changed from earlier versions of the file format so copying that description here:
+For event blobs within a MetadataBlock, the payload describes a type of event. The payload contains a V1 and V2 field description, V1 of the nettrace format did not support array types (arbritrary number of arguments of the same type) and the V2 field description was added to enable this.
 
 
 The PayloadBytes of such a MetaData definition are:
@@ -263,16 +263,32 @@ The PayloadBytes of such a MetaData definition are:
     int Version          // The version number for this event.
     int Level;           // The verbosity (5 is verbose, 1 is only critical) for the event.
 
-Following this header there is a Payload description.   This consists of 
+Following this header there is a V1 Payload description.   This consists of 
 
 *   int FieldCount;      // The number of fields in the payload
 
-Followed by FieldCount number of field Definitions 
+Followed by FieldCount number of field definitions 
 
     int TypeCode;	 // This is the System.Typecode enumeration
     <PAYLOAD_DESCRIPTION>
     string FieldName;    // The 2 byte Unicode, null terminated string representing the Name of the Field
 
+Following the FieldCount number of fields there are an optional set of metadata tags, where a tag consists of 
+    int TagPayloadBytes; // The number of bytes the tag payload uses, uninclusive of this field
+    byte TagKind;        // The type of tag it is. Currently used values are OpCode=1 V2Params=2
+
+Followed by a tag payload. If TagKind==OpCode the payload consists of
+    byte OpCode;         // The event's opcode
+
+If TagKind=V2Params the payload consists of
+    int V2FieldCount;    // The total number of fields in this payload
+
+Followed by V2FieldCount number of field definitions
+    int V2TypeCode;      // In V2 this can be EventPipeTypeCodeArray==19
+    int ArrayTypeCode;   // This optional field only appears when V2TypeCode==EventPipeTypeCodeArray
+    <PAYLOAD_DESCRIPTION>
+    string FieldName
+
 
 For primitive types and strings <PAYLOAD_DESCRIPTION> is not present, however if TypeCode == Object (1) then <PAYLOAD_DESCRIPTION> another payload
 description (that is a field count, followed by a list of field definitions).   These can be nested to arbitrary depth.  

src/TraceEvent/TraceEvent.Tests/EventPipeParsing.cs

@@ -26,7 +26,7 @@ public EventPipeParsing(ITestOutputHelper output)
             : base(output)
         {
         }
-
+        
         [Theory()]
         [MemberData(nameof(TestEventPipeFiles))]
         public void Basic(string eventPipeFileName)
@@ -276,6 +276,102 @@ public void CanReadV4EventPipeTraceBiggerThan4GB()
             }
         }
 
+        [Fact]
+        public void CanReadV4EventPipeArrayTypes()
+        {
+            PrepareTestData();
+
+            const string eventPipeFileName = "eventpipe-dotnetcore5.0-win-x64-arraytypes.nettrace";
+
+            string eventPipeFilePath = Path.Combine(UnZippedDataDir, eventPipeFileName);
+
+            using (var traceSource = new EventPipeEventSource(eventPipeFilePath))
+            {
+                int successCount = 0;
+                Action<TraceEvent> handler = delegate (TraceEvent traceEvent)
+                {
+                    string traceLevelValidationMessage = "Expected level {0} but got level {1} for EventSource={2} Event={3}";
+                    string traceKeywordValidationMessage = "Expected keywords {0} but got keywords{1} for EventSource={2} Event={3}";
+                    string traceOpcodeValidationMessage = "Expected opcode {0} but got opcode {1} for EventSource={2} Event={3}";
+                    string tracePayloadValidationMessage = "Expected {0} payload items but got {1} items for EventSource={2} Event={3}";
+                    string tracePayloadNamesValidationMessage = "Expected argument name {0} but got name {1} for EventSource={2} Event={3}";
+                    string tracePayloadTypeValidationMessage = "Expected type {0} but got type {1} for EventSource={2} Event={3} Argument={4}";
+                    string tracePayloadValueValidationMessage = "Expected argument value {0} but got value {1} for EventSource={2} Event={3} Argument={4}";
+
+                    if (traceEvent.ProviderName == "TestEventSource0")
+                    {
+                        if (traceEvent.EventName == "TestEvent1")
+                        {
+                            if ((int)traceEvent.Level != 2) { throw new Exception(String.Format(traceLevelValidationMessage, 2, (int)traceEvent.Level, "TestEventSource0", "TestEvent1")); }
+                            if ((int)traceEvent.Keywords != 0) { throw new Exception(String.Format(traceKeywordValidationMessage, 0, (int)traceEvent.Keywords, "TestEventSource0", "TestEvent1")); }
+                            if ((int)traceEvent.Opcode != 0) { throw new Exception(String.Format(traceOpcodeValidationMessage, 0, (int)traceEvent.Opcode, "TestEventSource0", "TestEvent1")); }
+                            if (traceEvent.PayloadNames.Count() != 3) { throw new Exception(String.Format(tracePayloadValidationMessage, 3, traceEvent.PayloadNames.Count(), "TestEventSource0", "TestEvent1")); }
+                            uint[] testEvent1Array0 = new uint[] { (uint)2032854139, (uint)1608221689, (uint)1470019200, (uint)199494339, (uint)1238846140, (uint)1366609043 };
+                            if (traceEvent.PayloadNames[0] != "arg0") { throw new Exception(String.Format(tracePayloadNamesValidationMessage, "arg0", traceEvent.PayloadNames[0], "TestEventSource0", "TestEvent1")); }
+                            if (traceEvent.PayloadValue(0).GetType() != typeof(uint[])) { throw new Exception(String.Format(tracePayloadTypeValidationMessage, "uint[]", traceEvent.PayloadValue(0).GetType(), "TestEventSource0", "TestEvent1", "arg0")); }
+                            if (!Enumerable.SequenceEqual((uint[])traceEvent.PayloadValue(0), testEvent1Array0)) { throw new Exception(String.Format(tracePayloadValueValidationMessage, testEvent1Array0, traceEvent.PayloadValue(0), "TestEventSource0", "TestEvent1", "arg0")); }
+                            short[] testEvent1Array1 = new short[] { (short)26494, (short)17115, (short)-7229, (short)18850, (short)27931 };
+                            if (traceEvent.PayloadNames[1] != "arg1") { throw new Exception(String.Format(tracePayloadNamesValidationMessage, "arg1", traceEvent.PayloadNames[1], "TestEventSource0", "TestEvent1")); }
+                            if (traceEvent.PayloadValue(1).GetType() != typeof(short[])) { throw new Exception(String.Format(tracePayloadTypeValidationMessage, "short[]", traceEvent.PayloadValue(1).GetType(), "TestEventSource0", "TestEvent1", "arg1")); }
+                            if (!Enumerable.SequenceEqual((short[])traceEvent.PayloadValue(1), testEvent1Array1)) { throw new Exception(String.Format(tracePayloadValueValidationMessage, testEvent1Array1, traceEvent.PayloadValue(1), "TestEventSource0", "TestEvent1", "arg1")); }
+                            if (traceEvent.PayloadNames[2] != "arg2") { throw new Exception(String.Format(tracePayloadNamesValidationMessage, "arg2", traceEvent.PayloadNames[2], "TestEventSource0", "TestEvent1")); }
+                            if (traceEvent.PayloadValue(2).GetType() != typeof(long)) { throw new Exception(String.Format(tracePayloadTypeValidationMessage, "long", traceEvent.PayloadValue(2).GetType(), "TestEventSource0", "TestEvent1", "arg2")); }
+                            if ((long)traceEvent.PayloadValue(2) != 2033417279) { throw new Exception(String.Format(tracePayloadValueValidationMessage, 2033417279, traceEvent.PayloadValue(2), "TestEventSource0", "TestEvent1", "arg2")); }
+
+                            ++successCount;
+                            return;
+                        }
+                        if (traceEvent.EventName == "TestEvent2")
+                        {
+                            if ((int)traceEvent.Level != 4) { throw new Exception(String.Format(traceLevelValidationMessage, 4, (int)traceEvent.Level, "TestEventSource0", "TestEvent2")); }
+                            if ((int)traceEvent.Keywords != 1231036291) { throw new Exception(String.Format(traceKeywordValidationMessage, 1231036291, (int)traceEvent.Keywords, "TestEventSource0", "TestEvent2")); }
+                            if ((int)traceEvent.Opcode != 0) { throw new Exception(String.Format(traceOpcodeValidationMessage, 0, (int)traceEvent.Opcode, "TestEventSource0", "TestEvent2")); }
+                            if (traceEvent.PayloadNames.Count() != 4) { throw new Exception(String.Format(tracePayloadValidationMessage, 4, traceEvent.PayloadNames.Count(), "TestEventSource0", "TestEvent2")); }
+                            if (traceEvent.PayloadNames[0] != "arg0") { throw new Exception(String.Format(tracePayloadNamesValidationMessage, "arg0", traceEvent.PayloadNames[0], "TestEventSource0", "TestEvent2")); }
+                            if (traceEvent.PayloadValue(0).GetType() != typeof(long)) { throw new Exception(String.Format(tracePayloadTypeValidationMessage, "long", traceEvent.PayloadValue(0).GetType(), "TestEventSource0", "TestEvent2", "arg0")); }
+                            if ((long)traceEvent.PayloadValue(0) != -1001479330) { throw new Exception(String.Format(tracePayloadValueValidationMessage, -1001479330, traceEvent.PayloadValue(0), "TestEventSource0", "TestEvent2", "arg0")); }
+                            if (traceEvent.PayloadNames[1] != "arg1") { throw new Exception(String.Format(tracePayloadNamesValidationMessage, "arg1", traceEvent.PayloadNames[1], "TestEventSource0", "TestEvent2")); }
+                            if (traceEvent.PayloadValue(1).GetType() != typeof(int)) { throw new Exception(String.Format(tracePayloadTypeValidationMessage, "int", traceEvent.PayloadValue(1).GetType(), "TestEventSource0", "TestEvent2", "arg1")); }
+                            if ((int)traceEvent.PayloadValue(1) != -1397908228) { throw new Exception(String.Format(tracePayloadValueValidationMessage, -1397908228, traceEvent.PayloadValue(1), "TestEventSource0", "TestEvent2", "arg1")); }
+                            int[] testEvent2Array0 = new int[] { 1470938110, 172564262, 1133558854, -1572049829 };
+                            if (traceEvent.PayloadNames[2] != "arg2") { throw new Exception(String.Format(tracePayloadNamesValidationMessage, "arg2", traceEvent.PayloadNames[2], "TestEventSource0", "TestEvent2")); }
+                            if (traceEvent.PayloadValue(2).GetType() != typeof(int[])) { throw new Exception(String.Format(tracePayloadTypeValidationMessage, "int[]", traceEvent.PayloadValue(2).GetType(), "TestEventSource0", "TestEvent2", "arg2")); }
+                            if (!Enumerable.SequenceEqual((int[])traceEvent.PayloadValue(2), testEvent2Array0)) { throw new Exception(String.Format(tracePayloadValueValidationMessage, testEvent2Array0, traceEvent.PayloadValue(2), "TestEventSource0", "TestEvent2", "arg2")); }
+                            ulong[] testEvent2Array1 = new ulong[] { (ulong)2055126903, (ulong)593325874, (ulong)2130052527, (ulong)162795177 };
+                            if (traceEvent.PayloadNames[3] != "arg3") { throw new Exception(String.Format(tracePayloadNamesValidationMessage, "arg3", traceEvent.PayloadNames[3], "TestEventSource0", "TestEvent2")); }
+                            if (traceEvent.PayloadValue(3).GetType() != typeof(ulong[])) { throw new Exception(String.Format(tracePayloadTypeValidationMessage, "ulong[]", traceEvent.PayloadValue(3).GetType(), "TestEventSource0", "TestEvent2", "arg3")); }
+                            if (!Enumerable.SequenceEqual((ulong[])traceEvent.PayloadValue(3), testEvent2Array1)) { throw new Exception(String.Format(tracePayloadValueValidationMessage, testEvent2Array1, traceEvent.PayloadValue(3), "TestEventSource0", "TestEvent2", "arg3")); }
+
+                            ++successCount;
+                            return;
+                        }
+                        if (traceEvent.EventName == "TestEvent3/24")
+                        {
+                            if ((int)traceEvent.Level != 2) { throw new Exception(String.Format(traceLevelValidationMessage, 2, (int)traceEvent.Level, "TestEventSource0", "TestEvent3")); }
+                            if ((int)traceEvent.Keywords != 0) { throw new Exception(String.Format(traceKeywordValidationMessage, 0, (int)traceEvent.Keywords, "TestEventSource0", "TestEvent3")); }
+                            if ((int)traceEvent.Opcode != 24) { throw new Exception(String.Format(traceOpcodeValidationMessage, 24, (int)traceEvent.Opcode, "TestEventSource0", "TestEvent3")); }
+                            if (traceEvent.PayloadNames.Count() != 1) { throw new Exception(String.Format(tracePayloadValidationMessage, 1, traceEvent.PayloadNames.Count(), "TestEventSource0", "TestEvent3")); }
+                            if (traceEvent.PayloadNames[0] != "arg0") { throw new Exception(String.Format(tracePayloadNamesValidationMessage, "arg0", traceEvent.PayloadNames[0], "TestEventSource0", "TestEvent3")); }
+                            if (traceEvent.PayloadValue(0).GetType() != typeof(char[])) { throw new Exception(String.Format(tracePayloadTypeValidationMessage, "char[]", traceEvent.PayloadValue(0).GetType(), "TestEventSource0", "TestEvent3", "arg0")); }
+                            char[] testEvent3Array = new char[] { (char)59299, (char)13231, (char)38541, (char)7407, (char)35812 };
+                            if (!Enumerable.SequenceEqual((char[])traceEvent.PayloadValue(0), testEvent3Array)) { throw new Exception(String.Format(tracePayloadValueValidationMessage, testEvent3Array, traceEvent.PayloadValue(0), "TestEventSource0", "TestEvent3", "arg0")); }
+
+                            ++successCount;
+                            return;
+                        }
+                    }
+                };
+
+                var privateClr = new ClrPrivateTraceEventParser(traceSource);
+                traceSource.Dynamic.All += handler;
+
+                // Process
+                traceSource.Process();
+
+                Assert.Equal(3, successCount);
+            }
+        }
+
         [Fact]
         public void AllEventsLeavesNoUnhandledEvents()
         {

src/TraceEvent/TraceEvent.Tests/EventPipeTestBase.cs

@@ -32,54 +32,57 @@ public static IEnumerable<object[]> StreamableTestEventPipeFiles
                                     .Select(file => new[] { Path.GetFileNameWithoutExtension(file) });
 
         private static bool s_fileUnzipped;
+        private static object s_fileLock = new object();
 
-        [MethodImpl(MethodImplOptions.Synchronized)]
         protected void PrepareTestData()
         {
-            Assert.True(Directory.Exists(TestDataDir));
-            TestDataDir = Path.GetFullPath(TestDataDir);
-            Assert.True(Directory.Exists(OriginalBaselineDir));
-            OriginalBaselineDir = Path.GetFullPath(OriginalBaselineDir);
-
-            // This is atomic because this method is synchronized.  
-            if (!s_fileUnzipped)
+            lock (s_fileLock)
             {
-                Directory.CreateDirectory(UnZippedDataDir);
+                Assert.True(Directory.Exists(TestDataDir));
+                TestDataDir = Path.GetFullPath(TestDataDir);
+                Assert.True(Directory.Exists(OriginalBaselineDir));
+                OriginalBaselineDir = Path.GetFullPath(OriginalBaselineDir);
 
-                foreach (var zipFile in TestEventPipeZipFiles)
+                // This is atomic because this method is synchronized.  
+                if (!s_fileUnzipped)
                 {
-                    string eventPipeFilePath = Path.Combine(UnZippedDataDir, Path.GetFileNameWithoutExtension(zipFile));
+                    Directory.CreateDirectory(UnZippedDataDir);
 
-                    if (!File.Exists(eventPipeFilePath) || File.GetLastWriteTimeUtc(eventPipeFilePath) < File.GetLastWriteTimeUtc(zipFile))
+                    foreach (var zipFile in TestEventPipeZipFiles)
                     {
-                        File.Delete(eventPipeFilePath);
-                        ZipFile.ExtractToDirectory(zipFile, UnZippedDataDir);
+                        string eventPipeFilePath = Path.Combine(UnZippedDataDir, Path.GetFileNameWithoutExtension(zipFile));
+
+                        if (!File.Exists(eventPipeFilePath) || File.GetLastWriteTimeUtc(eventPipeFilePath) < File.GetLastWriteTimeUtc(zipFile))
+                        {
+                            File.Delete(eventPipeFilePath);
+                            ZipFile.ExtractToDirectory(zipFile, UnZippedDataDir);
+                        }
+
+                        Assert.True(File.Exists(eventPipeFilePath));
                     }
 
-                    Assert.True(File.Exists(eventPipeFilePath));
+                    s_fileUnzipped = true;
                 }
 
-                s_fileUnzipped = true;
-            }
-
-            if (Directory.Exists(OutputDir))
-            {
-                Directory.Delete(OutputDir, true);
-            }
+                if (Directory.Exists(OutputDir))
+                {
+                    Directory.Delete(OutputDir, true);
+                }
 
-            Directory.CreateDirectory(OutputDir);
-            Output.WriteLine(string.Format("OutputDir: {0}", Path.GetFullPath(OutputDir)));
-            Assert.True(Directory.Exists(OutputDir));
+                Directory.CreateDirectory(OutputDir);
+                Output.WriteLine(string.Format("OutputDir: {0}", Path.GetFullPath(OutputDir)));
+                Assert.True(Directory.Exists(OutputDir));
 
-            Directory.CreateDirectory(NewBaselineDir);
-            NewBaselineDir = Path.GetFullPath(NewBaselineDir);
-            Output.WriteLine(string.Format("NewBaselineDir: {0}", NewBaselineDir));
+                Directory.CreateDirectory(NewBaselineDir);
+                NewBaselineDir = Path.GetFullPath(NewBaselineDir);
+                Output.WriteLine(string.Format("NewBaselineDir: {0}", NewBaselineDir));
 
-            Assert.True(Directory.Exists(UnZippedDataDir));
-            UnZippedDataDir = Path.GetFullPath(UnZippedDataDir);
-            Assert.True(Directory.Exists(BaseOutputDir));
-            BaseOutputDir = Path.GetFullPath(BaseOutputDir);
-            Assert.True(Directory.Exists(NewBaselineDir));
+                Assert.True(Directory.Exists(UnZippedDataDir));
+                UnZippedDataDir = Path.GetFullPath(UnZippedDataDir);
+                Assert.True(Directory.Exists(BaseOutputDir));
+                BaseOutputDir = Path.GetFullPath(BaseOutputDir);
+                Assert.True(Directory.Exists(NewBaselineDir));
+            }
         }
     }
 }