From a8b83c6aceba2b9acc4d0a4e303bc9846aa9f7c5 Mon Sep 17 00:00:00 2001 From: Bo Wang Date: Thu, 1 Feb 2024 14:20:29 +0800 Subject: [PATCH 1/8] Test avro dataset schema. --- .../src/Customized/AvroDataset.cs | 193 ++++++++++++++++++ .../Models/AvroDataset.Serialization.cs | 146 ------------- .../DataFactoryDatasetResourceTests.cs | 39 ++++ 3 files changed, 232 insertions(+), 146 deletions(-) create mode 100644 sdk/datafactory/Azure.ResourceManager.DataFactory/src/Customized/AvroDataset.cs diff --git a/sdk/datafactory/Azure.ResourceManager.DataFactory/src/Customized/AvroDataset.cs b/sdk/datafactory/Azure.ResourceManager.DataFactory/src/Customized/AvroDataset.cs new file mode 100644 index 0000000000000..f2d249e51c5cc --- /dev/null +++ b/sdk/datafactory/Azure.ResourceManager.DataFactory/src/Customized/AvroDataset.cs @@ -0,0 +1,193 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#nullable disable + +using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.Text.Json; +using Azure.Core; +using Azure.Core.Expressions.DataFactory; + +namespace Azure.ResourceManager.DataFactory.Models +{ + /// Avro dataset. + [CodeGenSuppress(nameof(AvroDataset), typeof(string), typeof(string), typeof(DataFactoryElement>), + typeof(DataFactoryElement), typeof(Core.Expressions.DataFactory.DataFactoryLinkedServiceReference), typeof(IDictionary), + typeof(IList), typeof(string), typeof(IDictionary), typeof(DatasetLocation), typeof(DataFactoryElement), typeof(int))] + public partial class AvroDataset : DataFactoryDatasetProperties + { + /// Initializes a new instance of . + /// Type of dataset. + /// Dataset description. + /// Columns that define the structure of the dataset. Type: array (or Expression with resultType array), itemType: DatasetDataElement. + /// Columns that define the physical type schema of the dataset. Type: array (or Expression with resultType array), itemType: DatasetSchemaDataElement. + /// Linked service reference. + /// Parameters for dataset. + /// List of tags that can be used for describing the Dataset. + /// The folder that this Dataset is in. If not specified, Dataset will appear at the root level. + /// Additional Properties. + /// + /// The location of the avro storage. + /// Please note is the base class. According to the scenario, a derived class of the base class might need to be assigned here, or this property needs to be casted to one of the possible derived classes. + /// The available derived classes include , , , , , , , , , , , , and . + /// + /// The data avroCompressionCodec. Type: string (or Expression with resultType string). + /// + internal AvroDataset(string datasetType, string description, DataFactoryElement> structure, DataFactoryElement schema, Core.Expressions.DataFactory.DataFactoryLinkedServiceReference linkedServiceName, IDictionary parameters, IList annotations, DatasetFolder folder, IDictionary additionalProperties, DatasetLocation dataLocation, DataFactoryElement avroCompressionCodec, int? avroCompressionLevel) : base(datasetType, description, structure, null, linkedServiceName, parameters, annotations, folder, additionalProperties) + { + DataLocation = dataLocation; + AvroCompressionCodec = avroCompressionCodec; + AvroCompressionLevel = avroCompressionLevel; + DatasetType = datasetType ?? "Avro"; + Schema = schema; + } + + /// Columns that define the physical type schema of the dataset. Type: array (or Expression with resultType array), itemType: DatasetSchemaDataElement. + public new DataFactoryElement Schema { get; set; } + + internal static AvroDataset DeserializeAvroDataset(JsonElement element) + { + if (element.ValueKind == JsonValueKind.Null) + { + return null; + } + string type = default; + Optional description = default; + Optional>> structure = default; + Optional> schema = default; + Core.Expressions.DataFactory.DataFactoryLinkedServiceReference linkedServiceName = default; + Optional> parameters = default; + Optional> annotations = default; + Optional folder = default; + Optional location = default; + Optional> avroCompressionCodec = default; + Optional avroCompressionLevel = default; + IDictionary additionalProperties = default; + Dictionary additionalPropertiesDictionary = new Dictionary(); + foreach (var property in element.EnumerateObject()) + { + if (property.NameEquals("type"u8)) + { + type = property.Value.GetString(); + continue; + } + if (property.NameEquals("description"u8)) + { + description = property.Value.GetString(); + continue; + } + if (property.NameEquals("structure"u8)) + { + if (property.Value.ValueKind == JsonValueKind.Null) + { + continue; + } + structure = JsonSerializer.Deserialize>>(property.Value.GetRawText()); + continue; + } + if (property.NameEquals("schema"u8)) + { + if (property.Value.ValueKind == JsonValueKind.Null) + { + continue; + } + schema = JsonSerializer.Deserialize>(property.Value.GetRawText()); + continue; + } + if (property.NameEquals("linkedServiceName"u8)) + { + linkedServiceName = JsonSerializer.Deserialize(property.Value.GetRawText()); + continue; + } + if (property.NameEquals("parameters"u8)) + { + if (property.Value.ValueKind == JsonValueKind.Null) + { + continue; + } + Dictionary dictionary = new Dictionary(); + foreach (var property0 in property.Value.EnumerateObject()) + { + dictionary.Add(property0.Name, EntityParameterSpecification.DeserializeEntityParameterSpecification(property0.Value)); + } + parameters = dictionary; + continue; + } + if (property.NameEquals("annotations"u8)) + { + if (property.Value.ValueKind == JsonValueKind.Null) + { + continue; + } + List array = new List(); + foreach (var item in property.Value.EnumerateArray()) + { + if (item.ValueKind == JsonValueKind.Null) + { + array.Add(null); + } + else + { + array.Add(BinaryData.FromString(item.GetRawText())); + } + } + annotations = array; + continue; + } + if (property.NameEquals("folder"u8)) + { + if (property.Value.ValueKind == JsonValueKind.Null) + { + continue; + } + folder = DatasetFolder.DeserializeDatasetFolder(property.Value); + continue; + } + if (property.NameEquals("typeProperties"u8)) + { + if (property.Value.ValueKind == JsonValueKind.Null) + { + property.ThrowNonNullablePropertyIsNull(); + continue; + } + foreach (var property0 in property.Value.EnumerateObject()) + { + if (property0.NameEquals("location"u8)) + { + if (property0.Value.ValueKind == JsonValueKind.Null) + { + continue; + } + location = DatasetLocation.DeserializeDatasetLocation(property0.Value); + continue; + } + if (property0.NameEquals("avroCompressionCodec"u8)) + { + if (property0.Value.ValueKind == JsonValueKind.Null) + { + continue; + } + avroCompressionCodec = JsonSerializer.Deserialize>(property0.Value.GetRawText()); + continue; + } + if (property0.NameEquals("avroCompressionLevel"u8)) + { + if (property0.Value.ValueKind == JsonValueKind.Null) + { + continue; + } + avroCompressionLevel = property0.Value.GetInt32(); + continue; + } + } + continue; + } + additionalPropertiesDictionary.Add(property.Name, BinaryData.FromString(property.Value.GetRawText())); + } + additionalProperties = additionalPropertiesDictionary; + return new AvroDataset(type, description.Value, structure.Value, schema.Value, linkedServiceName, Optional.ToDictionary(parameters), Optional.ToList(annotations), folder.Value, additionalProperties, location.Value, avroCompressionCodec.Value, Optional.ToNullable(avroCompressionLevel)); + } + } +} diff --git a/sdk/datafactory/Azure.ResourceManager.DataFactory/src/Generated/Models/AvroDataset.Serialization.cs b/sdk/datafactory/Azure.ResourceManager.DataFactory/src/Generated/Models/AvroDataset.Serialization.cs index cd880b7d82190..9ef75d946f5eb 100644 --- a/sdk/datafactory/Azure.ResourceManager.DataFactory/src/Generated/Models/AvroDataset.Serialization.cs +++ b/sdk/datafactory/Azure.ResourceManager.DataFactory/src/Generated/Models/AvroDataset.Serialization.cs @@ -5,11 +5,8 @@ #nullable disable -using System; -using System.Collections.Generic; using System.Text.Json; using Azure.Core; -using Azure.Core.Expressions.DataFactory; namespace Azure.ResourceManager.DataFactory.Models { @@ -107,148 +104,5 @@ void IUtf8JsonSerializable.Write(Utf8JsonWriter writer) } writer.WriteEndObject(); } - - internal static AvroDataset DeserializeAvroDataset(JsonElement element) - { - if (element.ValueKind == JsonValueKind.Null) - { - return null; - } - string type = default; - Optional description = default; - Optional>> structure = default; - Optional>> schema = default; - DataFactoryLinkedServiceReference linkedServiceName = default; - Optional> parameters = default; - Optional> annotations = default; - Optional folder = default; - Optional location = default; - Optional> avroCompressionCodec = default; - Optional avroCompressionLevel = default; - IDictionary additionalProperties = default; - Dictionary additionalPropertiesDictionary = new Dictionary(); - foreach (var property in element.EnumerateObject()) - { - if (property.NameEquals("type"u8)) - { - type = property.Value.GetString(); - continue; - } - if (property.NameEquals("description"u8)) - { - description = property.Value.GetString(); - continue; - } - if (property.NameEquals("structure"u8)) - { - if (property.Value.ValueKind == JsonValueKind.Null) - { - continue; - } - structure = JsonSerializer.Deserialize>>(property.Value.GetRawText()); - continue; - } - if (property.NameEquals("schema"u8)) - { - if (property.Value.ValueKind == JsonValueKind.Null) - { - continue; - } - schema = JsonSerializer.Deserialize>>(property.Value.GetRawText()); - continue; - } - if (property.NameEquals("linkedServiceName"u8)) - { - linkedServiceName = JsonSerializer.Deserialize(property.Value.GetRawText()); - continue; - } - if (property.NameEquals("parameters"u8)) - { - if (property.Value.ValueKind == JsonValueKind.Null) - { - continue; - } - Dictionary dictionary = new Dictionary(); - foreach (var property0 in property.Value.EnumerateObject()) - { - dictionary.Add(property0.Name, EntityParameterSpecification.DeserializeEntityParameterSpecification(property0.Value)); - } - parameters = dictionary; - continue; - } - if (property.NameEquals("annotations"u8)) - { - if (property.Value.ValueKind == JsonValueKind.Null) - { - continue; - } - List array = new List(); - foreach (var item in property.Value.EnumerateArray()) - { - if (item.ValueKind == JsonValueKind.Null) - { - array.Add(null); - } - else - { - array.Add(BinaryData.FromString(item.GetRawText())); - } - } - annotations = array; - continue; - } - if (property.NameEquals("folder"u8)) - { - if (property.Value.ValueKind == JsonValueKind.Null) - { - continue; - } - folder = DatasetFolder.DeserializeDatasetFolder(property.Value); - continue; - } - if (property.NameEquals("typeProperties"u8)) - { - if (property.Value.ValueKind == JsonValueKind.Null) - { - property.ThrowNonNullablePropertyIsNull(); - continue; - } - foreach (var property0 in property.Value.EnumerateObject()) - { - if (property0.NameEquals("location"u8)) - { - if (property0.Value.ValueKind == JsonValueKind.Null) - { - continue; - } - location = DatasetLocation.DeserializeDatasetLocation(property0.Value); - continue; - } - if (property0.NameEquals("avroCompressionCodec"u8)) - { - if (property0.Value.ValueKind == JsonValueKind.Null) - { - continue; - } - avroCompressionCodec = JsonSerializer.Deserialize>(property0.Value.GetRawText()); - continue; - } - if (property0.NameEquals("avroCompressionLevel"u8)) - { - if (property0.Value.ValueKind == JsonValueKind.Null) - { - continue; - } - avroCompressionLevel = property0.Value.GetInt32(); - continue; - } - } - continue; - } - additionalPropertiesDictionary.Add(property.Name, BinaryData.FromString(property.Value.GetRawText())); - } - additionalProperties = additionalPropertiesDictionary; - return new AvroDataset(type, description.Value, structure.Value, schema.Value, linkedServiceName, Optional.ToDictionary(parameters), Optional.ToList(annotations), folder.Value, additionalProperties, location.Value, avroCompressionCodec.Value, Optional.ToNullable(avroCompressionLevel)); - } } } diff --git a/sdk/datafactory/Azure.ResourceManager.DataFactory/tests/Scenario/DataFactoryDatasetResourceTests.cs b/sdk/datafactory/Azure.ResourceManager.DataFactory/tests/Scenario/DataFactoryDatasetResourceTests.cs index fea326466d708..abccda22dabed 100644 --- a/sdk/datafactory/Azure.ResourceManager.DataFactory/tests/Scenario/DataFactoryDatasetResourceTests.cs +++ b/sdk/datafactory/Azure.ResourceManager.DataFactory/tests/Scenario/DataFactoryDatasetResourceTests.cs @@ -2213,5 +2213,44 @@ await DatasetCreate("sharepoint", CreateSharePointOnlineListLinkedService, (stri }); }); } + + [Test] + [RecordedTest] + public async Task Dataset_Avro_Create() + { + string name = "avro"; + // Get the resource group + string rgName = Recording.GenerateAssetName($"adf-rg-{name}-"); + var resourceGroup = await CreateResourceGroup(rgName, AzureLocation.WestUS2); + // Create a DataFactory + string dataFactoryName = Recording.GenerateAssetName($"adf-{name}-"); + DataFactoryResource dataFactory = await CreateDataFactory(resourceGroup, dataFactoryName); + // Create a LinkedService + string accessKey = GetStorageAccountAccessKey(resourceGroup); + string linkedServiceName = Recording.GenerateAssetName($"adf-linkedservice-{name}-"); + //await CreateLinkedService(dataFactory, linkedServiceName, accessKey); + + // Create Dataset + string datasetName = Recording.GenerateAssetName($"adf-dataset-{name}-"); + //var result = await dataFactory.GetDataFactoryDatasets().CreateOrUpdateAsync(WaitUntil.Completed, datasetName, dataFunc(linkedServiceName)); + //Assert.NotNull(result.Value.Id); + + await CreateAzureDataLakeGen2LinkedService(dataFactory, linkedServiceName, "fakePassword"); + + DataFactoryLinkedServiceReference dataFactoryLinkedServiceReference = new DataFactoryLinkedServiceReference(DataFactoryLinkedServiceReferenceType.LinkedServiceReference, linkedServiceName); + DataFactoryDatasetData data = new DataFactoryDatasetData(new AvroDataset(dataFactoryLinkedServiceReference) + { + //Schema = DataFactoryElement.FromLiteral(BinaryData.FromString("{\"type\": \"record\",\"name\": \"HybridDelivery.ClientLibraryJob\",\"fields\": [{\"name\": \"TIMESTAMP\",\"type\": [\"string\",\"null\"]}]}")), + Schema = DataFactoryElement.FromLiteral(BinaryData.FromString("123")), + DataLocation = new AzureBlobFSLocation() + { + FileName = "TestQuerySchema.avro", + FileSystem = "0-querytest", + FolderPath = "querytest" + } + }); + + await dataFactory.GetDataFactoryDatasets().CreateOrUpdateAsync(WaitUntil.Completed, datasetName, data); + } } } From 7bd2c21e54474177cf4158979f9a6c4f34778253 Mon Sep 17 00:00:00 2001 From: Ruoxuan Wang Date: Mon, 5 Feb 2024 11:10:58 +0800 Subject: [PATCH 2/8] add jsonDataSet code --- .../src/Customized/JsonDataset.cs | 193 ++++++++++++++++++ .../Models/JsonDataset.Serialization.cs | 146 ------------- 2 files changed, 193 insertions(+), 146 deletions(-) create mode 100644 sdk/datafactory/Azure.ResourceManager.DataFactory/src/Customized/JsonDataset.cs diff --git a/sdk/datafactory/Azure.ResourceManager.DataFactory/src/Customized/JsonDataset.cs b/sdk/datafactory/Azure.ResourceManager.DataFactory/src/Customized/JsonDataset.cs new file mode 100644 index 0000000000000..c8f69b9edd1e8 --- /dev/null +++ b/sdk/datafactory/Azure.ResourceManager.DataFactory/src/Customized/JsonDataset.cs @@ -0,0 +1,193 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#nullable disable + +using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.Text.Json; +using Azure.Core; +using Azure.Core.Expressions.DataFactory; + +namespace Azure.ResourceManager.DataFactory.Models +{ + /// Json dataset. + [CodeGenSuppress(nameof(JsonDataset), typeof(string), typeof(string), typeof(DataFactoryElement>), + typeof(DataFactoryElement), typeof(Core.Expressions.DataFactory.DataFactoryLinkedServiceReference), typeof(IDictionary), + typeof(IList), typeof(string), typeof(IDictionary), typeof(DatasetLocation), typeof(DataFactoryElement), typeof(DatasetCompression))] + public partial class JsonDataset : DataFactoryDatasetProperties + { + /// Initializes a new instance of . + /// Type of dataset. + /// Dataset description. + /// Columns that define the structure of the dataset. Type: array (or Expression with resultType array), itemType: DatasetDataElement. + /// Columns that define the physical type schema of the dataset. Type: array (or Expression with resultType array), itemType: DatasetSchemaDataElement. + /// Linked service reference. + /// Parameters for dataset. + /// List of tags that can be used for describing the Dataset. + /// The folder that this Dataset is in. If not specified, Dataset will appear at the root level. + /// Additional Properties. + /// + /// The location of the json data storage. + /// Please note is the base class. According to the scenario, a derived class of the base class might need to be assigned here, or this property needs to be casted to one of the possible derived classes. + /// The available derived classes include , , , , , , , , , , , , and . + /// + /// The code page name of the preferred encoding. If not specified, the default value is UTF-8, unless BOM denotes another Unicode encoding. Refer to the name column of the table in the following link to set supported values: https://msdn.microsoft.com/library/system.text.encoding.aspx. Type: string (or Expression with resultType string). + /// The data compression method used for the json dataset. + internal JsonDataset(string datasetType, string description, DataFactoryElement> structure, DataFactoryElement schema, Core.Expressions.DataFactory.DataFactoryLinkedServiceReference linkedServiceName, IDictionary parameters, IList annotations, DatasetFolder folder, IDictionary additionalProperties, DatasetLocation dataLocation, DataFactoryElement encodingName, DatasetCompression compression) : base(datasetType, description, structure, null, linkedServiceName, parameters, annotations, folder, additionalProperties) + { + DataLocation = dataLocation; + EncodingName = encodingName; + Compression = compression; + Schema = schema; + DatasetType = datasetType ?? "Json"; + } + + /// Columns that define the physical type schema of the dataset. Type: array (or Expression with resultType array), itemType: DatasetSchemaDataElement. + public new DataFactoryElement Schema { get; set; } + + internal static JsonDataset DeserializeJsonDataset(JsonElement element) + { + if (element.ValueKind == JsonValueKind.Null) + { + return null; + } + string type = default; + Optional description = default; + Optional>> structure = default; + Optional> schema = default; + Core.Expressions.DataFactory.DataFactoryLinkedServiceReference linkedServiceName = default; + Optional> parameters = default; + Optional> annotations = default; + Optional folder = default; + Optional location = default; + Optional> encodingName = default; + Optional compression = default; + IDictionary additionalProperties = default; + Dictionary additionalPropertiesDictionary = new Dictionary(); + foreach (var property in element.EnumerateObject()) + { + if (property.NameEquals("type"u8)) + { + type = property.Value.GetString(); + continue; + } + if (property.NameEquals("description"u8)) + { + description = property.Value.GetString(); + continue; + } + if (property.NameEquals("structure"u8)) + { + if (property.Value.ValueKind == JsonValueKind.Null) + { + continue; + } + structure = JsonSerializer.Deserialize>>(property.Value.GetRawText()); + continue; + } + if (property.NameEquals("schema"u8)) + { + if (property.Value.ValueKind == JsonValueKind.Null) + { + continue; + } + schema = JsonSerializer.Deserialize>(property.Value.GetRawText()); + continue; + } + if (property.NameEquals("linkedServiceName"u8)) + { + linkedServiceName = JsonSerializer.Deserialize(property.Value.GetRawText()); + continue; + } + if (property.NameEquals("parameters"u8)) + { + if (property.Value.ValueKind == JsonValueKind.Null) + { + continue; + } + Dictionary dictionary = new Dictionary(); + foreach (var property0 in property.Value.EnumerateObject()) + { + dictionary.Add(property0.Name, EntityParameterSpecification.DeserializeEntityParameterSpecification(property0.Value)); + } + parameters = dictionary; + continue; + } + if (property.NameEquals("annotations"u8)) + { + if (property.Value.ValueKind == JsonValueKind.Null) + { + continue; + } + List array = new List(); + foreach (var item in property.Value.EnumerateArray()) + { + if (item.ValueKind == JsonValueKind.Null) + { + array.Add(null); + } + else + { + array.Add(BinaryData.FromString(item.GetRawText())); + } + } + annotations = array; + continue; + } + if (property.NameEquals("folder"u8)) + { + if (property.Value.ValueKind == JsonValueKind.Null) + { + continue; + } + folder = DatasetFolder.DeserializeDatasetFolder(property.Value); + continue; + } + if (property.NameEquals("typeProperties"u8)) + { + if (property.Value.ValueKind == JsonValueKind.Null) + { + property.ThrowNonNullablePropertyIsNull(); + continue; + } + foreach (var property0 in property.Value.EnumerateObject()) + { + if (property0.NameEquals("location"u8)) + { + if (property0.Value.ValueKind == JsonValueKind.Null) + { + continue; + } + location = DatasetLocation.DeserializeDatasetLocation(property0.Value); + continue; + } + if (property0.NameEquals("encodingName"u8)) + { + if (property0.Value.ValueKind == JsonValueKind.Null) + { + continue; + } + encodingName = JsonSerializer.Deserialize>(property0.Value.GetRawText()); + continue; + } + if (property0.NameEquals("compression"u8)) + { + if (property0.Value.ValueKind == JsonValueKind.Null) + { + continue; + } + compression = DatasetCompression.DeserializeDatasetCompression(property0.Value); + continue; + } + } + continue; + } + additionalPropertiesDictionary.Add(property.Name, BinaryData.FromString(property.Value.GetRawText())); + } + additionalProperties = additionalPropertiesDictionary; + return new JsonDataset(type, description.Value, structure.Value, schema.Value, linkedServiceName, Optional.ToDictionary(parameters), Optional.ToList(annotations), folder.Value, additionalProperties, location.Value, encodingName.Value, compression.Value); + } + } +} diff --git a/sdk/datafactory/Azure.ResourceManager.DataFactory/src/Generated/Models/JsonDataset.Serialization.cs b/sdk/datafactory/Azure.ResourceManager.DataFactory/src/Generated/Models/JsonDataset.Serialization.cs index 1c74df50190de..ec018692d1b79 100644 --- a/sdk/datafactory/Azure.ResourceManager.DataFactory/src/Generated/Models/JsonDataset.Serialization.cs +++ b/sdk/datafactory/Azure.ResourceManager.DataFactory/src/Generated/Models/JsonDataset.Serialization.cs @@ -5,11 +5,8 @@ #nullable disable -using System; -using System.Collections.Generic; using System.Text.Json; using Azure.Core; -using Azure.Core.Expressions.DataFactory; namespace Azure.ResourceManager.DataFactory.Models { @@ -107,148 +104,5 @@ void IUtf8JsonSerializable.Write(Utf8JsonWriter writer) } writer.WriteEndObject(); } - - internal static JsonDataset DeserializeJsonDataset(JsonElement element) - { - if (element.ValueKind == JsonValueKind.Null) - { - return null; - } - string type = default; - Optional description = default; - Optional>> structure = default; - Optional>> schema = default; - DataFactoryLinkedServiceReference linkedServiceName = default; - Optional> parameters = default; - Optional> annotations = default; - Optional folder = default; - Optional location = default; - Optional> encodingName = default; - Optional compression = default; - IDictionary additionalProperties = default; - Dictionary additionalPropertiesDictionary = new Dictionary(); - foreach (var property in element.EnumerateObject()) - { - if (property.NameEquals("type"u8)) - { - type = property.Value.GetString(); - continue; - } - if (property.NameEquals("description"u8)) - { - description = property.Value.GetString(); - continue; - } - if (property.NameEquals("structure"u8)) - { - if (property.Value.ValueKind == JsonValueKind.Null) - { - continue; - } - structure = JsonSerializer.Deserialize>>(property.Value.GetRawText()); - continue; - } - if (property.NameEquals("schema"u8)) - { - if (property.Value.ValueKind == JsonValueKind.Null) - { - continue; - } - schema = JsonSerializer.Deserialize>>(property.Value.GetRawText()); - continue; - } - if (property.NameEquals("linkedServiceName"u8)) - { - linkedServiceName = JsonSerializer.Deserialize(property.Value.GetRawText()); - continue; - } - if (property.NameEquals("parameters"u8)) - { - if (property.Value.ValueKind == JsonValueKind.Null) - { - continue; - } - Dictionary dictionary = new Dictionary(); - foreach (var property0 in property.Value.EnumerateObject()) - { - dictionary.Add(property0.Name, EntityParameterSpecification.DeserializeEntityParameterSpecification(property0.Value)); - } - parameters = dictionary; - continue; - } - if (property.NameEquals("annotations"u8)) - { - if (property.Value.ValueKind == JsonValueKind.Null) - { - continue; - } - List array = new List(); - foreach (var item in property.Value.EnumerateArray()) - { - if (item.ValueKind == JsonValueKind.Null) - { - array.Add(null); - } - else - { - array.Add(BinaryData.FromString(item.GetRawText())); - } - } - annotations = array; - continue; - } - if (property.NameEquals("folder"u8)) - { - if (property.Value.ValueKind == JsonValueKind.Null) - { - continue; - } - folder = DatasetFolder.DeserializeDatasetFolder(property.Value); - continue; - } - if (property.NameEquals("typeProperties"u8)) - { - if (property.Value.ValueKind == JsonValueKind.Null) - { - property.ThrowNonNullablePropertyIsNull(); - continue; - } - foreach (var property0 in property.Value.EnumerateObject()) - { - if (property0.NameEquals("location"u8)) - { - if (property0.Value.ValueKind == JsonValueKind.Null) - { - continue; - } - location = DatasetLocation.DeserializeDatasetLocation(property0.Value); - continue; - } - if (property0.NameEquals("encodingName"u8)) - { - if (property0.Value.ValueKind == JsonValueKind.Null) - { - continue; - } - encodingName = JsonSerializer.Deserialize>(property0.Value.GetRawText()); - continue; - } - if (property0.NameEquals("compression"u8)) - { - if (property0.Value.ValueKind == JsonValueKind.Null) - { - continue; - } - compression = DatasetCompression.DeserializeDatasetCompression(property0.Value); - continue; - } - } - continue; - } - additionalPropertiesDictionary.Add(property.Name, BinaryData.FromString(property.Value.GetRawText())); - } - additionalProperties = additionalPropertiesDictionary; - return new JsonDataset(type, description.Value, structure.Value, schema.Value, linkedServiceName, Optional.ToDictionary(parameters), Optional.ToList(annotations), folder.Value, additionalProperties, location.Value, encodingName.Value, compression.Value); - } } } From d97e57a9b9021dee057761b6d44fd463b6b3a915 Mon Sep 17 00:00:00 2001 From: Bo Wang Date: Mon, 5 Feb 2024 15:52:27 +0800 Subject: [PATCH 3/8] Update schema --- .../DataFactoryDatasetResourceTests.cs | 55 +++++++++---------- 1 file changed, 26 insertions(+), 29 deletions(-) diff --git a/sdk/datafactory/Azure.ResourceManager.DataFactory/tests/Scenario/DataFactoryDatasetResourceTests.cs b/sdk/datafactory/Azure.ResourceManager.DataFactory/tests/Scenario/DataFactoryDatasetResourceTests.cs index abccda22dabed..ef8907d1935ac 100644 --- a/sdk/datafactory/Azure.ResourceManager.DataFactory/tests/Scenario/DataFactoryDatasetResourceTests.cs +++ b/sdk/datafactory/Azure.ResourceManager.DataFactory/tests/Scenario/DataFactoryDatasetResourceTests.cs @@ -2218,39 +2218,36 @@ await DatasetCreate("sharepoint", CreateSharePointOnlineListLinkedService, (stri [RecordedTest] public async Task Dataset_Avro_Create() { - string name = "avro"; - // Get the resource group - string rgName = Recording.GenerateAssetName($"adf-rg-{name}-"); - var resourceGroup = await CreateResourceGroup(rgName, AzureLocation.WestUS2); - // Create a DataFactory - string dataFactoryName = Recording.GenerateAssetName($"adf-{name}-"); - DataFactoryResource dataFactory = await CreateDataFactory(resourceGroup, dataFactoryName); - // Create a LinkedService - string accessKey = GetStorageAccountAccessKey(resourceGroup); - string linkedServiceName = Recording.GenerateAssetName($"adf-linkedservice-{name}-"); - //await CreateLinkedService(dataFactory, linkedServiceName, accessKey); - - // Create Dataset - string datasetName = Recording.GenerateAssetName($"adf-dataset-{name}-"); - //var result = await dataFactory.GetDataFactoryDatasets().CreateOrUpdateAsync(WaitUntil.Completed, datasetName, dataFunc(linkedServiceName)); - //Assert.NotNull(result.Value.Id); - - await CreateAzureDataLakeGen2LinkedService(dataFactory, linkedServiceName, "fakePassword"); - - DataFactoryLinkedServiceReference dataFactoryLinkedServiceReference = new DataFactoryLinkedServiceReference(DataFactoryLinkedServiceReferenceType.LinkedServiceReference, linkedServiceName); - DataFactoryDatasetData data = new DataFactoryDatasetData(new AvroDataset(dataFactoryLinkedServiceReference) + await DatasetCreate("avro", CreateAzureBlobStorageLinkedService, (string linkedServiceName) => { - //Schema = DataFactoryElement.FromLiteral(BinaryData.FromString("{\"type\": \"record\",\"name\": \"HybridDelivery.ClientLibraryJob\",\"fields\": [{\"name\": \"TIMESTAMP\",\"type\": [\"string\",\"null\"]}]}")), - Schema = DataFactoryElement.FromLiteral(BinaryData.FromString("123")), - DataLocation = new AzureBlobFSLocation() + return new DataFactoryDatasetData(new AvroDataset(new DataFactoryLinkedServiceReference(DataFactoryLinkedServiceReferenceType.LinkedServiceReference, linkedServiceName)) { - FileName = "TestQuerySchema.avro", - FileSystem = "0-querytest", - FolderPath = "querytest" - } + Schema = DataFactoryElement.FromLiteral(BinaryData.FromString("{\"type\": \"record\",\"name\": \"HybridDelivery.ClientLibraryJob\",\"fields\": [{\"name\": \"TIMESTAMP\",\"type\": [\"string\",\"null\"]}]}")), + DataLocation = new AzureBlobStorageLocation() + { + FileName = "TestQuerySchema.avro", + FolderPath = "querytest" + } + }); }); + } - await dataFactory.GetDataFactoryDatasets().CreateOrUpdateAsync(WaitUntil.Completed, datasetName, data); + [Test] + [RecordedTest] + public async Task Dataset_Json_Create() + { + await DatasetCreate("json", CreateAzureBlobStorageLinkedService, (string linkedServiceName) => + { + return new DataFactoryDatasetData(new JsonDataset(new DataFactoryLinkedServiceReference(DataFactoryLinkedServiceReferenceType.LinkedServiceReference, linkedServiceName)) + { + Schema = DataFactoryElement.FromLiteral(BinaryData.FromString("{\"type\": \"object\",\"properties\": {\"studentName\": {\"type\": \"string\"},\"age\": {\"type\": \"integer\"},\"gender\": {\"type\": \"string\"},\"studentID\": {\"type\": \"string\"},\"major\": {\"type\": \"string\"},\"grades\": {\"type\": \"object\",\"properties\": {\"math\": {\"type\": \"integer\"},\"english\": {\"type\": \"integer\"},\"programming\": {\"type\": \"integer\"}}},\"contact\": {\"type\":\"object\",\"properties\": {\"phone\": {\"type\": \"string\"},\"email\": {\"type\": \"string\"}}}}}")), + DataLocation = new AzureBlobStorageLocation() + { + FileName = "TestQuerySchema.json", + FolderPath = "querytest" + } + }); + }); } } } From fdbd3fbdb882d71126a9a90827ba8715a519be23 Mon Sep 17 00:00:00 2001 From: Ruoxuan Wang Date: Mon, 5 Feb 2024 15:58:50 +0800 Subject: [PATCH 4/8] fix CI error --- .../api/Azure.ResourceManager.DataFactory.netstandard2.0.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sdk/datafactory/Azure.ResourceManager.DataFactory/api/Azure.ResourceManager.DataFactory.netstandard2.0.cs b/sdk/datafactory/Azure.ResourceManager.DataFactory/api/Azure.ResourceManager.DataFactory.netstandard2.0.cs index 13419eafb8126..8248b4606a805 100644 --- a/sdk/datafactory/Azure.ResourceManager.DataFactory/api/Azure.ResourceManager.DataFactory.netstandard2.0.cs +++ b/sdk/datafactory/Azure.ResourceManager.DataFactory/api/Azure.ResourceManager.DataFactory.netstandard2.0.cs @@ -919,6 +919,7 @@ public AvroDataset(Azure.Core.Expressions.DataFactory.DataFactoryLinkedServiceRe public Azure.Core.Expressions.DataFactory.DataFactoryElement AvroCompressionCodec { get { throw null; } set { } } public int? AvroCompressionLevel { get { throw null; } set { } } public Azure.ResourceManager.DataFactory.Models.DatasetLocation DataLocation { get { throw null; } set { } } + public new Azure.Core.Expressions.DataFactory.DataFactoryElement Schema { get { throw null; } set { } } } public partial class AvroSink : Azure.ResourceManager.DataFactory.Models.CopySink { @@ -4566,6 +4567,7 @@ public JsonDataset(Azure.Core.Expressions.DataFactory.DataFactoryLinkedServiceRe public Azure.ResourceManager.DataFactory.Models.DatasetCompression Compression { get { throw null; } set { } } public Azure.ResourceManager.DataFactory.Models.DatasetLocation DataLocation { get { throw null; } set { } } public Azure.Core.Expressions.DataFactory.DataFactoryElement EncodingName { get { throw null; } set { } } + public new Azure.Core.Expressions.DataFactory.DataFactoryElement Schema { get { throw null; } set { } } } public partial class JsonReadSettings : Azure.ResourceManager.DataFactory.Models.FormatReadSettings { From 9936fac48cf852eef83ad8c04b626b64b469fbb5 Mon Sep 17 00:00:00 2001 From: Ruoxuan Wang Date: Mon, 5 Feb 2024 15:58:50 +0800 Subject: [PATCH 5/8] fix CI error --- .../api/Azure.ResourceManager.DataFactory.netstandard2.0.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sdk/datafactory/Azure.ResourceManager.DataFactory/api/Azure.ResourceManager.DataFactory.netstandard2.0.cs b/sdk/datafactory/Azure.ResourceManager.DataFactory/api/Azure.ResourceManager.DataFactory.netstandard2.0.cs index 13419eafb8126..8248b4606a805 100644 --- a/sdk/datafactory/Azure.ResourceManager.DataFactory/api/Azure.ResourceManager.DataFactory.netstandard2.0.cs +++ b/sdk/datafactory/Azure.ResourceManager.DataFactory/api/Azure.ResourceManager.DataFactory.netstandard2.0.cs @@ -919,6 +919,7 @@ public AvroDataset(Azure.Core.Expressions.DataFactory.DataFactoryLinkedServiceRe public Azure.Core.Expressions.DataFactory.DataFactoryElement AvroCompressionCodec { get { throw null; } set { } } public int? AvroCompressionLevel { get { throw null; } set { } } public Azure.ResourceManager.DataFactory.Models.DatasetLocation DataLocation { get { throw null; } set { } } + public new Azure.Core.Expressions.DataFactory.DataFactoryElement Schema { get { throw null; } set { } } } public partial class AvroSink : Azure.ResourceManager.DataFactory.Models.CopySink { @@ -4566,6 +4567,7 @@ public JsonDataset(Azure.Core.Expressions.DataFactory.DataFactoryLinkedServiceRe public Azure.ResourceManager.DataFactory.Models.DatasetCompression Compression { get { throw null; } set { } } public Azure.ResourceManager.DataFactory.Models.DatasetLocation DataLocation { get { throw null; } set { } } public Azure.Core.Expressions.DataFactory.DataFactoryElement EncodingName { get { throw null; } set { } } + public new Azure.Core.Expressions.DataFactory.DataFactoryElement Schema { get { throw null; } set { } } } public partial class JsonReadSettings : Azure.ResourceManager.DataFactory.Models.FormatReadSettings { From 95646113efce0093d7fd8f14ace04fc1115dd1f7 Mon Sep 17 00:00:00 2001 From: Bo Wang Date: Mon, 5 Feb 2024 16:30:31 +0800 Subject: [PATCH 6/8] Update code --- sdk/datafactory/Azure.ResourceManager.DataFactory/assets.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/datafactory/Azure.ResourceManager.DataFactory/assets.json b/sdk/datafactory/Azure.ResourceManager.DataFactory/assets.json index 3bec5ebcf58a7..6b936c8ca39c9 100644 --- a/sdk/datafactory/Azure.ResourceManager.DataFactory/assets.json +++ b/sdk/datafactory/Azure.ResourceManager.DataFactory/assets.json @@ -2,5 +2,5 @@ "AssetsRepo": "Azure/azure-sdk-assets", "AssetsRepoPrefixPath": "net", "TagPrefix": "net/datafactory/Azure.ResourceManager.DataFactory", - "Tag": "net/datafactory/Azure.ResourceManager.DataFactory_23a911dae9" + "Tag": "net/datafactory/Azure.ResourceManager.DataFactory_e3cf1b5acb" } From d8125ce56bd3634b51ee43e90fc9384ca1f0708c Mon Sep 17 00:00:00 2001 From: Ruoxuan Wang Date: Tue, 6 Feb 2024 13:15:27 +0800 Subject: [PATCH 7/8] resolve conflict --- .../Azure.ResourceManager.DataFactory.netstandard2.0.cs | 8 +++++++- .../src/Generated/Models/AvroDataset.Serialization.cs | 4 +--- .../src/Generated/Models/JsonDataset.Serialization.cs | 4 +--- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/sdk/datafactory/Azure.ResourceManager.DataFactory/api/Azure.ResourceManager.DataFactory.netstandard2.0.cs b/sdk/datafactory/Azure.ResourceManager.DataFactory/api/Azure.ResourceManager.DataFactory.netstandard2.0.cs index f4ba9c59828da..4949eda1f2092 100644 --- a/sdk/datafactory/Azure.ResourceManager.DataFactory/api/Azure.ResourceManager.DataFactory.netstandard2.0.cs +++ b/sdk/datafactory/Azure.ResourceManager.DataFactory/api/Azure.ResourceManager.DataFactory.netstandard2.0.cs @@ -1099,12 +1099,12 @@ public AvroDataset(Azure.Core.Expressions.DataFactory.DataFactoryLinkedServiceRe public Azure.Core.Expressions.DataFactory.DataFactoryElement AvroCompressionCodec { get { throw null; } set { } } public int? AvroCompressionLevel { get { throw null; } set { } } public Azure.ResourceManager.DataFactory.Models.DatasetLocation DataLocation { get { throw null; } set { } } + public new Azure.Core.Expressions.DataFactory.DataFactoryElement Schema { get { throw null; } set { } } Azure.ResourceManager.DataFactory.Models.AvroDataset System.ClientModel.Primitives.IJsonModel.Create(ref System.Text.Json.Utf8JsonReader reader, System.ClientModel.Primitives.ModelReaderWriterOptions options) { throw null; } void System.ClientModel.Primitives.IJsonModel.Write(System.Text.Json.Utf8JsonWriter writer, System.ClientModel.Primitives.ModelReaderWriterOptions options) { } Azure.ResourceManager.DataFactory.Models.AvroDataset System.ClientModel.Primitives.IPersistableModel.Create(System.BinaryData data, System.ClientModel.Primitives.ModelReaderWriterOptions options) { throw null; } string System.ClientModel.Primitives.IPersistableModel.GetFormatFromOptions(System.ClientModel.Primitives.ModelReaderWriterOptions options) { throw null; } System.BinaryData System.ClientModel.Primitives.IPersistableModel.Write(System.ClientModel.Primitives.ModelReaderWriterOptions options) { throw null; } - public new Azure.Core.Expressions.DataFactory.DataFactoryElement Schema { get { throw null; } set { } } } public partial class AvroSink : Azure.ResourceManager.DataFactory.Models.CopySink, System.ClientModel.Primitives.IJsonModel, System.ClientModel.Primitives.IPersistableModel { @@ -6487,6 +6487,12 @@ public JsonDataset(Azure.Core.Expressions.DataFactory.DataFactoryLinkedServiceRe public Azure.ResourceManager.DataFactory.Models.DatasetCompression Compression { get { throw null; } set { } } public Azure.ResourceManager.DataFactory.Models.DatasetLocation DataLocation { get { throw null; } set { } } public Azure.Core.Expressions.DataFactory.DataFactoryElement EncodingName { get { throw null; } set { } } + public new Azure.Core.Expressions.DataFactory.DataFactoryElement Schema { get { throw null; } set { } } + Azure.ResourceManager.DataFactory.Models.JsonDataset System.ClientModel.Primitives.IJsonModel.Create(ref System.Text.Json.Utf8JsonReader reader, System.ClientModel.Primitives.ModelReaderWriterOptions options) { throw null; } + void System.ClientModel.Primitives.IJsonModel.Write(System.Text.Json.Utf8JsonWriter writer, System.ClientModel.Primitives.ModelReaderWriterOptions options) { } + Azure.ResourceManager.DataFactory.Models.JsonDataset System.ClientModel.Primitives.IPersistableModel.Create(System.BinaryData data, System.ClientModel.Primitives.ModelReaderWriterOptions options) { throw null; } + string System.ClientModel.Primitives.IPersistableModel.GetFormatFromOptions(System.ClientModel.Primitives.ModelReaderWriterOptions options) { throw null; } + System.BinaryData System.ClientModel.Primitives.IPersistableModel.Write(System.ClientModel.Primitives.ModelReaderWriterOptions options) { throw null; } } public partial class JsonReadSettings : Azure.ResourceManager.DataFactory.Models.FormatReadSettings, System.ClientModel.Primitives.IJsonModel, System.ClientModel.Primitives.IPersistableModel { diff --git a/sdk/datafactory/Azure.ResourceManager.DataFactory/src/Generated/Models/AvroDataset.Serialization.cs b/sdk/datafactory/Azure.ResourceManager.DataFactory/src/Generated/Models/AvroDataset.Serialization.cs index a03eac4143047..9126f286ff3fe 100644 --- a/sdk/datafactory/Azure.ResourceManager.DataFactory/src/Generated/Models/AvroDataset.Serialization.cs +++ b/sdk/datafactory/Azure.ResourceManager.DataFactory/src/Generated/Models/AvroDataset.Serialization.cs @@ -10,6 +10,7 @@ using System.Collections.Generic; using System.Text.Json; using Azure.Core; +using Azure.Core.Expressions.DataFactory; namespace Azure.ResourceManager.DataFactory.Models { @@ -115,8 +116,6 @@ void IJsonModel.Write(Utf8JsonWriter writer, ModelReaderWriterOptio } writer.WriteEndObject(); } -<<<<<<< HEAD -======= AvroDataset IJsonModel.Create(ref Utf8JsonReader reader, ModelReaderWriterOptions options) { @@ -305,6 +304,5 @@ AvroDataset IPersistableModel.Create(BinaryData data, ModelReaderWr } string IPersistableModel.GetFormatFromOptions(ModelReaderWriterOptions options) => "J"; ->>>>>>> origin/main } } diff --git a/sdk/datafactory/Azure.ResourceManager.DataFactory/src/Generated/Models/JsonDataset.Serialization.cs b/sdk/datafactory/Azure.ResourceManager.DataFactory/src/Generated/Models/JsonDataset.Serialization.cs index 60b347e2e80e4..054a4e99edb4d 100644 --- a/sdk/datafactory/Azure.ResourceManager.DataFactory/src/Generated/Models/JsonDataset.Serialization.cs +++ b/sdk/datafactory/Azure.ResourceManager.DataFactory/src/Generated/Models/JsonDataset.Serialization.cs @@ -10,6 +10,7 @@ using System.Collections.Generic; using System.Text.Json; using Azure.Core; +using Azure.Core.Expressions.DataFactory; namespace Azure.ResourceManager.DataFactory.Models { @@ -115,8 +116,6 @@ void IJsonModel.Write(Utf8JsonWriter writer, ModelReaderWriterOptio } writer.WriteEndObject(); } -<<<<<<< HEAD -======= JsonDataset IJsonModel.Create(ref Utf8JsonReader reader, ModelReaderWriterOptions options) { @@ -305,6 +304,5 @@ JsonDataset IPersistableModel.Create(BinaryData data, ModelReaderWr } string IPersistableModel.GetFormatFromOptions(ModelReaderWriterOptions options) => "J"; ->>>>>>> origin/main } } From 04d6138aacbf69bc22789f91b09caa52134ab171 Mon Sep 17 00:00:00 2001 From: Ruoxuan Wang Date: Tue, 6 Feb 2024 14:47:31 +0800 Subject: [PATCH 8/8] update recording --- sdk/datafactory/Azure.ResourceManager.DataFactory/assets.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/datafactory/Azure.ResourceManager.DataFactory/assets.json b/sdk/datafactory/Azure.ResourceManager.DataFactory/assets.json index 6b936c8ca39c9..3a456ab611217 100644 --- a/sdk/datafactory/Azure.ResourceManager.DataFactory/assets.json +++ b/sdk/datafactory/Azure.ResourceManager.DataFactory/assets.json @@ -2,5 +2,5 @@ "AssetsRepo": "Azure/azure-sdk-assets", "AssetsRepoPrefixPath": "net", "TagPrefix": "net/datafactory/Azure.ResourceManager.DataFactory", - "Tag": "net/datafactory/Azure.ResourceManager.DataFactory_e3cf1b5acb" + "Tag": "net/datafactory/Azure.ResourceManager.DataFactory_8bebc7d7e6" }