Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Internal PR for Batch SDK 9.0 #1

Closed
wants to merge 10 commits into from
Closed

Internal PR for Batch SDK 9.0 #1

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
059e340
Increase client side timeout for some server requests
matthchr Aug 1, 2018
fee9013
Stop doing hacky %5C/%2F handling in shared key auth
matthchr Aug 1, 2018
58d2675
Fix bug where sharedkey auth fails in netcoreapp 2.1
matthchr Aug 1, 2018
597d0cc
Add CopyNodeFileContentToStream and CopyNodeFileContentToString
matthchr Aug 2, 2018
84ccb46
Add default retry policy on BatchClient
matthchr Aug 2, 2018
90ababd
Correct FileConventions error string
matthchr Aug 2, 2018
e320fc9
Remove OpenAsync since we don't make any async calls
matthchr Aug 3, 2018
dd76a48
Fix confusing ResizeErrors documentation
matthchr Aug 7, 2018
d64602e
Regenerate code based on 2018-08-01.7.0 Swagger spec
matthchr Aug 14, 2018
689a258
Fix tests
matthchr Aug 16, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/SDKs/Batch/DataPlane/Azure.Batch.IntegrationTests/AuthenticationTest.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ public async Task CanAuthenticateToServiceWithAADToken()
{
Func<Task<string>> tokenProvider = () => IntegrationTestCommon.GetAuthenticationTokenAsync("https://batch.core.windows.net/");

using (var client = await BatchClient.OpenAsync(new BatchTokenCredentials(TestCommon.Configuration.BatchAccountUrl, tokenProvider)))
using (var client = BatchClient.Open(new BatchTokenCredentials(TestCommon.Configuration.BatchAccountUrl, tokenProvider)))
{
await client.JobOperations.ListJobs().ToListAsync();
}
Expand Down
1 change: 0 additions & 1 deletion src/SDKs/Batch/DataPlane/Azure.Batch.IntegrationTests/CloudJobIntegrationTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -925,7 +925,6 @@ public async Task Job_GetTaskCounts_ReturnsCorrectCount()
var counts = await unboundJob.GetTaskCountsAsync().ConfigureAwait(false);

Assert.Equal(2, counts.Active);
Assert.Equal(TaskCountValidationStatus.Validated, counts.ValidationStatus);
}
finally
{
Expand Down
3 changes: 2 additions & 1 deletion src/SDKs/Batch/DataPlane/Azure.Batch.IntegrationTests/CloudPoolIntegrationTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -615,7 +615,7 @@ public void TestPoolObjectResizeStopResize()
CloudPool pool = batchCli.PoolOperations.CreatePool(poolId, PoolFixture.VMSize, new CloudServiceConfiguration(PoolFixture.OSFamily), targetDedicatedComputeNodes: targetDedicated);
pool.Commit();

this.testOutputHelper.WriteLine("Created pool {0}", poolId);
this.testOutputHelper.WriteLine($"Created pool {poolId}");


CloudPool boundPool = batchCli.PoolOperations.GetPool(poolId);
Expand All @@ -630,6 +630,7 @@ public void TestPoolObjectResizeStopResize()
boundPool.Refresh();

//The pool could be in stopping or steady state
this.testOutputHelper.WriteLine($"Pool allocation state: {boundPool.AllocationState}");
Assert.True(boundPool.AllocationState == AllocationState.Steady || boundPool.AllocationState == AllocationState.Stopping);
}
finally
Expand Down
7 changes: 3 additions & 4 deletions ...Ks/Batch/DataPlane/Azure.Batch.IntegrationTests/IntegrationTestUtilities/TestUtilities.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ public static BatchSharedKeyCredentials GetCredentialsFromEnvironment()

public static async Task<BatchClient> OpenBatchClientAsync(BatchSharedKeyCredentials sharedKeyCredentials, bool addDefaultRetryPolicy = true)
{
BatchClient client = await BatchClient.OpenAsync(sharedKeyCredentials);
BatchClient client = BatchClient.Open(sharedKeyCredentials);

//Force us to get exception if the server returns something we don't expect
//TODO: To avoid including this test assembly via "InternalsVisibleTo" we resort to some reflection trickery... maybe this property
Expand All @@ -48,10 +48,9 @@ public static async Task<BatchClient> OpenBatchClientAsync(BatchSharedKeyCredent
//TODO: Disabled for now because the swagger spec does not accurately reflect all properties returned by the server
//SetDeserializationSettings(client);

//Set up some common stuff like a retry policy
if (addDefaultRetryPolicy)
if (!addDefaultRetryPolicy)
{
client.CustomBehaviors.Add(RetryPolicyProvider.LinearRetryProvider(TimeSpan.FromSeconds(3), 5));
client.CustomBehaviors = client.CustomBehaviors.Where(behavior => !(behavior is RetryPolicyProvider)).ToList();
}

return client;
Expand Down
92 changes: 54 additions & 38 deletions src/SDKs/Batch/DataPlane/Azure.Batch.IntegrationTests/NodeFileIntegrationTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -157,12 +157,11 @@ public void Bug230385SupportDeleteNodeFileByTask()
//

//Delete single file
const string stdOutFileName = "stdout.txt";
NodeFile file = batchCli.JobOperations.GetNodeFile(jobId, taskId, stdOutFileName);
NodeFile file = batchCli.JobOperations.GetNodeFile(jobId, taskId, Constants.StandardOutFileName);
file.Delete();

//Ensure delete succeeded
TestUtilities.AssertThrows<BatchException>(() => batchCli.JobOperations.GetNodeFile(jobId, taskId, stdOutFileName));
TestUtilities.AssertThrows<BatchException>(() => batchCli.JobOperations.GetNodeFile(jobId, taskId, Constants.StandardOutFileName));

//Delete directory

Expand All @@ -176,12 +175,11 @@ public void Bug230385SupportDeleteNodeFileByTask()
//
// JobScheduleOperations delete task file
//
const string stdErrFileName = "stderr.txt";
batchCli.JobOperations.GetNodeFile(jobId, taskId, stdErrFileName);
batchCli.JobOperations.DeleteNodeFile(jobId, taskId, stdErrFileName);
batchCli.JobOperations.GetNodeFile(jobId, taskId, Constants.StandardErrorFileName);
batchCli.JobOperations.DeleteNodeFile(jobId, taskId, Constants.StandardErrorFileName);

//Ensure delete succeeded
TestUtilities.AssertThrows<BatchException>(() => batchCli.JobOperations.GetNodeFile(jobId, taskId, stdErrFileName));
TestUtilities.AssertThrows<BatchException>(() => batchCli.JobOperations.GetNodeFile(jobId, taskId, Constants.StandardErrorFileName));

//Delete directory
directory = batchCli.JobOperations.ListNodeFiles(jobId, directoryCreationTaskId2, recursive: true).First(item => item.Path.Contains(directoryNameTwo));
Expand All @@ -203,13 +201,13 @@ public void Bug230385SupportDeleteNodeFileByTask()

[Fact]
[Trait(TestTraits.Duration.TraitName, TestTraits.Duration.Values.MediumDuration)]
public void Bug1771166_1771181_1771038_GetListDeleteNodeFile()
public void TestNode_GetListDeleteFiles()
{
Action test = () =>
{
using (BatchClient batchCli = TestUtilities.OpenBatchClientAsync(TestUtilities.GetCredentialsFromEnvironment()).Result)
{
string jobId = "Bug1771166Job-" + TestUtilities.GetMyName();
string jobId = "TestNodeGetListDeleteFiles-" + TestUtilities.GetMyName();

try
{
Expand Down Expand Up @@ -246,10 +244,10 @@ public void Bug1771166_1771181_1771038_GetListDeleteNodeFile()
TaskStateMonitor taskStateMonitor = utilities.CreateTaskStateMonitor();

taskStateMonitor.WaitAll(
boundJob.ListTasks(),
Microsoft.Azure.Batch.Common.TaskState.Completed,
TimeSpan.FromMinutes(3));
boundJob.ListTasks(),
Microsoft.Azure.Batch.Common.TaskState.Completed,
TimeSpan.FromMinutes(3));

CloudTask boundTask = boundJob.GetTask(taskId);
//Since the compute node name comes back as "Node:<computeNodeId>" we need to split on : to get the actual compute node name
string computeNodeId = boundTask.ComputeNodeInformation.AffinityId.Split(':')[1];
Expand Down Expand Up @@ -312,12 +310,22 @@ public void Bug1771166_1771181_1771038_GetListDeleteNodeFile()
//
// Get file from operations
//
string filePathToGet = fileListFromComputeNode.First(f => !f.IsDirectory.Value).Path;
string filePathToGet = fileListFromComputeNode.First(f => !f.IsDirectory.Value && f.Properties.ContentLength > 0).Path;
this.testOutputHelper.WriteLine("Getting file: {0}", filePathToGet);
NodeFile computeNodeFileFromManager = batchCli.PoolOperations.GetNodeFile(this.poolFixture.PoolId, computeNodeId, filePathToGet);
this.testOutputHelper.WriteLine("Successfully retrieved file: {0}", filePathToGet);
this.testOutputHelper.WriteLine("---- File data: ----");
this.testOutputHelper.WriteLine(computeNodeFileFromManager.ReadAsString());
var computeNodeFileContentFromManager = computeNodeFileFromManager.ReadAsString();
this.testOutputHelper.WriteLine(computeNodeFileContentFromManager);
Assert.NotEmpty(computeNodeFileContentFromManager);

//
// Get file directly from operations (bypassing the properties call)
//
var computeNodeFileContentDirect = batchCli.PoolOperations.CopyNodeFileContentToString(this.poolFixture.PoolId, computeNodeId, filePathToGet);
this.testOutputHelper.WriteLine("---- File data: ----");
this.testOutputHelper.WriteLine(computeNodeFileContentDirect);
Assert.NotEmpty(computeNodeFileContentDirect);

//
// Get file from compute node
Expand All @@ -326,13 +334,22 @@ public void Bug1771166_1771181_1771038_GetListDeleteNodeFile()
NodeFile fileFromComputeNode = computeNode.GetNodeFile(filePathToGet);
this.testOutputHelper.WriteLine("Successfully retrieved file: {0}", filePathToGet);
this.testOutputHelper.WriteLine("---- File data: ----");
this.testOutputHelper.WriteLine(fileFromComputeNode.ReadAsString());
var computeNodeFileContentFromNode = fileFromComputeNode.ReadAsString();
this.testOutputHelper.WriteLine(computeNodeFileContentFromNode);
Assert.NotEmpty(computeNodeFileContentFromNode);

//
// Get file from compute node (bypassing the properties call)
//
computeNodeFileContentDirect = computeNode.CopyNodeFileContentToString(filePathToGet);
this.testOutputHelper.WriteLine("---- File data: ----");
this.testOutputHelper.WriteLine(computeNodeFileContentDirect);
Assert.NotEmpty(computeNodeFileContentDirect);

//
// NodeFile delete
//
const string stdOutFileName = "stdout.txt";
string filePath = Path.Combine(@"workitems", jobId, "job-1", taskId, stdOutFileName);
string filePath = Path.Combine(@"workitems", jobId, "job-1", taskId, Constants.StandardOutFileName);
NodeFile nodeFile = batchCli.PoolOperations.GetNodeFile(this.poolFixture.PoolId, computeNodeId, filePath);

nodeFile.Delete();
Expand All @@ -353,14 +370,13 @@ public void Bug1771166_1771181_1771038_GetListDeleteNodeFile()
//
// PoolManager delete node file
//
const string stdErrFileName = "stderr.txt";
filePath = Path.Combine(@"workitems", jobId, "job-1", taskId, stdErrFileName);
filePath = Path.Combine(@"workitems", jobId, "job-1", taskId, Constants.StandardErrorFileName);

NodeFile file = batchCli.JobOperations.GetNodeFile(jobId, taskId, stdErrFileName);
NodeFile file = batchCli.JobOperations.GetNodeFile(jobId, taskId, Constants.StandardErrorFileName);
batchCli.PoolOperations.DeleteNodeFile(this.poolFixture.PoolId, computeNodeId, filePath);

//Ensure delete succeeded
TestUtilities.AssertThrows<BatchException>(() => batchCli.JobOperations.GetNodeFile(jobId, taskId, stdErrFileName));
TestUtilities.AssertThrows<BatchException>(() => batchCli.JobOperations.GetNodeFile(jobId, taskId, Constants.StandardErrorFileName));

//Delete directory
directory = batchCli.PoolOperations.ListNodeFiles(this.poolFixture.PoolId, computeNodeId, recursive: true).First(item => item.Path.Contains(directoryNameTwo));
Expand Down Expand Up @@ -485,17 +501,16 @@ public void Bug1480491NodeFileFileProperties()
//
// NodeFile by task
//
const string stdOutFileName = "stdout.txt";
NodeFile file = batchCli.JobOperations.GetNodeFile(jobId, taskId, stdOutFileName);
NodeFile file = batchCli.JobOperations.GetNodeFile(jobId, taskId, Constants.StandardOutFileName);

this.testOutputHelper.WriteLine("File {0} has content length: {1}", stdOutFileName, file.Properties.ContentLength);
this.testOutputHelper.WriteLine("File {0} has content type: {1}", stdOutFileName, file.Properties.ContentType);
this.testOutputHelper.WriteLine("File {0} has content length: {1}", Constants.StandardOutFileName, file.Properties.ContentLength);
this.testOutputHelper.WriteLine("File {0} has content type: {1}", Constants.StandardOutFileName, file.Properties.ContentType);

this.testOutputHelper.WriteLine("File {0} has creation time: {1}", stdOutFileName, file.Properties.CreationTime);
this.testOutputHelper.WriteLine("File {0} has last modified time: {1}", stdOutFileName, file.Properties.LastModified);
this.testOutputHelper.WriteLine("File {0} has creation time: {1}", Constants.StandardOutFileName, file.Properties.CreationTime);
this.testOutputHelper.WriteLine("File {0} has last modified time: {1}", Constants.StandardOutFileName, file.Properties.LastModified);

Assert.Equal(expectedFileSize, file.Properties.ContentLength);
Assert.Equal("application/octet-stream", file.Properties.ContentType);
Assert.Equal("text/plain", file.Properties.ContentType);

//
// NodeFile by node
Expand All @@ -513,7 +528,7 @@ public void Bug1480491NodeFileFileProperties()
this.testOutputHelper.WriteLine("Found file: {0}", nodeFile.Path);
}

string filePathToGet = string.Format("workitems/{0}/{1}/{2}/{3}", jobId, "job-1", taskId, stdOutFileName);
string filePathToGet = string.Format("workitems/{0}/{1}/{2}/{3}", jobId, "job-1", taskId, Constants.StandardOutFileName);
file = computeNode.GetNodeFile(filePathToGet);

this.testOutputHelper.WriteLine("File {0} has content length: {1}", filePathToGet, file.Properties.ContentLength);
Expand All @@ -523,7 +538,7 @@ public void Bug1480491NodeFileFileProperties()
this.testOutputHelper.WriteLine("File {0} has last modified time: {1}", filePathToGet, file.Properties.LastModified);

Assert.Equal(expectedFileSize, file.Properties.ContentLength);
Assert.Equal("application/octet-stream", file.Properties.ContentType);
Assert.Equal("text/plain", file.Properties.ContentType);
}
finally
{
Expand All @@ -537,15 +552,15 @@ public void Bug1480491NodeFileFileProperties()

[Fact]
[Trait(TestTraits.Duration.TraitName, TestTraits.Duration.Values.MediumDuration)]
public void Bug1501413TestGetNodeFileByTask()
public void TestGetNodeFileByTask()
{
Action test = () =>
{
using (BatchClient batchCli = TestUtilities.OpenBatchClientAsync(TestUtilities.GetCredentialsFromEnvironment()).Result)
{
JobOperations jobOperations = batchCli.JobOperations;

string jobId = Microsoft.Azure.Batch.Constants.DefaultConveniencePrefix + TestUtilities.GetMyName() + "-Bug1501413TestGetNodeFileByTask";
string jobId = Constants.DefaultConveniencePrefix + TestUtilities.GetMyName() + "-" + nameof(TestGetNodeFileByTask);
try
{
//
Expand Down Expand Up @@ -585,20 +600,21 @@ public void Bug1501413TestGetNodeFileByTask()
TaskStateMonitor taskStateMonitor = utilities.CreateTaskStateMonitor();

//Wait for the task state to be running

taskStateMonitor.WaitAll(
jobOperations.ListTasks(jobId),
TaskState.Completed,
TimeSpan.FromSeconds(30));

//Download the data
this.testOutputHelper.WriteLine("Downloading the stdout for the file");
NodeFile file = jobOperations.GetNodeFile(jobId, taskId, "stdout.txt");

string data = file.ReadAsString(encoding: Encoding.UTF8);

NodeFile file = jobOperations.GetNodeFile(jobId, taskId, Constants.StandardOutFileName);
string data = file.ReadAsString();
this.testOutputHelper.WriteLine("Data: {0}", data);
Assert.Contains(taskMessage, data);

// Download the data again using the JobOperations read file content helper
data = batchCli.JobOperations.CopyNodeFileContentToString(jobId, taskId, Constants.StandardOutFileName);
this.testOutputHelper.WriteLine("Data: {0}", data);
Assert.Contains(taskMessage, data);
}
finally
Expand Down
1 change: 0 additions & 1 deletion src/SDKs/Batch/DataPlane/Azure.Batch.Unit.Tests/.gitignore
View file
Open in desktop

This file was deleted.

Loading