Ignore timeout error while polling operation status from ARM (#11531)

* ignore timeout error while polling for operation * add ignore timeout error threshold * review comments * updating tasks version using common module
microsoft · Oct 22, 2019 · 0453a78 · 0453a78
1 parent ffb9409
commit 0453a78
Show file tree

Hide file tree

Showing 31 changed files with 60 additions and 46 deletions.
diff --git a/Tasks/AzureAppServiceManageV0/task.json b/Tasks/AzureAppServiceManageV0/task.json
@@ -19,7 +19,7 @@
     "version": {
         "Major": 0,
         "Minor": 2,
-        "Patch": 61
+        "Patch": 62
     },
     "minimumAgentVersion": "1.102.0",
     "instanceNameFormat": "$(Action): $(WebAppName)",

diff --git a/Tasks/AzureAppServiceManageV0/task.loc.json b/Tasks/AzureAppServiceManageV0/task.loc.json
@@ -19,7 +19,7 @@
   "version": {
     "Major": 0,
     "Minor": 2,
-    "Patch": 61
+    "Patch": 62
   },
   "minimumAgentVersion": "1.102.0",
   "instanceNameFormat": "ms-resource:loc.instanceNameFormat",

diff --git a/Tasks/AzureFileCopyV2/task.json b/Tasks/AzureFileCopyV2/task.json
@@ -14,7 +14,7 @@
     "version": {
         "Major": 2,
         "Minor": 154,
-        "Patch": 10
+        "Patch": 11
     },
     "demands": [
         "azureps"

diff --git a/Tasks/AzureFileCopyV2/task.loc.json b/Tasks/AzureFileCopyV2/task.loc.json
@@ -14,7 +14,7 @@
   "version": {
     "Major": 2,
     "Minor": 154,
-    "Patch": 10
+    "Patch": 11
   },
   "demands": [
     "azureps"

diff --git a/Tasks/AzureFileCopyV3/task.json b/Tasks/AzureFileCopyV3/task.json
@@ -14,7 +14,7 @@
     "version": {
         "Major": 3,
         "Minor": 154,
-        "Patch": 10
+        "Patch": 11
     },
     "preview": true,
     "demands": [

diff --git a/Tasks/AzureFileCopyV3/task.loc.json b/Tasks/AzureFileCopyV3/task.loc.json
@@ -14,7 +14,7 @@
   "version": {
     "Major": 3,
     "Minor": 154,
-    "Patch": 10
+    "Patch": 11
   },
   "preview": true,
   "demands": [

diff --git a/Tasks/AzureKeyVaultV1/task.json b/Tasks/AzureKeyVaultV1/task.json
@@ -15,7 +15,7 @@
     "version": {
         "Major": 1,
         "Minor": 155,
-        "Patch": 4
+        "Patch": 5
     },
     "demands": [],
     "minimumAgentVersion": "2.0.0",

diff --git a/Tasks/AzureKeyVaultV1/task.loc.json b/Tasks/AzureKeyVaultV1/task.loc.json
@@ -15,7 +15,7 @@
   "version": {
     "Major": 1,
     "Minor": 155,
-    "Patch": 4
+    "Patch": 5
   },
   "demands": [],
   "minimumAgentVersion": "2.0.0",

diff --git a/Tasks/AzureMonitorAlertsV0/task.json b/Tasks/AzureMonitorAlertsV0/task.json
@@ -14,7 +14,7 @@
     "version": {
         "Major": 0,
         "Minor": 159,
-        "Patch": 0
+        "Patch": 1
     },
     "minimumAgentVersion": "2.111.0",
     "instanceNameFormat": "Configure Azure Alerts : $(ResourceName)",

diff --git a/Tasks/AzureMonitorAlertsV0/task.loc.json b/Tasks/AzureMonitorAlertsV0/task.loc.json
@@ -14,7 +14,7 @@
   "version": {
     "Major": 0,
     "Minor": 159,
-    "Patch": 0
+    "Patch": 1
   },
   "minimumAgentVersion": "2.111.0",
   "instanceNameFormat": "ms-resource:loc.instanceNameFormat",

diff --git a/Tasks/AzureMysqlDeploymentV1/task.json b/Tasks/AzureMysqlDeploymentV1/task.json
@@ -17,7 +17,7 @@
     "version": {
         "Major": 1,
         "Minor": 156,
-        "Patch": 11
+        "Patch": 12
     },
     "demands": [],
     "minimumAgentVersion": "1.100.0",

diff --git a/Tasks/AzureMysqlDeploymentV1/task.loc.json b/Tasks/AzureMysqlDeploymentV1/task.loc.json
@@ -17,7 +17,7 @@
   "version": {
     "Major": 1,
     "Minor": 156,
-    "Patch": 11
+    "Patch": 12
   },
   "demands": [],
   "minimumAgentVersion": "1.100.0",

diff --git a/Tasks/AzurePowerShellV4/task.json b/Tasks/AzurePowerShellV4/task.json
@@ -17,7 +17,7 @@
     "version": {
         "Major": 4,
         "Minor": 159,
-        "Patch": 2
+        "Patch": 3
     },
     "releaseNotes": "Added support for Az Module and cross platform agents.",
     "groups": [

diff --git a/Tasks/AzurePowerShellV4/task.loc.json b/Tasks/AzurePowerShellV4/task.loc.json
@@ -17,7 +17,7 @@
   "version": {
     "Major": 4,
     "Minor": 159,
-    "Patch": 2
+    "Patch": 3
   },
   "releaseNotes": "ms-resource:loc.releaseNotes",
   "groups": [

diff --git a/Tasks/AzureResourceGroupDeploymentV2/task.json b/Tasks/AzureResourceGroupDeploymentV2/task.json
@@ -15,7 +15,7 @@
     "version": {
         "Major": 2,
         "Minor": 157,
-        "Patch": 4
+        "Patch": 5
     },
     "demands": [],
     "minimumAgentVersion": "2.119.1",

diff --git a/Tasks/AzureResourceGroupDeploymentV2/task.loc.json b/Tasks/AzureResourceGroupDeploymentV2/task.loc.json
@@ -15,7 +15,7 @@
   "version": {
     "Major": 2,
     "Minor": 157,
-    "Patch": 4
+    "Patch": 5
   },
   "demands": [],
   "minimumAgentVersion": "2.119.1",

diff --git a/Tasks/AzureResourceManagerTemplateDeploymentV3/task.json b/Tasks/AzureResourceManagerTemplateDeploymentV3/task.json
@@ -15,7 +15,7 @@
     "version": {
         "Major": 3,
         "Minor": 1,
-        "Patch": 0
+        "Patch": 1
     },
     "preview": "true",
     "demands": [],

diff --git a/Tasks/AzureResourceManagerTemplateDeploymentV3/task.loc.json b/Tasks/AzureResourceManagerTemplateDeploymentV3/task.loc.json
@@ -15,7 +15,7 @@
   "version": {
     "Major": 3,
     "Minor": 1,
-    "Patch": 0
+    "Patch": 1
   },
   "preview": "true",
   "demands": [],

diff --git a/Tasks/AzureRmWebAppDeploymentV4/task.json b/Tasks/AzureRmWebAppDeploymentV4/task.json
@@ -18,7 +18,7 @@
     "version": {
         "Major": 4,
         "Minor": 157,
-        "Patch": 5
+        "Patch": 6
     },
     "releaseNotes": "What's new in version 4.*<br />Supports Zip Deploy, Run From Package, War Deploy [Details here](https://aka.ms/appServiceDeploymentMethods)<br />Supports App Service Environments<br />Improved UI for discovering different App service types supported by the task<br/>Run From Package is the preferred deployment method, which makes files in wwwroot folder read-only<br/>Click [here](https://aka.ms/azurermwebdeployreadme) for more information.",
     "minimumAgentVersion": "2.104.1",

diff --git a/Tasks/AzureRmWebAppDeploymentV4/task.loc.json b/Tasks/AzureRmWebAppDeploymentV4/task.loc.json
@@ -18,7 +18,7 @@
   "version": {
     "Major": 4,
     "Minor": 157,
-    "Patch": 5
+    "Patch": 6
   },
   "releaseNotes": "ms-resource:loc.releaseNotes",
   "minimumAgentVersion": "2.104.1",

diff --git a/Tasks/AzureVmssDeploymentV0/task.json b/Tasks/AzureVmssDeploymentV0/task.json
@@ -15,7 +15,7 @@
     "version": {
         "Major": 0,
         "Minor": 1,
-        "Patch": 3
+        "Patch": 4
     },
     "demands": [],
     "minimumAgentVersion": "2.0.0",

diff --git a/Tasks/AzureVmssDeploymentV0/task.loc.json b/Tasks/AzureVmssDeploymentV0/task.loc.json
@@ -15,7 +15,7 @@
   "version": {
     "Major": 0,
     "Minor": 1,
-    "Patch": 3
+    "Patch": 4
   },
   "demands": [],
   "minimumAgentVersion": "2.0.0",

diff --git a/Tasks/Common/azure-arm-rest-v2/AzureServiceClientBase.ts b/Tasks/Common/azure-arm-rest-v2/AzureServiceClientBase.ts
@@ -140,32 +140,46 @@ export class AzureServiceClientBase {
         timeoutInMinutes = timeoutInMinutes || this.longRunningOperationRetryTimeout;
         var timeout = new Date().getTime() + timeoutInMinutes * 60 * 1000;
         var waitIndefinitely = timeoutInMinutes == 0;
+        var ignoreTimeoutErrorThreshold = 5;
         var request = new webClient.WebRequest();
         request.method = "GET";
         request.uri = response.headers["azure-asyncoperation"] || response.headers["location"];
         if (!request.uri) {
             throw new Error(tl.loc("InvalidResponseLongRunningOperation"));
         }
         while (true) {
-            response = await this.beginRequest(request);
-            tl.debug(`Response status code : ${response.statusCode}`);
-            if (response.statusCode === 202 || (response.body && (response.body.status == "Accepted" || response.body.status == "Running" || response.body.status == "InProgress"))) {
-                if(response.body && response.body.status) {
-                    tl.debug(`Response status : ${response.body.status}`);
+            try {
+                response = await this.beginRequest(request);
+                tl.debug(`Response status code : ${response.statusCode}`);
+                if (response.statusCode === 202 || (response.body && (response.body.status == "Accepted" || response.body.status == "Running" || response.body.status == "InProgress"))) {
+                    if (response.body && response.body.status) {
+                        tl.debug(`Response status : ${response.body.status}`);
+                    }
+                    // If timeout; throw;
+                    if (!waitIndefinitely && timeout < new Date().getTime()) {
+                        throw new Error(tl.loc("TimeoutWhileWaiting"));
+                    }
+
+                    // Retry after given interval.
+                    var sleepDuration = 15;
+                    if (response.headers["retry-after"]) {
+                        sleepDuration = parseInt(response.headers["retry-after"]);
+                    }
+                    await this.sleepFor(sleepDuration);
+                } else {
+                    break;
                 }
-                // If timeout; throw;
-                if (!waitIndefinitely && timeout < new Date().getTime()) {
-                    throw new Error(tl.loc("TimeoutWhileWaiting"));
+            }
+            catch (error) {
+                let errorString: string = (!!error && error.toString()) || "";
+                if(!!errorString && errorString.toLowerCase().indexOf("request timeout") >= 0 && ignoreTimeoutErrorThreshold > 0) {
+                    // Ignore Request Timeout error and continue polling operation
+                    tl.debug(`Request Timeout: ${request.uri}`);
+                    ignoreTimeoutErrorThreshold--;
                 }
-
-                // Retry after given interval.
-                var sleepDuration = 15;
-                if (response.headers["retry-after"]) {
-                    sleepDuration = parseInt(response.headers["retry-after"]);
+                else {
+                    throw error;
                 }
-                await this.sleepFor(sleepDuration);
-            } else {
-                break;
             }
         }
 

diff --git a/Tasks/ContainerBuildV0/task.json b/Tasks/ContainerBuildV0/task.json
@@ -14,7 +14,7 @@
     "version": {
         "Major": 0,
         "Minor": 160,
-        "Patch": 1
+        "Patch": 2
     },
     "demands": [],
     "satisfies": [

diff --git a/Tasks/ContainerBuildV0/task.loc.json b/Tasks/ContainerBuildV0/task.loc.json
@@ -14,7 +14,7 @@
   "version": {
     "Major": 0,
     "Minor": 160,
-    "Patch": 1
+    "Patch": 2
   },
   "demands": [],
   "satisfies": [

diff --git a/Tasks/HelmDeployV0/task.json b/Tasks/HelmDeployV0/task.json
@@ -14,7 +14,7 @@
     "version": {
         "Major": 0,
         "Minor": 159,
-        "Patch": 5
+        "Patch": 6
     },
     "demands": [],
     "groups": [

diff --git a/Tasks/HelmDeployV0/task.loc.json b/Tasks/HelmDeployV0/task.loc.json
@@ -14,7 +14,7 @@
   "version": {
     "Major": 0,
     "Minor": 159,
-    "Patch": 5
+    "Patch": 6
   },
   "demands": [],
   "groups": [

diff --git a/Tasks/KubernetesV1/task.json b/Tasks/KubernetesV1/task.json
@@ -14,7 +14,7 @@
     "version": {
         "Major": 1,
         "Minor": 160,
-        "Patch": 2
+        "Patch": 3
     },
     "demands": [],
     "releaseNotes": "What's new in Version 1.0:<br/>&nbsp;Added new service connection type input for easy selection of Azure AKS cluster.<br/>&nbsp;Replaced output variable input with output variables section that we had added in all tasks.",

diff --git a/Tasks/KubernetesV1/task.loc.json b/Tasks/KubernetesV1/task.loc.json
@@ -14,7 +14,7 @@
   "version": {
     "Major": 1,
     "Minor": 160,
-    "Patch": 2
+    "Patch": 3
   },
   "demands": [],
   "releaseNotes": "ms-resource:loc.releaseNotes",

diff --git a/Tasks/PackerBuildV1/task.json b/Tasks/PackerBuildV1/task.json
@@ -15,7 +15,7 @@
     "version": {
         "Major": 1,
         "Minor": 1,
-        "Patch": 3
+        "Patch": 4
     },
     "demands": [],
     "minimumAgentVersion": "2.0.0",

diff --git a/Tasks/PackerBuildV1/task.loc.json b/Tasks/PackerBuildV1/task.loc.json
@@ -15,7 +15,7 @@
   "version": {
     "Major": 1,
     "Minor": 1,
-    "Patch": 3
+    "Patch": 4
   },
   "demands": [],
   "minimumAgentVersion": "2.0.0",