diff --git a/Tasks/AzureIoTEdgeV2/Strings/resources.resjson/en-US/resources.resjson b/Tasks/AzureIoTEdgeV2/Strings/resources.resjson/en-US/resources.resjson index 08bdb05afb45..a5f04de2592a 100644 --- a/Tasks/AzureIoTEdgeV2/Strings/resources.resjson/en-US/resources.resjson +++ b/Tasks/AzureIoTEdgeV2/Strings/resources.resjson/en-US/resources.resjson @@ -37,6 +37,8 @@ "loc.input.help.defaultPlatform": "In your **.template.json**, you can leave the modules platform unspecified. For these modules, the **default platform** will be used.", "loc.input.label.fillRegistryCredential": "Add registry credential to deployment manifest", "loc.input.help.fillRegistryCredential": "Add registry credential for pushing docker images to deployment manifest", + "loc.input.label.deploymentManifestOutputPath": "Output path", + "loc.input.help.deploymentManifestOutputPath": "The output path of generated deployment manifest", "loc.input.label.bypassModules": "Bypass module(s)", "loc.input.help.bypassModules": "Select the module(s) that you **DO NOT** need to build(or push) in the .template.json, specify module names and separate with comma.\n Example: if you have 2 modules **SampleModule1,SampleModule2** in your .template.json, you want to just build or push **SampleModule1**, then you set the bypass modules as **SampleModule2**. Leave empty if you would like to build all the modules in .template.json.", "loc.messages.BuildingModules": "Building module images...", @@ -57,7 +59,7 @@ "loc.messages.DependencyInstallSuccess": "%s installed with version: %s", "loc.messages.DependencyInstallFail": "%s installation failed, see detailed error in debug mode", "loc.messages.TemplateFileInvalid": "The path of template file is not valid: %s", - "loc.messages.ContainerRegistryInvalid": "Failed to fetch container registry authentication token, please check you container registry setting in build task. The token is %s", + "loc.messages.InvalidContainerRegistry": "Failed to fetch container registry authentication token, please check you container registry setting in build task. The username for container registry is %s", "loc.messages.DeploymentFileNotFound": "Deployment file can't be found. Please ensure Path of deployment file is correctly set in the task.", "loc.messages.ValidDeploymentFileNotFound": "Cannot find a valid deployment file. Please ensure Path of deployment file is correctly set in the task.", "loc.messages.AzureSdkNotFound": "Azure SDK not found", @@ -66,5 +68,7 @@ "loc.messages.InvalidRegistryCredentialWarning": "Failed to login %s with given credential. %s", "loc.messages.CheckModuleImageExistenceError": "%s does not exist or the credential is not set correctly. Error: %s", "loc.messages.StartGenerateDeploymentManifest": "Start generating deployment manifest...", - "loc.messages.FinishGenerateDeploymentManifest": "Finished generating deployment manifest." + "loc.messages.FinishGenerateDeploymentManifest": "Finished generating deployment manifest.", + "loc.messages.LoginRegistrySucess": "Successfully logged in to registry %s", + "loc.messages.SkipSettingEnvironmentVariableForSecret": "Environment variable %s already exist. Skip setting environment varialbe for secret: %s." } \ No newline at end of file diff --git a/Tasks/AzureIoTEdgeV2/buildimage.ts b/Tasks/AzureIoTEdgeV2/buildimage.ts index 784c57febab8..71170da0624a 100644 --- a/Tasks/AzureIoTEdgeV2/buildimage.ts +++ b/Tasks/AzureIoTEdgeV2/buildimage.ts @@ -17,21 +17,13 @@ export async function run() { util.setupIotedgedev(); - let envList = { - [Constants.iotedgedevEnv.deploymentFileOutputFolder]: tl.getVariable(Constants.outputFileFolder), - }; - - // Pass task variable to sub process - let tlVariables = tl.getVariables(); - for (let v of tlVariables) { - // The variables in VSTS build contains dot, need to convert to underscore. - let name = v.name.replace('.', '_').toUpperCase(); - if (!envList[name]) { - envList[name] = v.value; - } - } + let envList = process.env; + util.setCliVarialbe(envList, Constants.iotedgedevEnv.deploymentFileOutputFolder, tl.getVariable(Constants.outputFileFolder)); + + // Pass secrets to sub process + util.populateSecretToEnvironmentVariable(envList); - tl.debug(`Following variables will be passed to the iotedgedev command: ${JSON.stringify(envList)}`); + tl.debug(`Following variables will be passed to the iotedgedev command: ${Object.keys(envList).join(", ")}`); let outputStream: EchoStream = new EchoStream(); @@ -49,7 +41,7 @@ export async function run() { let outLog: string = outputStream.content; let filterReg: RegExp = /Expanding '[^']*' to '([^']*)'/g; let matches: RegExpMatchArray = filterReg.exec(outLog); - if(matches && matches[1]) { + if (matches && matches[1]) { tl.setVariable(Constants.outputVariableDeploymentPathKey, matches[1]); tl.setVariable('_' + Constants.outputVariableDeploymentPathKey, matches[1]); tl.debug(`Set ${Constants.outputVariableDeploymentPathKey} to ${matches[1]}`); diff --git a/Tasks/AzureIoTEdgeV2/constant.ts b/Tasks/AzureIoTEdgeV2/constant.ts index 75f838f62e78..56f6d378c4da 100644 --- a/Tasks/AzureIoTEdgeV2/constant.ts +++ b/Tasks/AzureIoTEdgeV2/constant.ts @@ -9,13 +9,13 @@ export default class Constants { public static folderNameConfig = "config"; public static iotedgedev = "iotedgedev"; public static iotedgedevLockVersionKey = "IOTEDGEDEV_VERSION"; - public static iotedgedevDefaultVersion = "1.1.0"; + public static iotedgedevDefaultVersion = "2.0"; public static iotedgedevEnv = { registryServer: "CONTAINER_REGISTRY_SERVER", registryUsername: "CONTAINER_REGISTRY_USERNAME", registryPassword: "CONTAINER_REGISTRY_PASSWORD", bypassModules: "BYPASS_MODULES", - deploymentFileOutputPath: "DEPLOYMENT_CONFIG_FILE", + deploymentFileOutputName: "DEPLOYMENT_CONFIG_FILE", deploymentFileOutputFolder: "CONFIG_OUTPUT_DIR", }; public static outputFileFolder = "Build.ArtifactStagingDirectory"; @@ -25,6 +25,7 @@ export default class Constants { public static defaultDockerHubHostname = "docker.io"; public static variableKeyDisableTelemetry = "DISABLE_TELEMETRY"; public static execSyncSilentOption = { silent: true } as IExecSyncOptions; + public static defaultExecOption = {} as IExecSyncOptions; public static UTF8 = "utf8"; public static outputVariableDeploymentPathKey = "DEPLOYMENT_FILE_PATH"; } \ No newline at end of file diff --git a/Tasks/AzureIoTEdgeV2/deployimage.ts b/Tasks/AzureIoTEdgeV2/deployimage.ts index 876c44454a47..3142f69224f6 100644 --- a/Tasks/AzureIoTEdgeV2/deployimage.ts +++ b/Tasks/AzureIoTEdgeV2/deployimage.ts @@ -6,6 +6,9 @@ import util from "./util"; import Constants from "./constant"; import { IExecSyncOptions } from 'azure-pipelines-task-lib/toolrunner'; import { TelemetryEvent } from './telemetry'; +import * as stream from "stream"; +import EchoStream from './echostream'; +import { IExecOptions } from 'azure-pipelines-task-lib/toolrunner'; class azureclitask { private static isLoggedIn = false; @@ -87,10 +90,15 @@ class azureclitask { // If error when get iot hub information, ignore. } + let outputStream: EchoStream = new EchoStream(); + let execOptions: IExecOptions = { + errStream: outputStream as stream.Writable + } as IExecOptions; + let result1 = tl.execSync('az', script1, Constants.execSyncSilentOption); - let result2 = await tl.exec('az', script2); + let result2 = await tl.exec('az', script2, execOptions); if (result2 !== 0) { - throw new Error(`Error for deployment`); + throw new Error(`Failed to create deployment. Error: ${outputStream.content}`); } } catch (err) { @@ -222,8 +230,12 @@ class imagevalidationtask { tl.debug(JSON.stringify(loginResult)); if (loginResult.code != 0) { tl.warning(tl.loc("InvalidRegistryCredentialWarning", credential.address, loginResult.stderr)); + } else { + tl.loc("LoginRegistrySucess", credential.address); } }); + } else { + tl.debug("No registry credentials found in deployment manifest.") } tl.setVariable("DOCKER_CLI_EXPERIMENTAL", "enabled"); diff --git a/Tasks/AzureIoTEdgeV2/genconfig.ts b/Tasks/AzureIoTEdgeV2/genconfig.ts index c9e8dab28afa..203b9dbbb6e0 100644 --- a/Tasks/AzureIoTEdgeV2/genconfig.ts +++ b/Tasks/AzureIoTEdgeV2/genconfig.ts @@ -14,20 +14,20 @@ export async function run() { util.setTaskRootPath(path.dirname(templateFilePath)); util.setupIotedgedev(); - - let envList = { - [Constants.iotedgedevEnv.deploymentFileOutputFolder]: tl.getVariable(Constants.outputFileFolder), - }; - - // Pass task variable to sub process - let tlVariables = tl.getVariables(); - for (let v of tlVariables) { - // The variables in VSTS build contains dot, need to convert to underscore. - let name = v.name.replace('.', '_').toUpperCase(); - if (!envList[name]) { - envList[name] = v.value; - } - } + + let outputPath = tl.getInput('deploymentManifestOutputPath', true); + let outputFileFolder = path.dirname(outputPath); + let outputFileName = path.basename(outputPath); + + let envList = process.env; + //Set output path of iotedgedev genconfig command + tl.debug(`Setting deployment manifest output folder to ${outputFileFolder}`); + util.setCliVarialbe(envList, Constants.iotedgedevEnv.deploymentFileOutputFolder, outputFileFolder); + tl.debug(`Setting deployment manifest output file name to ${outputFileName}`) + util.setCliVarialbe(envList, Constants.iotedgedevEnv.deploymentFileOutputName, outputFileName) + + // Pass secrets to sub process + util.populateSecretToEnvironmentVariable(envList); let execOptions: IExecOptions = { cwd: tl.cwd(), @@ -38,4 +38,7 @@ export async function run() { command += ` --file "${templateFilePath}"`; command += ` --platform "${defaultPlatform}"`; await tl.exec(`${Constants.iotedgedev}`, command, execOptions); + + tl.setVariable(Constants.outputVariableDeploymentPathKey, outputPath); + tl.debug(`Set ${Constants.outputVariableDeploymentPathKey} to ${outputPath}`); } \ No newline at end of file diff --git a/Tasks/AzureIoTEdgeV2/pushimage.ts b/Tasks/AzureIoTEdgeV2/pushimage.ts index 10584c47b653..31068363f267 100644 --- a/Tasks/AzureIoTEdgeV2/pushimage.ts +++ b/Tasks/AzureIoTEdgeV2/pushimage.ts @@ -20,7 +20,11 @@ function getRegistryAuthenticationToken(): RegistryCredential { } if (token == null || token.username == null || token.password == null || token.serverUrl == null) { - throw Error(tl.loc('ContainerRegistryInvalid', JSON.stringify(token))); + let username = ""; + if (token != null && token.username != null) { + username = token.username; + } + throw Error(tl.loc('InvalidContainerRegistry', username)); } return token; } @@ -52,24 +56,18 @@ export async function run() { */ tl.execSync(`docker`, `login -u "${registryAuthenticationToken.username}" -p "${registryAuthenticationToken.password}" ${registryAuthenticationToken.serverUrl}`, Constants.execSyncSilentOption) - let envList = { - [Constants.iotedgedevEnv.bypassModules]: bypassModules, - [Constants.iotedgedevEnv.registryServer]: registryAuthenticationToken.serverUrl, - [Constants.iotedgedevEnv.registryUsername]: registryAuthenticationToken.username, - [Constants.iotedgedevEnv.registryPassword]: registryAuthenticationToken.password, - }; - - // Pass task variable to sub process - let tlVariables = tl.getVariables(); - for (let v of tlVariables) { - // The variables in VSTS build contains dot, need to convert to underscore. - let name = v.name.replace('.', '_').toUpperCase(); - if (!envList[name]) { - envList[name] = v.value; - } - } + let envList = process.env; + // Set bypass modules + util.setCliVarialbe(envList, Constants.iotedgedevEnv.bypassModules, bypassModules); + // Set registry credentials + util.setCliVarialbe(envList, Constants.iotedgedevEnv.registryServer, registryAuthenticationToken.serverUrl); + util.setCliVarialbe(envList, Constants.iotedgedevEnv.registryUsername, registryAuthenticationToken.username); + util.setCliVarialbe(envList, Constants.iotedgedevEnv.registryPassword, registryAuthenticationToken.password); + + // Pass secrets to sub process + util.populateSecretToEnvironmentVariable(envList); - tl.debug(`Following variables will be passed to the iotedgedev command: ${JSON.stringify(envList)}`); + tl.debug(`Following variables will be passed to the iotedgedev command: ${Object.keys(envList).join(", ")}`); try { let execOptions: IExecOptions = { diff --git a/Tasks/AzureIoTEdgeV2/task.json b/Tasks/AzureIoTEdgeV2/task.json index 09bcdaa6991e..fd9c76b632e8 100644 --- a/Tasks/AzureIoTEdgeV2/task.json +++ b/Tasks/AzureIoTEdgeV2/task.json @@ -13,10 +13,11 @@ "author": "Microsoft Corporation", "version": { "Major": 2, - "Minor": 1, - "Patch": 5 + "Minor": 2, + "Patch": 0 }, - "preview": true, + "preview": false, + "showEnvironmentVariables": true, "instanceNameFormat": "Azure IoT Edge - $(action)", "groups": [ { @@ -51,7 +52,7 @@ "name": "deploymentFilePath", "type": "filePath", "label": "Deployment file", - "defaultValue": "$(System.DefaultWorkingDirectory)/**/*.json", + "defaultValue": "$(System.DefaultWorkingDirectory)/config/deployment.json", "required": true, "visibleRule": "action == Deploy to IoT Edge devices", "helpMarkDown": "Select the deployment json file.\n If this task is in **release pipeline**, you need to set the location of deployment file in artifact.(The default value works for most conditions).\n If this task is in **build pipeline**, you need to set it to the path of **Path of output deployment file**." @@ -200,6 +201,15 @@ }, "helpMarkDown": "Add registry credential for pushing docker images to deployment manifest" }, + { + "name": "deploymentManifestOutputPath", + "type": "filePath", + "label": "Output path", + "defaultValue": "$(System.DefaultWorkingDirectory)/config/deployment.json", + "visibleRule": "action == Generate deployment manifest", + "required": true, + "helpMarkDown": "The output path of generated deployment manifest" + }, { "name": "bypassModules", "type": "string", @@ -247,7 +257,7 @@ "DependencyInstallSuccess": "%s installed with version: %s", "DependencyInstallFail": "%s installation failed, see detailed error in debug mode", "TemplateFileInvalid": "The path of template file is not valid: %s", - "ContainerRegistryInvalid": "Failed to fetch container registry authentication token, please check you container registry setting in build task. The token is %s", + "InvalidContainerRegistry": "Failed to fetch container registry authentication token, please check you container registry setting in build task. The username for container registry is %s", "DeploymentFileNotFound": "Deployment file can't be found. Please ensure Path of deployment file is correctly set in the task.", "ValidDeploymentFileNotFound": "Cannot find a valid deployment file. Please ensure Path of deployment file is correctly set in the task.", "AzureSdkNotFound": "Azure SDK not found", @@ -256,7 +266,9 @@ "InvalidRegistryCredentialWarning": "Failed to login %s with given credential. %s", "CheckModuleImageExistenceError": "%s does not exist or the credential is not set correctly. Error: %s", "StartGenerateDeploymentManifest": "Start generating deployment manifest...", - "FinishGenerateDeploymentManifest": "Finished generating deployment manifest." + "FinishGenerateDeploymentManifest": "Finished generating deployment manifest.", + "LoginRegistrySucess": "Successfully logged in to registry %s", + "SkipSettingEnvironmentVariableForSecret": "Environment variable %s already exist. Skip setting environment varialbe for secret: %s." }, "OutputVariables": [ { diff --git a/Tasks/AzureIoTEdgeV2/task.loc.json b/Tasks/AzureIoTEdgeV2/task.loc.json index 64fa938e5631..9148401600a1 100644 --- a/Tasks/AzureIoTEdgeV2/task.loc.json +++ b/Tasks/AzureIoTEdgeV2/task.loc.json @@ -13,10 +13,11 @@ "author": "Microsoft Corporation", "version": { "Major": 2, - "Minor": 1, - "Patch": 5 + "Minor": 2, + "Patch": 0 }, - "preview": true, + "preview": false, + "showEnvironmentVariables": true, "instanceNameFormat": "ms-resource:loc.instanceNameFormat", "groups": [ { @@ -51,7 +52,7 @@ "name": "deploymentFilePath", "type": "filePath", "label": "ms-resource:loc.input.label.deploymentFilePath", - "defaultValue": "$(System.DefaultWorkingDirectory)/**/*.json", + "defaultValue": "$(System.DefaultWorkingDirectory)/config/deployment.json", "required": true, "visibleRule": "action == Deploy to IoT Edge devices", "helpMarkDown": "ms-resource:loc.input.help.deploymentFilePath" @@ -200,6 +201,15 @@ }, "helpMarkDown": "ms-resource:loc.input.help.fillRegistryCredential" }, + { + "name": "deploymentManifestOutputPath", + "type": "filePath", + "label": "ms-resource:loc.input.label.deploymentManifestOutputPath", + "defaultValue": "$(System.DefaultWorkingDirectory)/config/deployment.json", + "visibleRule": "action == Generate deployment manifest", + "required": true, + "helpMarkDown": "ms-resource:loc.input.help.deploymentManifestOutputPath" + }, { "name": "bypassModules", "type": "string", @@ -247,7 +257,7 @@ "DependencyInstallSuccess": "ms-resource:loc.messages.DependencyInstallSuccess", "DependencyInstallFail": "ms-resource:loc.messages.DependencyInstallFail", "TemplateFileInvalid": "ms-resource:loc.messages.TemplateFileInvalid", - "ContainerRegistryInvalid": "ms-resource:loc.messages.ContainerRegistryInvalid", + "InvalidContainerRegistry": "ms-resource:loc.messages.InvalidContainerRegistry", "DeploymentFileNotFound": "ms-resource:loc.messages.DeploymentFileNotFound", "ValidDeploymentFileNotFound": "ms-resource:loc.messages.ValidDeploymentFileNotFound", "AzureSdkNotFound": "ms-resource:loc.messages.AzureSdkNotFound", @@ -256,7 +266,9 @@ "InvalidRegistryCredentialWarning": "ms-resource:loc.messages.InvalidRegistryCredentialWarning", "CheckModuleImageExistenceError": "ms-resource:loc.messages.CheckModuleImageExistenceError", "StartGenerateDeploymentManifest": "ms-resource:loc.messages.StartGenerateDeploymentManifest", - "FinishGenerateDeploymentManifest": "ms-resource:loc.messages.FinishGenerateDeploymentManifest" + "FinishGenerateDeploymentManifest": "ms-resource:loc.messages.FinishGenerateDeploymentManifest", + "LoginRegistrySucess": "ms-resource:loc.messages.LoginRegistrySucess", + "SkipSettingEnvironmentVariableForSecret": "ms-resource:loc.messages.SkipSettingEnvironmentVariableForSecret" }, "OutputVariables": [ { diff --git a/Tasks/AzureIoTEdgeV2/telemetry.ts b/Tasks/AzureIoTEdgeV2/telemetry.ts index bf4031025368..3ba7201d3c8c 100644 --- a/Tasks/AzureIoTEdgeV2/telemetry.ts +++ b/Tasks/AzureIoTEdgeV2/telemetry.ts @@ -1,7 +1,7 @@ import * as appInsights from 'applicationinsights'; const metadata = { id: 'iot-edge-build-deploy', - version: '2.0.1', + version: '2.2.0', publisher: 'vsc-iot', } diff --git a/Tasks/AzureIoTEdgeV2/util.ts b/Tasks/AzureIoTEdgeV2/util.ts index a20c3be8a37d..d5af5b270c6f 100644 --- a/Tasks/AzureIoTEdgeV2/util.ts +++ b/Tasks/AzureIoTEdgeV2/util.ts @@ -89,7 +89,8 @@ export default class Util { cmds = [ { path: `sudo`, arg: `apt-get update`, execOption: Constants.execSyncSilentOption }, { path: `sudo`, arg: `apt-get install -y python-setuptools`, execOption: Constants.execSyncSilentOption }, - { path: `sudo`, arg: `pip install ${Constants.iotedgedev}==${version}`, execOption: Constants.execSyncSilentOption }, + { path: `sudo`, arg: `pip install --upgrade cryptography`, execOption: Constants.execSyncSilentOption}, + { path: `sudo`, arg: `pip install ${Constants.iotedgedev}~=${version}`, execOption: Constants.execSyncSilentOption }, ] } else if (tl.osType() === Constants.osTypeWindows) { cmds = [ @@ -100,6 +101,7 @@ export default class Util { try { for (let cmd of cmds) { let result = tl.execSync(cmd.path, cmd.arg, cmd.execOption); + tl.debug(result.stdout); if (result.code !== 0) { tl.debug(result.stderr); } @@ -110,9 +112,11 @@ export default class Util { } let result = tl.execSync(`${Constants.iotedgedev}`, `--version`, Constants.execSyncSilentOption); + tl.debug(result.stdout); if (result.code === 0) { console.log(tl.loc('DependencyInstallSuccess', Constants.iotedgedev, result.stdout.substring(result.stdout.indexOf("version")))); } else { + tl.error(result.stderr); throw Error(tl.loc('DependencyInstallFail', Constants.iotedgedev)); } } @@ -204,7 +208,7 @@ export default class Util { } public static normalizeDeploymentId(id: string): string { - if(id.length > 128) { + if (id.length > 128) { id = id.substring(0, 128); } id = id.toLowerCase(); @@ -221,4 +225,28 @@ export default class Util { } return true; } + + public static setCliVarialbe(envList: NodeJS.ProcessEnv, envName: string, envValue: string): void { + if (envList[envName]) { + tl.debug(`Use parameters from task config and ignore existing variable ${envName}.`) + } + tl.debug(`Setting the value of CLI variable ${envName}`); + envList[envName] = envValue; + } + + public static populateSecretToEnvironmentVariable(envList: NodeJS.ProcessEnv){ + let tlVariables = tl.getVariables(); + for (let v of tlVariables) { + // The variables in VSTS build contains dot, need to convert to underscore. + if (v.secret){ + let envName = v.name.replace('.', '_').toUpperCase(); + tl.debug(`Setting environment varialbe ${envName} to the value of secret: ${v.name}`); + if (!envList[envName]) { + envList[envName] = v.value; + } else { + tl.loc("SkipSettingEnvironmentVariableForSecret", envName, v.name); + } + } + } + } } \ No newline at end of file