WIP - UI Tests in nodejs

.gitignore

@@ -1,7 +1,8 @@
 .DS_Store
 .huskyrc.json
 out
-node_modules
+log.log
+**/node_modules
 *.pyc
 *.vsix
 **/.vscode/.ropeproject/**
@@ -14,9 +15,10 @@ npm-debug.log
 **/.mypy_cache/**
 !yarn.lock
 coverage/
-.vscode-test/**
-.vscode test/**
-.vscode-smoke/**
+cucumber-report.json
+**/.vscode-test/**
+**/.vscode test/**
+**/.vscode-smoke/**
 **/.venv*/
 port.txt
 precommit.hook
@@ -25,11 +27,14 @@ pythonFiles/lib/**
 debug_coverage*/**
 languageServer/**
 languageServer.*/**
+!uitests/features/languageServer/**
+!uitests/src/languageServer/**
 bin/**
 obj/**
 .pytest_cache
 tmp/**
 .python-version
 .vs/
 test-results.xml
+uitests/out/**
 !build/

.prettierrc.js

@@ -0,0 +1,6 @@
+module.exports = {
+    semi: true,
+    singleQuote: true,
+    printWidth: 180,
+    tabWidth: 4
+};

.vscode/extensions.json

@@ -3,6 +3,7 @@
     // for the documentation about the extensions.json format
     "recommendations": [
         "ms-vscode.vscode-typescript-tslint-plugin",
-        "editorconfig.editorconfig"
+        "editorconfig.editorconfig",
+        "esbenp.prettier-vscode"
     ]
 }

.vscode/settings.json

@@ -2,16 +2,22 @@
 {
     "files.exclude": {
         "out": true, // set this to true to hide the "out" folder with the compiled JS files
+        "uitests/out": true,
         "**/*.pyc": true,
         ".nyc_output": true,
         "obj": true,
         "bin": true,
         "**/__pycache__": true,
+        "**/node_modules": true,
+        ".vscode-test": false,
+        ".vscode test": false,
         "**/.mypy_cache/**": true,
         "**/.ropeproject/**": true
     },
     "search.exclude": {
         "out": true, // set this to false to include "out" folder in search results
+        "uitests/out": true,
+        "**/node_modules": true,
         "coverage": true,
         "languageServer*/**": true,
         ".vscode-test": true,

.vscodeignore

@@ -18,6 +18,7 @@ CODING_STANDARDS.md
 CONTRIBUTING.md
 CONTRIBUTING - LANGUAGE SERVER.md
 coverconfig.json
+cucumber-report.json
 gulpfile.js
 package.datascience-ui.dependencies.json
 package-lock.json

build/ci/templates/jobs/merge_upload_uitest_report.yml

@@ -0,0 +1,46 @@
+# Overview:
+# Generic jobs template to compile and build extension
+
+jobs:
+- job: UI_Test_Report
+  pool:
+    vmImage: "macos-latest"
+  steps:
+    - template: ../steps/initialization.yml
+      parameters:
+        workingDirectory: $(Build.SourcesDirectory)/uitests
+        compile: 'true'
+
+    - bash: mkdir -p reports
+      workingDirectory: $(Build.SourcesDirectory)/uitests
+      displayName: "Create Reports Directory"
+
+    - task: DownloadBuildArtifacts@0
+      inputs:
+          buildType: "current"
+          allowPartiallySucceededBuilds: true
+          downloadType: "Specific"
+          itemPattern: "**/.vscode test/reports/cucumber_report_*.json"
+          downloadPath: "$(Build.SourcesDirectory)/uitests/reports"
+      displayName: "Restore Cucumber Reports"
+      condition: always()
+
+    - bash: node ./out/index.js report --jsonDir=./reports --htmlOutput=./reports
+      workingDirectory: $(Build.SourcesDirectory)/uitests
+      displayName: "Merge and generate report"
+      condition: always()
+
+    - task: CopyFiles@2
+      inputs:
+          sourceFolder: $(Build.SourcesDirectory)/uitests/reports
+          contents: "**"
+          targetFolder: $(Build.ArtifactStagingDirectory)
+      displayName: "Copy Report"
+      condition: always()
+
+    - task: PublishBuildArtifacts@1
+      inputs:
+          pathtoPublish: $(Build.ArtifactStagingDirectory)
+          artifactName: UIReport
+      displayName: "Publish Report"
+      condition: always()

build/ci/templates/jobs/uitest.yml

@@ -25,18 +25,18 @@
 #   parameters:
 #     jobs:
 #     - test: "Smoke"
-#       tags: "--tags=@smoke"
+#       tags: "@smoke"
 #     - test: "Test"
-#       tags: "--tags=@test"
+#       tags: "@test"
 #     - test: "Terminal"
-#       tags: "--tags=@terminal"
+#       tags: "@terminal"
 #   ```
 #   Based on this sample, we're running 3 tests with the names `Smoke`, `Test`, and `Terminal`.
 #   The tags inside each test contains the arguments that needs to be passd into behave.
 #   I.e. we're only testing BDD tests that contain the tag `@smoke`, `@test` & `@terminal` (as separate jobs).
 #   Please pass in just the `tags` arguments.
 #   Multiple tag values can be passed in as follows:
-#       tags: "--tags=@debug --tags=@remote"
+#       tags: "@debug and @remote"
 #   More information on --tags argument for behave can be found here:
 #       * https://behave.readthedocs.io/en/latest/tutorial.html#controlling-things-with-tags
 #       * https://behave.readthedocs.io/en/latest/tag_expressions.html
@@ -52,7 +52,7 @@
 #   vscodeChannels: ['stable']
 #   jobs:
 #   - test: "Smoke"
-#     tags: "--tags=@smoke"
+#     tags: "@smoke"
 #     ignorePythonVersions: "3.6,3.5"
 #   ```
 #   Based on this sample, we're running 1 test with the name `Smoke`.
@@ -73,7 +73,7 @@
 #   vscodeChannels: ['stable']
 #   jobs:
 #   - test: "Smoke"
-#     tags: "--tags=@smoke"
+#     tags: "@smoke"
 #     ignorePythonVersions: "3.6,3.5"
 #   ```
 #   Based on this sample, we're running 1 test with the name `Smoke`.
@@ -93,36 +93,33 @@ parameters:
   vscodeChannels: ['stable', 'insider']
   pythonVersions: [
       {
-          "version": "3.7",
+          "version": "3.7.4",
           "displayName": "37",
-          "excludeTags": "--tags=~@python3.6 --tags=~@python3.5 --tags=~@python2"
+          "excludeTags": "not @python3.6 and not @python3.5 and not @python2"
       },
       {
           "version": "3.6",
           "displayName": "36",
-          "excludeTags": "--tags=~@python3.7 --tags=~@python3.5 --tags=~@python2"
+          "excludeTags": "not @python3.7 and not @python3.5 and not @python2 and not @noNeedToTestInAllPython"
       },
       {
           "version": "3.5",
           "displayName": "35",
-          "excludeTags": "--tags=~@python3.7 --tags=~@python3.6 --tags=~@python2"
+          "excludeTags": "not @python3.7 and not @python3.6 and not @python2 and not @noNeedToTestInAllPython"
       },
       {
           "version": "2.7",
           "displayName": "27",
-          "excludeTags": "--tags=~@python3.7 --tags=~@python3.5 --tags=~@python3"
+          "excludeTags": "not @python3.7 and not @python3.5 and not @python3 and not @noNeedToTestInAllPython"
       }
     ]
 
 
 jobs:
 - job: UITest
-  dependsOn:
-    - Compile
-    - Build
   # Remember, some tests can take easily an hour (the `tests` features take just around 1 hour).
   timeoutInMinutes: 90
-  # Build our matrix (permutations of all environments & tests).
+  # Build our matrix (permutations of VS Code + Tests + Pyhton + OS).
   strategy:
     matrix:
       ${{ each channel in parameters.vscodeChannels }}:
@@ -134,24 +131,24 @@ jobs:
                     PythonVersion: ${{ py.version }}
                     VMImageName: "macos-latest"
                     VSCodeChannel: ${{ channel }}
-                    Tags: ${{ format('{0} {1} --tags=~@win --tags=~@linux', job.tags, py.excludeTags) }}
+                    Tags: ${{ format('{0} and {1} and not @win and not @linux', job.tags, py.excludeTags) }}
 
               ${{ if not(contains(coalesce(job.ignoreOperatingSystems, ''), 'win')) }}:
                 ${{ format('Win{2}{0}{1}', py.displayName, job.test, channel) }}:
                     PythonVersion: ${{ py.version }}
                     VSCodeChannel: ${{ channel }}
                     VMImageName: "vs2017-win2016"
-                    Tags: ${{ format('{0} {1} --tags=~@mac --tags=~@linux', job.tags, py.excludeTags) }}
+                    Tags: ${{ format('{0} and {1} and not @mac and not @linux', job.tags, py.excludeTags) }}
 
               ${{ if not(contains(coalesce(job.ignoreOperatingSystems, ''), 'linux')) }}:
                 ${{ format('Linux{2}{0}{1}', py.displayName, job.test, channel) }}:
                     PythonVersion: ${{ py.version }}
                     VSCodeChannel: ${{ channel }}
                     VMImageName: "ubuntu-latest"
-                    Tags: ${{ format('{0} {1} --tags=~@mac --tags=~@win', job.tags, py.excludeTags) }}
+                    Tags: ${{ format('{0} and {1} and not @mac and not @win', job.tags, py.excludeTags) }}
 
   pool:
     vmImage: $(VMImageName)
 
   steps:
-    - template: uitest_phases.yml
+    - template: ../steps/uitest.yml

build/ci/templates/steps/initialization.yml

@@ -17,7 +17,6 @@ parameters:
   PythonVersion: ''
   compile: 'true'
   sqlite: 'false'
-  installVSCEorNPX: 'true'
 
 steps:
     - bash: |
@@ -85,7 +84,6 @@ steps:
           npm install -g vsce
           npm install -g npx
       displayName: "Install vsce & npx"
-      condition: and(succeeded(), eq('${{ parameters.installVSCEorNPX }}', 'true'))
 
     - bash: npx tsc -p ./
       displayName: "compile (npx tsc -p ./)"

build/ci/templates/steps/initialization.yml~HEAD

@@ -0,0 +1,113 @@
+# -----------------------------------------------------------------------------------------------------------------------------
+# Overview:
+# -----------------------------------------------------------------------------------------------------------------------------
+# Set of common steps required when initializing a job.
+#
+# -----------------------------------------------------------------------------------------------------------------------------
+# Variables
+# -----------------------------------------------------------------------------------------------------------------------------
+# 1. workingDirectory
+#   Mandatory
+#   Working directory.
+# 2. PythonVersion
+#   Python version to be used.
+#   If not provided, python will not be setup on CI.
+parameters:
+  workingDirectory: ''
+  PythonVersion: ''
+  compile: 'true'
+  sqlite: 'false'
+  installVSCEorNPX: 'true'
+
+steps:
+    - bash: |
+          printenv
+      displayName: "Show all env vars"
+      condition: eq(variables['system.debug'], 'true')
+
+    - task: NodeTool@0
+      displayName: "Use Node $(NodeVersion)"
+      inputs:
+          versionSpec: $(NodeVersion)
+
+    - task: UsePythonVersion@0
+      displayName: "Use Python ${{ parameters.PythonVersion }}"
+      condition: and(succeeded(), not(eq('${{ parameters.PythonVersion }}', '')))
+      inputs:
+          versionSpec: ${{ parameters.PythonVersion }}
+
+    # Install the a version of python that works with sqlite3 until this bug is addressed
+    # https://mseng.visualstudio.com/AzureDevOps/_workitems/edit/1535830
+    #
+    # This task will only run if variable `NeedsPythonFunctionalReqs` is true.
+    - bash: |
+        sudo apt-get install libsqlite3-dev
+        version=$(python -V 2>&1 | grep -Po '(?<=Python )(.+)')
+        wget https://www.python.org/ftp/python/$version/Python-$version.tar.xz
+        tar xvf Python-$version.tar.xz
+        cd Python-$version
+        ./configure --enable-loadable-sqlite-extensions --with-ensurepip=install --prefix=$HOME/py-$version
+        make
+        sudo make install
+        sudo chmod -R 777 $HOME/py-$version
+        export PATH=$HOME/py-$version/bin:$PATH
+        sudo ln -s $HOME/py-$version/bin/python3 $HOME/py-$version/bin/python
+        echo '##vso[task.prependpath]'$HOME/py-$version/bin
+      displayName: 'Setup python to run with sqlite on 3.*'
+      condition: and(succeeded(), eq('${{ parameters.sqlite }}', 'true'), not(eq('${{ parameters.PythonVersion }}', '')), not(eq('${{ parameters.PythonVersion }}', '2.7')), eq(variables['Agent.Os'], 'Linux'))
+
+    - task: Npm@1
+      displayName: "Use NPM $(NpmVersion)"
+      inputs:
+          command: custom
+          verbose: true
+          customCommand: "install -g npm@$(NpmVersion)"
+
+    - task: Npm@1
+      displayName: "npm ci"
+      inputs:
+          workingDir: ${{ parameters.workingDirectory }}
+          command: custom
+          verbose: true
+          customCommand: ci
+
+    # On Mac, the command `node` doesn't always point to the current node version.
+    # Debugger tests use the variable process.env.NODE_PATH
+    - script: |
+          export NODE_PATH=`which node`
+          echo $NODE_PATH
+          echo '##vso[task.setvariable variable=NODE_PATH]'$NODE_PATH
+      displayName: "Setup NODE_PATH for extension (Debugger Tests)"
+      condition: and(succeeded(), eq(variables['agent.os'], 'Darwin'))
+
+    # We need these almost always.
+    - bash: |
+          npm install -g vsce
+          npm install -g npx
+      displayName: "Install vsce & npx"
+      condition: and(succeeded(), eq('${{ parameters.installVSCEorNPX }}', 'true'))
+
+    - bash: npx tsc -p ./
+      displayName: "compile (npx tsc -p ./)"
+      workingDirectory: ${{ parameters.workingDirectory }}
+      condition: and(succeeded(), eq('${{ parameters.compile }}', 'true'))
+
+    # https://code.visualstudio.com/api/working-with-extensions/continuous-integration#azure-pipelines
+    - bash: |
+          set -e
+          /usr/bin/Xvfb :10 -ac >> /tmp/Xvfb.out 2>&1 &
+          disown -ar
+      displayName: 'Start Display Server (xvfb) to launch VS Code)'
+      condition: and(succeeded(), eq(variables['Agent.Os'], 'Linux'))
+
+    # # Show all versions installed/available on PATH if in verbose mode.
+    # # Example command line (windows pwsh):
+    # # > Write-Host Node ver: $(& node -v) NPM Ver: $(& npm -v) Python ver: $(& python --version)"
+    # - bash: |
+    #     echo AVAILABLE DEPENDENCY VERSIONS
+    #     echo Node Version = `node -v`
+    #     echo NPM Version = `npm -v`
+    #     echo Python Version = `python --version`
+    #     echo Gulp Version = `gulp --version`
+    #   condition: and(succeeded(), eq(variables['system.debug'], 'true'))
+    #   displayName: Show Dependency Versions