An overhaul of Nim's CI

azure-pipelines.yml

@@ -14,14 +14,6 @@ jobs:
 
   strategy:
     matrix:
-      Linux_amd64:
-        vmImage: 'ubuntu-16.04'
-        CPU: amd64
-      # Linux_i386:
-      #   # bug #17325: fails on 'ubuntu-16.04' because it now errors with:
-      #   # g++-multilib : Depends: gcc-multilib (>= 4:5.3.1-1ubuntu1) but it is not going to be installed
-      #   vmImage: 'ubuntu-18.04'
-      #   CPU: i386
       OSX_amd64:
         vmImage: 'macOS-10.15'
         CPU: amd64
@@ -50,145 +42,29 @@ jobs:
     clean: all
 
   steps:
-    - bash: git config --global core.autocrlf false
-      displayName: 'Disable auto conversion to CRLF by git (Windows-only)'
-      condition: eq(variables['Agent.OS'], 'Windows_NT')
-
-    - checkout: self
-      fetchDepth: 1
-
-    - bash: git clone --depth 1 https://github.com/nim-lang/csources
-      displayName: 'Checkout Nim csources'
-
-    - task: NodeTool@0
-      inputs:
-        versionSpec: '12.x'
-      displayName: 'Install node.js 12.x'
-
-    - bash: |
-        set -e
-        . ci/funs.sh
-        echo_run sudo apt-fast update -qq
-        DEBIAN_FRONTEND='noninteractive' \
-          echo_run sudo apt-fast install --no-install-recommends -yq \
-            libcurl4-openssl-dev libgc-dev libsdl1.2-dev libsfml-dev valgrind libc6-dbg
-      displayName: 'Install dependencies (amd64 Linux)'
-      condition: and(eq(variables['Agent.OS'], 'Linux'), eq(variables['CPU'], 'amd64'))
-
-    - bash: |
-        set -e
-        . ci/funs.sh
-        echo_run sudo dpkg --add-architecture i386
-        # Downgrade llvm:
-        # - llvm has to be downgraded to have 32bit version installed for sfml.
-
-        cat << EOF | sudo tee /etc/apt/preferences.d/pin-to-rel
-        Package: libllvm6.0
-        Pin: origin "azure.archive.ubuntu.com"
-        Pin-Priority: 1001
-        EOF
-
-        # echo_run sudo apt-fast update -qq
-        echo_run sudo apt-fast update -qq || echo "failed, see bug #17343"
-        # `:i386` (e.g. in `libffi-dev:i386`) is needed otherwise you may get:
-        # `could not load: libffi.so` during dynamic loading.
-        DEBIAN_FRONTEND='noninteractive' \
-          echo_run sudo apt-fast install --no-install-recommends --allow-downgrades -yq \
-            g++-multilib gcc-multilib libcurl4-openssl-dev:i386 libgc-dev:i386 \
-            libsdl1.2-dev:i386 libsfml-dev:i386 libglib2.0-dev:i386 libffi-dev:i386
-
-        cat << EOF > bin/gcc
-        #!/bin/bash
-
-        exec $(which gcc) -m32 "\$@"
-        EOF
-
-        cat << EOF > bin/g++
-        #!/bin/bash
-
-        exec $(which g++) -m32 "\$@"
-        EOF
-
-        echo_run chmod 755 bin/gcc
-        echo_run chmod 755 bin/g++
+    - template: ci/azure/ci-steps.yml
 
-      displayName: 'Install dependencies (i386 Linux)'
-      condition: and(eq(variables['Agent.OS'], 'Linux'), eq(variables['CPU'], 'i386'))
+- job: linux
 
-    - bash: brew install boehmgc make sfml
-      displayName: 'Install dependencies (OSX)'
-      condition: eq(variables['Agent.OS'], 'Darwin')
-
-    - bash: |
-        set -e
-        . ci/funs.sh
-        echo_run mkdir dist
-        echo_run curl -L https://nim-lang.org/download/mingw64.7z -o dist/mingw64.7z
-        echo_run curl -L https://nim-lang.org/download/dlls.zip -o dist/dlls.zip
-        echo_run 7z x dist/mingw64.7z -odist
-        echo_run 7z x dist/dlls.zip -obin
-        echo_run echo '##vso[task.prependpath]$(System.DefaultWorkingDirectory)/dist/mingw64/bin'
-
-      displayName: 'Install dependencies (Windows)'
-      condition: eq(variables['Agent.OS'], 'Windows_NT')
-
-    - bash: echo '##vso[task.prependpath]$(System.DefaultWorkingDirectory)/bin'
-      displayName: 'Add build binaries to PATH'
-
-    - bash: |
-        set -e
-        . ci/funs.sh
-        echo_run echo 'PATH:' "$PATH"
-        echo_run echo '##[section]gcc version'
-        echo_run gcc -v
-        echo_run echo '##[section]nodejs version'
-        echo_run node -v
-        echo_run echo '##[section]make version'
-        echo_run make -v
-      displayName: 'System information'
-
-    - bash: echo '##vso[task.setvariable variable=csources_version]'"$(git -C csources rev-parse HEAD)"
-      displayName: 'Get csources version'
-
-    - task: Cache@2
-      inputs:
-        key: 'csources | "$(Agent.OS)" | $(CPU) | $(csources_version)'
-        path: csources/bin
-      displayName: 'Restore built csources'
-
-    - bash: |
-        set -e
-        . ci/funs.sh
-        ncpu=
-        ext=
-        case '$(Agent.OS)' in
-        'Linux')
-          ncpu=$(nproc)
-          ;;
-        'Darwin')
-          ncpu=$(sysctl -n hw.ncpu)
-          ;;
-        'Windows_NT')
-          ncpu=$NUMBER_OF_PROCESSORS
-          ext=.exe
-          ;;
-        esac
-        [[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=1
+  timeoutInMinutes: 90 # default `60` led to lots of cancelled jobs; use 0 for unlimited (may be undesirable)
 
-        if [[ -x csources/bin/nim$ext ]]; then
-          echo_run echo "Found cached compiler, skipping build"
-        else
-          echo_run make -C csources -j $ncpu CC=gcc ucpu=$(CPU) koch=no
-        fi
+  strategy:
+    matrix:
+      amd64:
+        containerImage: ghcr.io/alaviss/nim-ci:sha-aa56233@sha256:43dbe085d041c95b5d39bcbc5e0c4e05d922d407ac8a8938d33064402a5a5f22
+        CPU: amd64
+      i386:
+        containerImage: ghcr.io/alaviss/nim-ci:sha-aa56233@sha256:7f6d204d43e213ff8621e9f0c9f2b788858b9f3788fed95ac62e1c821baa4476
+        CPU: i386
 
-        echo_run cp csources/bin/nim$ext bin
-      displayName: 'Build 1-stage compiler from csources'
+  pool:
+    vmImage: ubuntu-latest
 
-    - bash: nim c koch
-      displayName: 'Build koch'
+  container:
+    image: $(containerImage)
+    options: --init --sysctl net.ipv6.conf.all.disable_ipv6=0
 
-      # set result to omit the "bash exited with error code '1'" message
-    - bash: ./koch runCI || echo '##vso[task.complete result=Failed]'
-      displayName: 'Run CI'
-      env:
-        SYSTEM_ACCESSTOKEN: $(System.AccessToken)
+  steps:
+    - template: ci/azure/ci-steps.yml
+      parameters:
+        installNode: false

ci/azure/ci-steps.yml

@@ -0,0 +1,105 @@
+parameters:
+  - name: installNode
+    type: boolean
+    default: true
+
+steps:
+  - bash: |
+      sysctl net.ipv6.conf.all.disable_ipv6
+      sysctl net.ipv6.conf.lo.disable_ipv6
+    displayName: Inspect container environment
+    condition: eq(variables['Agent.OS'], 'Linux')
+
+  - bash: git config --global core.autocrlf false
+    displayName: 'Disable auto conversion to CRLF by git (Windows-only)'
+    condition: eq(variables['Agent.OS'], 'Windows_NT')
+
+  - checkout: self
+    fetchDepth: 1
+
+  - bash: git clone --depth 1 https://github.com/nim-lang/csources
+    displayName: 'Checkout Nim csources'
+
+  - ${{ if eq(parameters.installNode, true) }}:
+    - task: NodeTool@0
+      inputs:
+        versionSpec: '12.x'
+      displayName: 'Install node.js 12.x'
+
+  - bash: brew install boehmgc make sfml
+    displayName: 'Install dependencies (OSX)'
+    condition: eq(variables['Agent.OS'], 'Darwin')
+
+  - bash: |
+      set -e
+      . ci/funs.sh
+      echo_run mkdir dist
+      echo_run curl -L https://nim-lang.org/download/mingw64.7z -o dist/mingw64.7z
+      echo_run curl -L https://nim-lang.org/download/dlls.zip -o dist/dlls.zip
+      echo_run 7z x dist/mingw64.7z -odist
+      echo_run 7z x dist/dlls.zip -obin
+      echo_run echo '##vso[task.prependpath]$(System.DefaultWorkingDirectory)/dist/mingw64/bin'
+
+    displayName: 'Install dependencies (Windows)'
+    condition: eq(variables['Agent.OS'], 'Windows_NT')
+
+  - bash: echo '##vso[task.prependpath]$(System.DefaultWorkingDirectory)/bin'
+    displayName: 'Add build binaries to PATH'
+
+  - bash: |
+      set -e
+      . ci/funs.sh
+      echo_run echo 'PATH:' "$PATH"
+      echo_run echo '##[section]gcc version'
+      echo_run gcc -v
+      echo_run echo '##[section]nodejs version'
+      echo_run node -v
+      echo_run echo '##[section]make version'
+      echo_run make -v
+    displayName: 'System information'
+
+  - bash: echo '##vso[task.setvariable variable=csources_version]'"$(git -C csources rev-parse HEAD)"
+    displayName: 'Get csources version'
+
+  - task: Cache@2
+    inputs:
+      key: 'csources | "$(Agent.OS)" | $(CPU) | $(csources_version)'
+      path: csources/bin
+    displayName: 'Restore built csources'
+
+  - bash: |
+      set -e
+      . ci/funs.sh
+      ncpu=
+      ext=
+      case '$(Agent.OS)' in
+      'Linux')
+        ncpu=$(nproc)
+        ;;
+      'Darwin')
+        ncpu=$(sysctl -n hw.ncpu)
+        ;;
+      'Windows_NT')
+        ncpu=$NUMBER_OF_PROCESSORS
+        ext=.exe
+        ;;
+      esac
+      [[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=1
+
+      if [[ -x csources/bin/nim$ext ]]; then
+        echo_run echo "Found cached compiler, skipping build"
+      else
+        echo_run make -C csources -j $ncpu CC=gcc ucpu=$(CPU) koch=no
+      fi
+
+      echo_run cp csources/bin/nim$ext bin
+    displayName: 'Build 1-stage compiler from csources'
+
+  - bash: nim c koch
+    displayName: 'Build koch'
+
+    # set result to omit the "bash exited with error code '1'" message
+  - bash: ./koch runCI || echo '##vso[task.complete result=Failed]'
+    displayName: 'Run CI'
+    env:
+      SYSTEM_ACCESSTOKEN: $(System.AccessToken)

koch.nim

@@ -552,7 +552,13 @@ proc runCI(cmd: string) =
     # main bottleneck here
     # xxx: even though this is the main bottleneck, we could speedup the rest via batching with `--batch`.
     # BUG: with initOptParser, `--batch:'' all` interprets `all` as the argument of --batch, pending bug #14343
-    execFold("Run tester", "nim c -r -d:nimCoroutines --putenv:NIM_TESTAMENT_REMOTE_NETWORKING:1 -d:nimStrictMode testament/testament $# all -d:nimCoroutines" % batchParam)
+    const Targets =
+      # Disable JS tests on i?86 systems since node.js has been deprecated there.
+      when defined(i386) and defined(linux):
+        "--targets:\"c cpp\""
+      else:
+        "--targets:\"c cpp js\""
+    execFold("Run tester", "nim c -r -d:nimCoroutines --putenv:NIM_TESTAMENT_REMOTE_NETWORKING:1 -d:nimStrictMode testament/testament $1 $2 all -d:nimCoroutines" % [batchParam, Targets])
 
     block CT_FFI:
       when defined(posix): # windows can be handled in future PR's

tests/exception/t13115.nim

@@ -13,10 +13,10 @@ else:
     const nim = getCurrentCompilerExe()
     const file = currentSourcePath
     for b in "c js cpp".split:
-      when defined(openbsd):
+      when defined(openbsd) or (defined(linux) and defined(i386)):
         if b == "js":
           # xxx bug: pending #13115
-          # remove special case once nodejs updated >= 12.16.2
+          # remove special case for openbsd once nodejs updated >= 12.16.2
           # refs https://github.com/nim-lang/Nim/pull/16167#issuecomment-738270751
           continue
 

tests/misc/trunner.nim

@@ -28,6 +28,8 @@ const
   mode = querySetting(backend)
   nimcache = buildDir / "nimcacheTrunner"
     # instead of `querySetting(nimcacheDir)`, avoids stomping on other parallel tests
+  noJs = defined(linux) and defined(i386)
+    # configurations where JS tests can't run
 
 proc runCmd(file, options = ""): auto =
   let fileabs = testsDir / file.unixToNativePath
@@ -194,7 +196,12 @@ sub/mmain.idx""", context
     # test for https://github.com/nim-lang/Nim/issues/13129
     # test for https://github.com/nim-lang/Nim/issues/13891
     let file = testsDir / "nimdoc/m13129.nim"
-    for backend in fmt"{mode} js".split:
+    const backends =
+      when not noJs:
+        fmt"{mode} js"
+      else:
+        mode
+    for backend in backends.split:
       # pending #14343 this fails on windows: --doccmd:"-d:m13129Foo2 --hints:off"
       let cmd = fmt"""{nim} doc -b:{backend} --nimcache:{nimcache} -d:m13129Foo1 "--doccmd:-d:m13129Foo2 --hints:off" --usenimcache --hints:off {file}"""
       check execCmdEx(cmd) == (&"ok1:{backend}\nok2: backend: {backend}\n", 0)
@@ -212,8 +219,9 @@ sub/mmain.idx""", context
     cmd = fmt"{nim} check -b:c -b:cpp --hints:off --nimcache:{nimcache} {file}"
     check execCmdEx(cmd) == ("", 0)
     # issue https://github.com/timotheecour/Nim/issues/175
-    cmd = fmt"{nim} c -b:js -b:cpp --hints:off --nimcache:{nimcache} {file}"
-    check execCmdEx(cmd) == ("", 0)
+    when not noJs:
+      cmd = fmt"{nim} c -b:js -b:cpp --hints:off --nimcache:{nimcache} {file}"
+      check execCmdEx(cmd) == ("", 0)
 
   block: # some importc tests
     # issue #14314
@@ -242,7 +250,8 @@ tests/newconfig/bar/mfoo.nims""".splitLines
     let opt = "--hints:off"
     check fmt"""{nim} {opt} --eval:"echo defined(nimscript)"""".execCmdEx == ("true\n", 0)
     check fmt"""{nim} r {opt} --eval:"echo defined(c)"""".execCmdEx == ("true\n", 0)
-    check fmt"""{nim} r -b:js {opt} --eval:"echo defined(js)"""".execCmdEx == ("true\n", 0)
+    when not noJs:
+      check fmt"""{nim} r -b:js {opt} --eval:"echo defined(js)"""".execCmdEx == ("true\n", 0)
 
   block: # `hintProcessing` dots should not interfere with `static: echo` + friends
     let cmd = fmt"""{nim} r {defaultHintsOff} --hint:processing -f --eval:"static: echo 1+1""""