From 6b3e1706c4cfa3565ce2c79214a1d341760e6661 Mon Sep 17 00:00:00 2001 From: Benoit Daloze Date: Thu, 16 Feb 2023 16:15:49 +0100 Subject: [PATCH] [GR-44299] Move global-common CI config from common.json to ci/common.jsonnet --- .github/workflows/main.yml | 9 +- ci.jsonnet | 4 +- ci/README.md | 12 + ci/ci_common/bench-common.libsonnet | 10 +- ci/ci_common/common-utils.libsonnet | 65 ----- ci/ci_common/common.jsonnet | 131 ++-------- ci/ci_common/run-spec-tools.libsonnet | 2 +- ci/common.jsonnet | 242 ++++++++++++++++++ common.json | 176 ++----------- .../ci/ci_common/benchmark-suites.libsonnet | 2 - compiler/ci/ci_common/gate.jsonnet | 2 +- compiler/mx.compiler/mx_compiler.py | 46 +--- .../graalvm/compiler/debug/DebugContext.java | 2 +- .../debug/StandardPathUtilitiesProvider.java | 4 +- espresso/ci/ci.jsonnet | 2 - espresso/ci/ci_common/common.jsonnet | 12 +- regex/ci/ci.jsonnet | 2 +- sdk/ci/ci.jsonnet | 4 +- substratevm/ci/ci.jsonnet | 4 +- substratevm/mx.substratevm/mx_substratevm.py | 23 -- .../oracle/svm/hosted/NativeImageOptions.java | 6 +- sulong/ci/ci_common/sulong-common.jsonnet | 15 +- tools/ci/ci.jsonnet | 10 +- truffle/ci/ci.jsonnet | 14 +- vm/ci/ci_common/common.jsonnet | 38 ++- vm/ci/ci_common/libgraal.jsonnet | 4 +- vm/ci/ci_includes/vm.jsonnet | 9 +- wasm/ci/ci.jsonnet | 12 +- wasm/ci/ci_common/common.jsonnet | 66 +---- 29 files changed, 376 insertions(+), 552 deletions(-) create mode 100644 ci/README.md create mode 100644 ci/common.jsonnet diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index fad2092c6e37..75971a94af7c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -161,8 +161,9 @@ jobs: if: ${{ env.MX_RUNS_STYLE == 'true' }} run: | sudo apt install python3-pip python-setuptools - jq -r '.deps.common.packages | to_entries[] | select(.key | startswith("pip:")) | (.key | split(":")[1]) + .value' common.json | - xargs sudo pip install + sudo pip install ninja_syntax $(jq -r '.pip.ninja_syntax' common.json) + sudo pip install lazy-object-proxy $(jq -r '.pip["lazy-object-proxy"]' common.json) + sudo pip install pylint $(jq -r '.pip.pylint' common.json) - name: Install additional pip packages if: ${{ matrix.env.PIP_PACKAGES != '' }} run: ${MX_PYTHON} -m pip install ${{ matrix.env.PIP_PACKAGES }} @@ -170,8 +171,8 @@ jobs: if: ${{ env.MX_RUNS_STYLE == 'true' }} run: | ECLIPSE_TAR=eclipse.tar.gz - ECLIPSE_ORG_VERSION=$(jq -r '.downloads.eclipse.eclipse_org.version' common.json) - ECLIPSE_ORG_TIMESTAMP=$(jq -r '.downloads.eclipse.eclipse_org.timestamp' common.json) + ECLIPSE_ORG_VERSION=$(jq -r '.eclipse.short_version' common.json) + ECLIPSE_ORG_TIMESTAMP=$(jq -r '.eclipse.timestamp' common.json) wget --no-verbose https://archive.eclipse.org/eclipse/downloads/drops4/R-${ECLIPSE_ORG_VERSION}-${ECLIPSE_ORG_TIMESTAMP}/eclipse-SDK-${ECLIPSE_ORG_VERSION}-linux-gtk-x86_64.tar.gz -O $ECLIPSE_TAR tar -xzf ${ECLIPSE_TAR} echo "ECLIPSE_EXE=${PWD}/eclipse/eclipse" >> $GITHUB_ENV diff --git a/ci.jsonnet b/ci.jsonnet index ca2280f9888c..35181fab0f0f 100644 --- a/ci.jsonnet +++ b/ci.jsonnet @@ -41,8 +41,8 @@ local visualizer = import 'visualizer/ci/ci.jsonnet'; local verify_ci = (import 'ci/ci_common/ci-check.libsonnet').verify_ci; { - # Ensure that entries in common.jsonnet can be resolved. - _checkCommon: (import 'ci/ci_common/common.jsonnet'), + # Ensure that non-hidden entries in ci/common.jsonnet and ci/ci_common/common.jsonnet can be resolved. + assert std.length(std.toString(import 'ci/ci_common/common.jsonnet')) > 0, ci_resources:: (import 'ci/ci_common/ci-resources.libsonnet'), overlay: graal_common.ci.overlay, specVersion: "3", diff --git a/ci/README.md b/ci/README.md new file mode 100644 index 000000000000..3c3ddc871b04 --- /dev/null +++ b/ci/README.md @@ -0,0 +1,12 @@ +# CI Shared Files + +## Files shared between many repositories + +* `common.json` +* `ci/common.jsonnet` + +## Files shared between graal and graal-enterprise + +* `graal-common.json` +* `ci/ci_common` +* `/ci/ci_common` in various suites diff --git a/ci/ci_common/bench-common.libsonnet b/ci/ci_common/bench-common.libsonnet index 7f3589ad658e..a2231605b74a 100644 --- a/ci/ci_common/bench-common.libsonnet +++ b/ci/ci_common/bench-common.libsonnet @@ -1,5 +1,5 @@ { - local common = import "ci/ci_common/common.jsonnet", + local common = import "../../ci/ci_common/common.jsonnet", local utils = import "common-utils.libsonnet", # benchmark job base with automatically generated name @@ -43,26 +43,26 @@ threads_per_node:: if self.is_numa then self.num_threads / std.length(self.numa_nodes) else self.num_threads, }, - x52:: common.linux + common.amd64 + self._bench_machine + { + x52:: common.linux_amd64 + self._bench_machine + { machine_name:: "x52", capabilities+: ["no_frequency_scaling", "tmpfs25g"], numa_nodes:: [0, 1], default_numa_node:: 0, num_threads:: 72 }, - x82:: common.linux + common.amd64 + self._bench_machine + { + x82:: common.linux_amd64 + self._bench_machine + { machine_name:: "x82", capabilities+: ["no_frequency_scaling", "tmpfs25g"], numa_nodes:: [0, 1], default_numa_node:: 0, num_threads:: 96 }, - xgene3:: common.linux + common.aarch64 + self._bench_machine + { + xgene3:: common.linux_aarch64 + self._bench_machine + { machine_name:: "xgene3", capabilities+: [], num_threads:: 32 }, - a12c:: common.linux + common.aarch64 + self._bench_machine + { + a12c:: common.linux_aarch64 + self._bench_machine + { machine_name:: "a12c", capabilities+: ["no_frequency_scaling", "tmpfs25g"], numa_nodes:: [0, 1], diff --git a/ci/ci_common/common-utils.libsonnet b/ci/ci_common/common-utils.libsonnet index 1624e349d98c..49459fe03ec5 100644 --- a/ci/ci_common/common-utils.libsonnet +++ b/ci/ci_common/common-utils.libsonnet @@ -1,69 +1,4 @@ { - # composable: make an object composable - # - # When composing objects with `+`, the RHS overrides the LHS when fields collide, - # unless the RHS defines fields with `+:`. This is especially an issue when - # importing nested objects from JSON, where all fields are implicitly defined - # with `:`. To solve this, `composable` essentially turns `:` fields into `+:`. - # - # See the following example: - # - # input.jsonnet - # - # { - # composable:: ..., - # obj1:: { - # key1: { - # foo: "foo", - # }, - # key2: "key2", - # }, - # obj2:: { - # key1: { - # baz : "baz", - # }, - # key3: "key3", - # }, - # res1: self.obj1 + self.obj2, - # res2: self.composable(self.obj1) + self.composable(self.obj2), - # } - # - # - # Output of jsonnet input.jsonnet - # - # { - # "res1": { - # "key1": { - # "baz": "baz" - # }, - # "key2": "key2", - # "key3": "key3" - # }, - # "res2": { - # "key1": { - # "baz": "baz", - # "foo": "foo" - # }, - # "key2": "key2", - # "key3": "key3" - # } - # } - # - # Note the missing `res1.key1.foo`! - # - local _composable(o) = - std.foldl(function(obj, key) - obj + - if std.type(o[key]) == "object" then - { [key] +: _composable(o[key]) } - else - { [key] : o[key] }, - std.objectFields(o), - {} - ), - # exported name - composable(o) :: _composable(o), - # prefixes the given number with 'jdk' prefixed_jdk(jdk_version):: if jdk_version == null || std.length(std.toString(jdk_version)) == 0 then diff --git a/ci/ci_common/common.jsonnet b/ci/ci_common/common.jsonnet index 4099e3a195ea..13c5ad1b7515 100644 --- a/ci/ci_common/common.jsonnet +++ b/ci/ci_common/common.jsonnet @@ -1,77 +1,14 @@ -local composable = (import "common-utils.libsonnet").composable; +# This file is only shared between the graal and graal-enterprise repositories. -local mx_version = (import "../../common.json").mx_version; -local common_json = composable(import "../../common.json"); +local common = import "../common.jsonnet"; local repo_config = import '../repo-configuration.libsonnet'; -local jdks = common_json.jdks; -local deps = common_json.deps; -local downloads = common_json.downloads; - -# Finds the first integer in a string and returns it as an integer. -local find_first_integer(versionString) = - local charToInt(c) = - std.codepoint(c) - std.codepoint("0"); - local firstNum(s, i) = - assert std.length(s) > i : "No number found in string " + s; - local n = charToInt(s[i]); - if n >=0 && n < 10 then i else firstNum(s, i + 1); - local lastNum(s, i) = - if i >= std.length(s) then - i - else - local n = charToInt(s[i]); - if n < 0 || n > 9 then i else lastNum(s, i + 1); - local versionIndexStart = firstNum(versionString, 0); - local versionIndexEnd = lastNum(versionString, versionIndexStart); - std.parseInt(versionString[versionIndexStart:versionIndexEnd]) -; -# jdk_version is an hidden field that can be used to generate job names -local add_jdk_version(name) = - local jdk = jdks[name]; - // this assumes that the version is the first number in the jdk.version string - local version = find_first_integer(jdk.version); - // santity check that the parsed version is also included in the name - assert std.length(std.findSubstr(std.toString(version), name)) == 1 : "Cannot find version %d in name %s" % [version, name]; - { jdk_version:: version } -; - -{ - - mx:: { - packages+: { - mx: mx_version - } - }, - - eclipse:: downloads.eclipse, - jdt:: downloads.jdt, - devkits:: common_json.devkits, - - svm_deps:: common_json.svm.deps + repo_config.native_image.extra_deps, +common + common.frequencies + { build_base:: { // holds location of CI resources that can easily be overwritten in an overlay ci_resources:: (import "ci/ci_common/ci-resources.libsonnet"), }, - // Job frequencies - // *************** - on_demand:: { - targets+: ["ondemand"], - }, - post_merge:: { - targets+: ["post-merge"], - }, - daily:: { - targets+: ["daily"], - }, - weekly:: { - targets+: ["weekly"], - }, - monthly:: { - targets+: ["monthly"], - }, - # Add a guard to `build` that prevents it from running in the gate # for a PR that only touches *.md files, the docs, are config files for GitHub add_excludes_guard(build):: build + { @@ -111,13 +48,7 @@ local add_jdk_version(name) = } }, -} + { - // JDK definitions - // *************** - // this adds all jdks from common.json - [name]: add_jdk_version(name) + { downloads+: { [if std.endsWith(name, "llvm") then "LLVM_JAVA_HOME" else "JAVA_HOME"] : jdks[name] }}, - for name in std.objectFieldsAll(jdks) -} + { +} + common.jdks + { # Aliases to edition specific labsjdks labsjdk17:: self["labsjdk-" + repo_config.graalvm_edition + "-17"], labsjdk19:: self["labsjdk-" + repo_config.graalvm_edition + "-19"], @@ -130,52 +61,28 @@ local add_jdk_version(name) = labsjdk20Debug:: self["labsjdk-" + repo_config.graalvm_edition + "-20Debug"], labsjdk20LLVM:: self["labsjdk-" + repo_config.graalvm_edition + "-20-llvm"], - // Hardware definitions // ******************** - common:: deps.common + self.mx + { - local where = if std.objectHas(self, "name") then " in " + self.name else "", - # enforce self.os (useful for generating job names) - os:: error "self.os not set" + where, - # enforce self.arch (useful for generating job names) - arch:: error "self.arch not set" + where, - capabilities +: [], - catch_files +: common_json.catch_files, - logs +: [ + local graal_common_extras = common.deps.pylint + { + logs+: [ "*.bgv", - "./" + repo_config.compiler.compiler_suite + "/graal_dumps/*/*" - ] + "./" + repo_config.compiler.compiler_suite + "/graal_dumps/*/*", + ], + timelimit: "30:00", }, - - ol7:: { - docker+: { - image: "buildslave_ol7", - mount_modules: true, + local linux_deps_extras = { + packages+: { + "apache/ant": ">=1.9.4", }, }, - linux:: deps.linux + self.common + {os::"linux", capabilities+: [self.os]}, - darwin:: deps.darwin + self.common + {os::"darwin", capabilities+: [self.os]}, - windows:: deps.windows + self.common + {os::"windows", capabilities+: [self.os]}, - windows_server_2016:: self.windows + {capabilities+: ["windows_server_2016"]}, - - amd64:: { arch::"amd64", capabilities+: [self.arch]}, - aarch64:: { arch::"aarch64", capabilities+: [self.arch]}, - - linux_amd64:: self.linux + self.amd64 + self.ol7, - darwin_amd64:: self.darwin + self.amd64, - darwin_aarch64:: self.darwin + self.aarch64 + { - # only needed until GR-22580 is resolved? - python_version: 3, - }, - windows_amd64:: self.windows + self.amd64, - windows_server_2016_amd64:: self.windows_server_2016 + self.amd64, - linux_aarch64:: self.linux + self.aarch64, + linux_amd64: linux_deps_extras + common.linux_amd64 + graal_common_extras, + linux_aarch64: linux_deps_extras + common.linux_aarch64 + graal_common_extras, + darwin_amd64: common.darwin_amd64 + graal_common_extras, + darwin_aarch64: common.darwin_aarch64 + graal_common_extras, + windows_amd64: common.windows_amd64 + graal_common_extras, + windows_server_2016_amd64: common.windows_server_2016_amd64 + graal_common_extras, + // Other mach5_target:: {targets+: ["mach5"]}, - - // Utils - disable_proxies:: { - setup+: [["unset", "HTTP_PROXY", "HTTPS_PROXY", "FTP_PROXY", "NO_PROXY", "http_proxy", "https_proxy", "ftp_proxy", "no_proxy"]], - }, } diff --git a/ci/ci_common/run-spec-tools.libsonnet b/ci/ci_common/run-spec-tools.libsonnet index eb9616d05c70..cada79432244 100644 --- a/ci/ci_common/run-spec-tools.libsonnet +++ b/ci/ci_common/run-spec-tools.libsonnet @@ -55,7 +55,7 @@ local std_get = (import "../../ci/ci_common/common-utils.libsonnet").std_get; table , // Removes the 'timelimit' property from an object. - // Usually, this is used to remove hard-coded (default) timelimits defined in `common.json`. + // Usually, this is used to remove hard-coded (default) timelimits defined in `ci/ci_common/common.jsonnet`. // These definitions assume that the os/arch definition comes first and will be refined later. // With run-spec, however, this is not true in general because the os/arch is only fixed later // in the pipeline. Thus, hard-coded timelimits would override any previous settings. To resolve diff --git a/ci/common.jsonnet b/ci/common.jsonnet new file mode 100644 index 000000000000..eeaf7597bb2c --- /dev/null +++ b/ci/common.jsonnet @@ -0,0 +1,242 @@ +# This file is shared between many repositories. +# All objects defined here are mixins, so you can use them like: +# myjob: common.linux_amd64 + common.jdks.labsjdk17ce + common.deps.sulong + ... +# Note that using a os-arch mixin like linux_amd64 mixin is required for using common.deps. + +local common_json = import "../common.json"; + +{ + # JDK definitions + # *************** + local variants(name) = [name, name + "Debug", name + "-llvm"], + local jdks_data = { + oraclejdk11: common_json.jdks["oraclejdk11"] + { jdk_version:: 11 }, + } + { + [name]: common_json.jdks[name] + { jdk_version:: 17 } + for name in ["oraclejdk17"] + variants("labsjdk-ce-17") + variants("labsjdk-ee-17") + } + { + [name]: common_json.jdks[name] + { jdk_version:: 19 } + for name in ["oraclejdk19"] + variants("labsjdk-ce-19") + variants("labsjdk-ee-19") + } + { + [name]: common_json.jdks[name] + { jdk_version:: 20 } + for name in ["oraclejdk20"] + variants("labsjdk-ce-20") + variants("labsjdk-ee-20") + }, + assert std.assertEqual(std.objectFields(common_json.jdks), std.objectFields(jdks_data)), + + # The raw jdk data, the same as common_json.jdks + { jdk_version:: } + jdks_data: jdks_data, + + # Mixins to include and download the given JDK + jdks: { + [name]: { + downloads+: { + [if std.endsWith(name, "llvm") then "LLVM_JAVA_HOME" else "JAVA_HOME"]: jdks_data[name] + }, + jdk_version:: jdks_data[name].jdk_version, + }, + for name in std.objectFields(jdks_data) + } + { + # Some convenient JDK aliases which don't require ["name"] for frequently-used JDKs + labsjdk17ce: self["labsjdk-ce-17"], + labsjdk17ee: self["labsjdk-ee-17"], + + labsjdk19ce: self["labsjdk-ce-19"], + labsjdk19ee: self["labsjdk-ee-19"], + + labsjdk20ce: self["labsjdk-ce-20"], + labsjdk20ee: self["labsjdk-ee-20"], + }, + + # The devkits versions reflect those used to build the JVMCI JDKs (e.g., see devkit_platform_revisions in /make/conf/jib-profiles.js) + devkits: { + "windows-jdk17": { packages+: { "devkit:VS2019-16.9.3+1": "==0" }}, + "windows-jdk19": { packages+: { "devkit:VS2022-17.1.0+1": "==0" }}, + "windows-jdk20": { packages+: { "devkit:VS2022-17.1.0+1": "==0" }}, + "linux-jdk17": { packages+: { "devkit:gcc10.3.0-OL6.4+1": "==0" }}, + "linux-jdk19": { packages+: { "devkit:gcc11.2.0-OL6.4+1": "==0" }}, + "linux-jdk20": { packages+: { "devkit:gcc11.2.0-OL6.4+1": "==0" }}, + }, + + deps: { + eclipse: { + downloads+: { + ECLIPSE: { + name: "eclipse", + version: common_json.eclipse.version, + platformspecific: true, + } + }, + environment+: { + ECLIPSE_EXE: "$ECLIPSE/eclipse", + }, + }, + + jdt: { + environment+: { + JDT: "builtin", + }, + }, + + pylint:: { + packages+: { + "pip:pylint": common_json.pip.pylint, + "pip:lazy-object-proxy": common_json.pip["lazy-object-proxy"], + }, + + # Required to keep pylint happy on Darwin + # https://coderwall.com/p/-k_93g/mac-os-x-valueerror-unknown-locale-utf-8-in-python" + environment+: if self.os == "darwin" then { + LC_ALL: "en_US.UTF-8", + } else {}, + }, + + sulong:: { + packages+: { + cmake: "==3.22.2", + } + if self.os == "windows" then { + msvc_source: "==14.0", + } else {}, + }, + + truffleruby:: { + packages+: if self.os == "linux" then { + ruby: "==2.6.3", + } else {}, + }, + + graalnodejs:: { + packages+: if self.os == "linux" then { + "00:devtoolset": "==7", + cmake: "==3.22.2", + } else {}, + }, + + svm:: { + environment+: { + DEFAULT_VM: "server", + LANG: "en_US.UTF-8", + }, + logs+: [ + "../.native-image/*/*/*/*.log", + "*/*.log", + "*/svmbuild/*.log", + "*/svmbuild/images/*.log", + "*/*/stripped/*.map", + "*/callgrind.*", + "*.log", + ], + + packages+: if self.os == "linux" && self.arch == "amd64" then { + "00:devtoolset": "==7", + "01:binutils": ">=2.34", + } else if self.os == "linux" && self.arch == "aarch64" then { + "00:devtoolset": "==7", + } else {}, + }, + }, + + # Job frequencies + # *************** + frequencies: { + gate: { + targets+: ["gate"], + }, + bench: { + targets+: ["bench"], + }, + on_demand: { + targets+: ["ondemand"], + }, + post_merge: { + targets+: ["post-merge"], + }, + daily: { + targets+: ["daily"], + }, + weekly: { + targets+: ["weekly"], + }, + monthly: { + targets+: ["monthly"], + }, + }, + + local deps_linux = { + packages+: { + git: ">=1.8.3", + }, + }, + local deps_darwin = { + }, + local deps_windows = { + }, + + local catch_files = [ + # Keep in sync with org.graalvm.compiler.debug.StandardPathUtilitiesProvider#DIAGNOSTIC_OUTPUT_DIRECTORY_MESSAGE_REGEXP + "Graal diagnostic output saved in '(?P[^']+)'", + # Keep in sync with org.graalvm.compiler.debug.DebugContext#DUMP_FILE_MESSAGE_REGEXP + "Dumping debug output to '(?P[^']+)'", + # Keep in sync with com.oracle.svm.hosted.NativeImageOptions#DEFAULT_ERROR_FILE_NAME + " (?P.+/svm_err_b_\\d+T\\d+\\.\\d+_pid\\d+\\.md)", + ], + + # Included in common + local mx = { + environment+: { + MX_PYTHON: "python3.8", + }, + packages+: { + python3: "==3.8.10", + "pip:ninja_syntax": common_json.pip.ninja_syntax, + mx: common_json.mx_version, + }, + python_version: "3", # To use the correct virtualenv + }, + + # Hardware definitions + # ******************** + # Note that only platforms (os-arch) are exposed (not os and arch separately), + # because this is the simplest way to ensure correct usage and dependencies (e.g. ol7 in linux_amd64). + # If you want to add extra "common" things for your CI, just define your own objects like: + # linux_amd64: common.linux_amd64 + extras, + local common = mx + { + local where = if std.objectHas(self, "name") then " in " + self.name else "", + # enforce self.os (useful for generating job names) + os:: error "self.os not set" + where, + # enforce self.arch (useful for generating job names) + arch:: error "self.arch not set" + where, + capabilities+: [], + catch_files+: catch_files, + }, + + local ol7 = { + docker+: { + image: "buildslave_ol7", + mount_modules: true, + }, + }, + + local linux = deps_linux + common + { os:: "linux", capabilities+: [self.os] }, + local darwin = deps_darwin + common + { os:: "darwin", capabilities+: [self.os] }, + local windows = deps_windows + common + { os:: "windows", capabilities+: [self.os] }, + local windows_server_2016 = windows + { capabilities+: ["windows_server_2016"] }, + + local amd64 = { arch:: "amd64", capabilities+: [self.arch] }, + local aarch64 = { arch:: "aarch64", capabilities+: [self.arch] }, + + linux_amd64: linux + amd64 + ol7, + linux_aarch64: linux + aarch64, + + darwin_amd64: darwin + amd64, + darwin_aarch64: darwin + aarch64, + + windows_amd64: windows + amd64, + windows_server_2016_amd64: windows_server_2016 + amd64, + + # Utils + disable_proxies: { + setup+: [ + ["unset", "HTTP_PROXY", "HTTPS_PROXY", "FTP_PROXY", "NO_PROXY", "http_proxy", "https_proxy", "ftp_proxy", "no_proxy"], + ], + }, +} diff --git a/common.json b/common.json index 3d0e228a33c5..535df2e12217 100644 --- a/common.json +++ b/common.json @@ -1,10 +1,15 @@ { - "README": "This file contains definitions that are useful for the hocon and jsonnet CI files of multiple repositories.", + "README": [ + "This file contains some data shared between CI files and various other usages, and is copied in many repositories.", + "Jsonnet files should not include this file directly but use ci/common.jsonnet instead." + ], - "mx_version" : "6.16.0", + "mx_version": "6.16.0", + "COMMENT.jdks": "When adding or removing JDKs keep in sync with JDKs in ci/common.jsonnet", "jdks": { "oraclejdk11": {"name": "jpg-jdk", "version": "11.0.11", "build_id": "9", "release": true, "platformspecific": true, "extrabundles": ["static-libs"] }, + "oraclejdk17": {"name": "jpg-jdk", "version": "17.0.1", "build_id": "12", "release": true, "platformspecific": true, "extrabundles": ["static-libs"]}, "labsjdk-ee-17": {"name": "labsjdk", "version": "ee-17.0.6+9-jvmci-23.0-b07", "platformspecific": true }, "labsjdk-ee-17Debug": {"name": "labsjdk", "version": "ee-17.0.6+9-jvmci-23.0-b07-debug", "platformspecific": true }, @@ -30,166 +35,15 @@ "labsjdk-ee-20-llvm": {"name": "labsjdk", "version": "ee-20.0.1+4-jvmci-23.0-b07-sulong", "platformspecific": true } }, - "COMMENT.devkits" : "The devkits versions reflect those used to build the JVMCI JDKs (e.g., see devkit_platform_revisions in /make/conf/jib-profiles.js)", - "devkits": { - "windows-jdk17": { "packages" : { "devkit:VS2019-16.9.3+1" : "==0" }}, - "windows-jdk19": { "packages" : { "devkit:VS2022-17.1.0+1" : "==0" }}, - "windows-jdk20": { "packages" : { "devkit:VS2022-17.1.0+1" : "==0" }}, - "linux-jdk17": { "packages" : { "devkit:gcc10.3.0-OL6.4+1" : "==0" }}, - "linux-jdk19": { "packages" : { "devkit:gcc11.2.0-OL6.4+1" : "==0" }}, - "linux-jdk20": { "packages" : { "devkit:gcc11.2.0-OL6.4+1" : "==0" }} - }, - - "catch_files" : [ - "Graal diagnostic output saved in '(?P[^']+)'", - "Dumping debug output to '(?P[^']+)'", - " (?P.+/svm_err_b_\\d+T\\d+\\.\\d+_pid\\d+\\.md)" - ], - - "deps": { - "common": { - "timelimit": "30:00", - "environment": { - "MX_PYTHON": "python3.8" - }, - "packages": { - "python3": "==3.8.10", - "pip:pylint": "==2.4.4", - "pip:lazy-object-proxy": "==1.6.0", - "pip:ninja_syntax": "==1.7.2" - }, - "python_version": "3" - }, - - "linux": { - "packages": { - "git": ">=1.8.3", - "mercurial": ">=2.2", - "apache/ant": ">=1.9.4" - } - }, - "COMMENT.darwin": [ - "Required to keep pylint happy on Darwin", - "https://coderwall.com/p/-k_93g/mac-os-x-valueerror-unknown-locale-utf-8-in-python" - ], - "darwin": { - "environment": { - "LC_ALL": "en_US.UTF-8" - } - }, - "windows": { - } - + "eclipse": { + "version": "4.14.0", + "short_version": "4.14", + "timestamp": "201912100610" }, - "downloads": { - "eclipse": { - "downloads": { - "ECLIPSE": { - "name": "eclipse", - "version": "4.14.0", - "platformspecific": true - } - }, - "environment": { - "ECLIPSE_EXE": "$ECLIPSE/eclipse" - }, - "COMMENT.eclipse_org": [ - "Coordinates for downloading same version as above directly from eclipse.org (used by GitHub actions).", - "Template URL: https://archive.eclipse.org/eclipse/downloads/drops4/R--/eclipse-SDK--linux-gtk-x86_64.tar.gz" - ], - "eclipse_org": { - "version": "4.14", - "timestamp": "201912100610" - } - }, - "jdt": { - "environment": { - "JDT": "builtin" - } - } - }, - "sulong": { - "deps": { - "common": { - "packages": { - "cmake": "==3.22.2" - } - }, - "linux": { - "packages": {} - }, - "darwin_amd64": { - "packages": {} - }, - "darwin_aarch64": { - "packages": {} - }, - "windows": { - "packages": { - "msvc_source": "==14.0" - } - } - } - }, - "truffleruby": { - "deps": { - "common": {}, - "linux": { - "packages": { - "ruby": "==2.6.3" - } - }, - "darwin": {} - } - }, - "graalnodejs": { - "deps": { - "common": {}, - "linux": { - "packages": { - "00:devtoolset": "==7", - "cmake": "==3.22.2" - } - }, - "darwin": {} - } - }, - "svm": { - "COMMENT.deps": "Please use svm_deps in common.jsonnet instead (the values below are without the repository-specific dependencies).", - "deps": { - "COMMENT.common": "graal/vm/ci_common/common.jsonnet hardcodes the properties of this object", - "common": { - "environment": { - "DEFAULT_VM": "server", - "LANG": "en_US.UTF-8" - }, - "logs": [ - "../.native-image/*/*/*/*.log", - "*/*.log", - "*/svmbuild/*.log", - "*/svmbuild/images/*.log", - "*/*/stripped/*.map", - "*/callgrind.*", - "*.log" - ] - }, - "linux_amd64": { - "packages": { - "00:devtoolset": "==7", - "01:binutils": ">=2.34", - "ruby": ">=2.1.0" - } - }, - "linux_aarch64": { - "packages": { - "00:devtoolset": "==7", - "ruby": ">=2.1.0" - } - }, - "darwin_amd64": {}, - "darwin_aarch64": {}, - "windows": {} - } + "pip": { + "ninja_syntax": "==1.7.2", + "lazy-object-proxy": "==1.6.0", + "pylint": "==2.4.4" } } diff --git a/compiler/ci/ci_common/benchmark-suites.libsonnet b/compiler/ci/ci_common/benchmark-suites.libsonnet index b90d3cdb4eb7..0b8502b364bf 100644 --- a/compiler/ci/ci_common/benchmark-suites.libsonnet +++ b/compiler/ci/ci_common/benchmark-suites.libsonnet @@ -1,5 +1,4 @@ { - local common_json = (import '../../../common.json'), local c = (import '../../../ci/ci_common/common.jsonnet'), local bc = (import '../../../ci/ci_common/bench-common.libsonnet'), local cc = (import 'compiler-common.libsonnet'), @@ -209,7 +208,6 @@ microservice_benchmarks: cc.compiler_benchmark + { suite:: "microservices", packages+: { - "python3": common_json.deps.common.packages["python3"], "pip:psutil": "==5.8.0" }, local bench_upload = ["bench-uploader.py", "bench-results.json"], diff --git a/compiler/ci/ci_common/gate.jsonnet b/compiler/ci/ci_common/gate.jsonnet index 5419a1cd52eb..1234ec560550 100644 --- a/compiler/ci/ci_common/gate.jsonnet +++ b/compiler/ci/ci_common/gate.jsonnet @@ -125,7 +125,7 @@ bootstrap_full:: s.base("build,bootstrapfullverify", no_warning_as_error=true), bootstrap_economy:: s.base("build,bootstrapeconomy", no_warning_as_error=true, extra_vm_args="-Dgraal.CompilerConfiguration=economy"), - style:: c.eclipse + c.jdt + s.base("style,fullbuild,javadoc"), + style:: c.deps.eclipse + c.deps.jdt + s.base("style,fullbuild,javadoc"), avx3:: { capabilities+: ["avx512"], diff --git a/compiler/mx.compiler/mx_compiler.py b/compiler/mx.compiler/mx_compiler.py index 8b5062c9dc0b..37b283c52f88 100644 --- a/compiler/mx.compiler/mx_compiler.py +++ b/compiler/mx.compiler/mx_compiler.py @@ -43,7 +43,7 @@ import mx import mx_gate -from mx_gate import Task, Tags +from mx_gate import Task from mx import SafeDirectoryUpdater import mx_unittest @@ -419,51 +419,7 @@ def _gate_scala_dacapo(name, iterations, extraVMarguments=None): mx.abort("Gate for scala-dacapo benchmark '{}' failed!".format(name)) return exit_code, suite, results -def get_catch_files_and_common_path(): - common_path = join(dirname(_suite.dir), 'common.json') - if not exists(common_path): - mx.abort('Required file does not exist: {}'.format(common_path)) - with open(common_path) as common_file: - common_cfg = json.load(common_file) - catch_files = common_cfg.get('catch_files') - if catch_files is None: - mx.abort('Could not find catch_files attribute in {}'.format(common_path)) - return catch_files, common_path - -def find_field_value(source_path, field_name): - decl = field_name + ' = "' - with open(source_path) as fp: - for line in fp.readlines(): - index = line.find(decl) - if index != -1: - start_index = index + len(decl) - end_index = line.find('"', start_index) - return line[start_index:end_index] - mx.abort('Could not find value of ' + field_name + ' in ' + source_path) - -def _check_catch_files(): - """ - Verifies that there is a "catch_files" array in common.json at the root of - the repository containing this suite and that the array contains elements - matching DebugContext.DUMP_FILE_MESSAGE_REGEXP and - StandardPathUtilitiesProvider.DIAGNOSTIC_OUTPUT_DIRECTORY_MESSAGE_REGEXP. - """ - catch_files_fields = ( - ('DebugContext', 'DUMP_FILE_MESSAGE_REGEXP'), - ('StandardPathUtilitiesProvider', 'DIAGNOSTIC_OUTPUT_DIRECTORY_MESSAGE_REGEXP') - ) - - catch_files, common_path = get_catch_files_and_common_path() - for class_name, field_name in catch_files_fields: - source_path = join(_suite.dir, 'src', 'org.graalvm.compiler.debug', 'src', 'org', 'graalvm', 'compiler', 'debug', class_name + '.java') - regexp = find_field_value(source_path, field_name).replace('(?<', '(?P<') # Convert from Java style regexp to Python style - if regexp not in catch_files: - mx.abort('Could not find catch_files entry in {} matching "{}"'.format(common_path, regexp)) - def compiler_gate_runner(suites, unit_test_runs, bootstrap_tests, tasks, extraVMarguments=None, extraUnitTestArguments=None): - with Task('CheckCatchFiles', tasks, tags=[Tags.style]) as t: - if t: _check_catch_files() - with Task('JDK_java_base_test', tasks, tags=['javabasetest'], report=True) as t: if t: java_base_unittest(_remove_empty_entries(extraVMarguments) + []) diff --git a/compiler/src/org.graalvm.compiler.debug/src/org/graalvm/compiler/debug/DebugContext.java b/compiler/src/org.graalvm.compiler.debug/src/org/graalvm/compiler/debug/DebugContext.java index 4abc1b6ee6b5..a234ffb64c18 100644 --- a/compiler/src/org.graalvm.compiler.debug/src/org/graalvm/compiler/debug/DebugContext.java +++ b/compiler/src/org.graalvm.compiler.debug/src/org/graalvm/compiler/debug/DebugContext.java @@ -95,7 +95,7 @@ public final class DebugContext implements AutoCloseable { * The regular expression for matching the message derived from * {@link #DUMP_FILE_MESSAGE_FORMAT}. * - * Keep in sync with the {@code catch_files} array in {@code common.json}. + * Keep in sync with the {@code catch_files} array in {@code ci/common.jsonnet}. */ public static final String DUMP_FILE_MESSAGE_REGEXP = "Dumping debug output to '(?[^']+)'"; diff --git a/compiler/src/org.graalvm.compiler.debug/src/org/graalvm/compiler/debug/StandardPathUtilitiesProvider.java b/compiler/src/org.graalvm.compiler.debug/src/org/graalvm/compiler/debug/StandardPathUtilitiesProvider.java index b5817718cd2c..dd26998a9b8c 100644 --- a/compiler/src/org.graalvm.compiler.debug/src/org/graalvm/compiler/debug/StandardPathUtilitiesProvider.java +++ b/compiler/src/org.graalvm.compiler.debug/src/org/graalvm/compiler/debug/StandardPathUtilitiesProvider.java @@ -64,7 +64,7 @@ public class StandardPathUtilitiesProvider implements PathUtilitiesProvider { * The regular expression for matching the message derived from * {@link #DIAGNOSTIC_OUTPUT_DIRECTORY_MESSAGE_FORMAT}. * - * Keep in sync with the {@code catch_files} array in {@code common.json}. + * Keep in sync with the {@code catch_files} array in {@code ci/common.jsonnet}. */ public static final String DIAGNOSTIC_OUTPUT_DIRECTORY_MESSAGE_REGEXP = "Graal diagnostic output saved in '(?[^']+)'"; @@ -190,7 +190,7 @@ public FileVisitResult postVisitDirectory(Path d, IOException exc) throws IOExce return FileVisitResult.CONTINUE; } }); - // Keep this in sync with the catch_files in common.json + // Keep this in sync with the catch_files in ci/common.jsonnet TTY.println(DIAGNOSTIC_OUTPUT_DIRECTORY_MESSAGE_FORMAT, zipFile); return zipFile.getAbsolutePath(); } catch (IOException e) { diff --git a/espresso/ci/ci.jsonnet b/espresso/ci/ci.jsonnet index 6b5af3547d28..9b1b0b9114e2 100644 --- a/espresso/ci/ci.jsonnet +++ b/espresso/ci/ci.jsonnet @@ -1,7 +1,5 @@ { - local graal_common = import '../../common.json', local common = import 'ci_common/common.jsonnet', - local jdks = graal_common.jdks, suite_name: 'espresso', diff --git a/espresso/ci/ci_common/common.jsonnet b/espresso/ci/ci_common/common.jsonnet index c1b8968df204..ee4d0125cd46 100644 --- a/espresso/ci/ci_common/common.jsonnet +++ b/espresso/ci/ci_common/common.jsonnet @@ -1,11 +1,7 @@ local graal_common = import '../../../ci/ci_common/common.jsonnet'; local base = import '../ci.jsonnet'; -local base_json = import '../../../common.json'; -local utils = import "../../../ci/ci_common/common-utils.libsonnet"; -local composable = utils.composable; -local sulong_deps = composable(base_json.sulong.deps); -local devkits = composable(graal_common.devkits); +local devkits = graal_common.devkits; local _version_suffix(java_version) = if java_version == 8 then '' else '-java' + java_version; @@ -20,7 +16,7 @@ local benchmark_suites = ['dacapo', 'renaissance', 'scala-dacapo']; local that = self, // platform-specific snippets - common: base_json.deps.common + graal_common.mx + sulong_deps.common + { + common: graal_common.deps.sulong + { python_version: '3', environment+: { GRAALVM_CHECK_EXPERIMENTAL_OPTIONS: "true", @@ -30,7 +26,7 @@ local benchmark_suites = ['dacapo', 'renaissance', 'scala-dacapo']; ], }, - linux: self.common + sulong_deps.linux + graal_common.linux_amd64 + { + linux: self.common + graal_common.linux_amd64 + { packages+: { '00:devtoolset': '==7', # GCC 7.3.1, make 4.2.1, binutils 2.28, valgrind 3.13.0 '01:binutils': '==2.34', @@ -42,7 +38,7 @@ local benchmark_suites = ['dacapo', 'renaissance', 'scala-dacapo']; capabilities+: ['no_frequency_scaling', 'tmpfs25g', 'x52'], }, - darwin_amd64: self.common + sulong_deps.darwin_amd64 + graal_common.darwin_amd64 + { + darwin_amd64: self.common + graal_common.darwin_amd64 + { environment+: { // for compatibility with macOS High Sierra MACOSX_DEPLOYMENT_TARGET: '10.13', diff --git a/regex/ci/ci.jsonnet b/regex/ci/ci.jsonnet index 9e59871281ba..c907af6fc49b 100644 --- a/regex/ci/ci.jsonnet +++ b/regex/ci/ci.jsonnet @@ -8,7 +8,7 @@ timelimit: "30:00", }, - local regex_gate = regex_common + common.eclipse + common.jdt + { + local regex_gate = regex_common + common.deps.eclipse + common.deps.jdt + { name: 'gate-regex-oraclejdk' + self.jdk_version, run: [["mx", "--strict-compliance", "gate", "--strict-mode"]], targets: ["gate"], diff --git a/sdk/ci/ci.jsonnet b/sdk/ci/ci.jsonnet index 216c7d5c62e4..35b306cc5b9a 100644 --- a/sdk/ci/ci.jsonnet +++ b/sdk/ci/ci.jsonnet @@ -18,8 +18,8 @@ }, builds: [ - common.linux_amd64 + common.oraclejdk20 + sdk_gate + common.eclipse + common.jdt, - common.linux_amd64 + common.oraclejdk17 + sdk_gate + common.eclipse + common.jdt + common.mach5_target, + common.linux_amd64 + common.oraclejdk20 + sdk_gate + common.deps.eclipse + common.deps.jdt, + common.linux_amd64 + common.oraclejdk17 + sdk_gate + common.deps.eclipse + common.deps.jdt + common.mach5_target, common.darwin_amd64 + common.oraclejdk20 + sdk_gate, common.darwin_amd64 + common.oraclejdk17 + sdk_gate + common.mach5_target, ] diff --git a/substratevm/ci/ci.jsonnet b/substratevm/ci/ci.jsonnet index cb07a1384877..199dd5acf3e7 100644 --- a/substratevm/ci/ci.jsonnet +++ b/substratevm/ci/ci.jsonnet @@ -16,8 +16,8 @@ // mx gate build config local mxgate(tags) = os_arch_jdk_mixin + sg.mxgate(tags, suite="substratevm", suite_short="svm"), - local eclipse = task_spec(common.eclipse), - local jdt = task_spec(common.jdt), + local eclipse = task_spec(common.deps.eclipse), + local jdt = task_spec(common.deps.jdt), local gate = sg.gate, local gdb(version) = task_spec(sg.gdb(version)), local use_musl = sg.use_musl, diff --git a/substratevm/mx.substratevm/mx_substratevm.py b/substratevm/mx.substratevm/mx_substratevm.py index 7f7cfe17c954..7f466548e38d 100644 --- a/substratevm/mx.substratevm/mx_substratevm.py +++ b/substratevm/mx.substratevm/mx_substratevm.py @@ -351,30 +351,7 @@ def truffle_unittest_task(extra_image_args=None): native_unittest(['org.graalvm.compiler.truffle.test.ContextLookupCompilationTest'] + compiler_args) -def _check_catch_files(): - """ - Verifies that there is a "catch_files" array in common.json at the root of - the repository containing this suite and that the array contains elements - matching NativeImageOptions.DEFAULT_ERROR_FILE_NAME. - """ - - expected_default_error_file_name_pattern = "svm_err_b_%t_pid%p.md" - expected_catch_files_entry = " (?P.+/svm_err_b_\\d+T\\d+\\.\\d+_pid\\d+\\.md)" - - source_path = join(suite.dir, 'src', 'com.oracle.svm.hosted', 'src', 'com', 'oracle', 'svm', 'hosted', 'NativeImageOptions.java') - actual_pattern = mx_compiler.find_field_value(source_path, 'DEFAULT_ERROR_FILE_NAME') - if expected_default_error_file_name_pattern != actual_pattern: - mx.abort('DEFAULT_ERROR_FILE_NAME pattern has changed (expect: "{}", got: "{}")'.format(expected_default_error_file_name_pattern, actual_pattern)) - - catch_files, common_path = mx_compiler.get_catch_files_and_common_path() - if not expected_catch_files_entry in catch_files: - mx.abort('Could not find catch_files entry in {} matching "{}"'.format(common_path, expected_catch_files_entry)) - - def svm_gate_body(args, tasks): - with Task('CheckCatchFiles', tasks, tags=[mx_gate.Tags.style]) as t: - if t: _check_catch_files() - with Task('image demos', tasks, tags=[GraalTags.helloworld]) as t: if t: with native_image_context(IMAGE_ASSERTION_FLAGS) as native_image: diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageOptions.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageOptions.java index 03565b59cb03..988026c37412 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageOptions.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageOptions.java @@ -184,7 +184,11 @@ public static CStandards getCStandard() { @Option(help = "Print unsafe operation offset warnings.)")// public static final HostedOptionKey UnsafeOffsetWarningsAreFatal = new HostedOptionKey<>(false); - // Inspired by HotSpot's hs_err_.log files and for build-time errors (err_b). + /** + * Inspired by HotSpot's hs_err_.log files and for build-time errors (err_b). + * + * Keep in sync with the {@code catch_files} array in {@code ci/common.jsonnet}. + */ private static final String DEFAULT_ERROR_FILE_NAME = "svm_err_b_%t_pid%p.md"; public static final Path getErrorFilePath(OptionValues hostedOptionValues) { diff --git a/sulong/ci/ci_common/sulong-common.jsonnet b/sulong/ci/ci_common/sulong-common.jsonnet index 40434c219eac..3627a51d5628 100644 --- a/sulong/ci/ci_common/sulong-common.jsonnet +++ b/sulong/ci/ci_common/sulong-common.jsonnet @@ -1,8 +1,7 @@ # File is formatted with # `jsonnetfmt --indent 2 --max-blank-lines 2 --sort-imports --string-style d --comment-style h -i ci.jsonnet` local common = import "../../../ci/ci_common/common.jsonnet"; -local composable = (import "../../../ci/ci_common/common-utils.libsonnet").composable; -local sulong_deps = composable((import "../../../common.json").sulong.deps); +local sulong_deps = common.deps.sulong; { local linux_amd64 = common.linux_amd64, @@ -77,12 +76,12 @@ local sulong_deps = composable((import "../../../common.json").sulong.deps); [b for b in result if !std.objectHasAll(b, "skipPlatform") || !b.skipPlatform] , - linux_amd64:: linux_amd64 + sulong_deps.common + sulong_deps.linux, - linux_aarch64:: linux_aarch64 + sulong_deps.common + sulong_deps.linux, + linux_amd64:: linux_amd64 + sulong_deps, + linux_aarch64:: linux_aarch64 + sulong_deps, # Avoid darwin_sierra builders in our CI. This is missing a declaration (fmemopen) that some of our tests need. - darwin_amd64:: darwin_amd64 + sulong_deps.common + sulong_deps.darwin_amd64 + { capabilities+: ["!darwin_sierra"] }, - darwin_aarch64:: darwin_aarch64 + sulong_deps.common + sulong_deps.darwin_aarch64, - windows_amd64:: windows_amd64 + sulong_deps.common + sulong_deps.windows + { + darwin_amd64:: darwin_amd64 + sulong_deps + { capabilities+: ["!darwin_sierra"] }, + darwin_aarch64:: darwin_aarch64 + sulong_deps, + windows_amd64:: windows_amd64 + sulong_deps + { packages+: common.devkits["windows-" + self.jdk].packages }, @@ -155,7 +154,7 @@ local sulong_deps = composable((import "../../../common.json").sulong.deps); gateTags:: std.split(tags, ","), }, - style:: common.eclipse + common.jdt + $.gateTags("style,fullbuild") + { + style:: common.deps.eclipse + common.deps.jdt + $.gateTags("style,fullbuild") + { extra_gate_args+:: ["--strict-mode"], }, diff --git a/tools/ci/ci.jsonnet b/tools/ci/ci.jsonnet index 21bed916642e..15a94b77e505 100644 --- a/tools/ci/ci.jsonnet +++ b/tools/ci/ci.jsonnet @@ -1,11 +1,9 @@ { - local common_json = import '../../common.json', local common = import '../../ci/ci_common/common.jsonnet', - local composable = (import '../../ci/ci_common/common-utils.libsonnet').composable, local top_level_ci = (import '../../ci/ci_common/common-utils.libsonnet').top_level_ci, - local devkits = composable(common_json.devkits), + local devkits = common.devkits, - local tools_common = composable(common_json.deps.common) + common.mx + { + local tools_common = { setup+: [ ["cd", "./tools"], ], @@ -23,7 +21,7 @@ } }, - local tools_gate = gate_guard + tools_common + common.eclipse + common.jdt + { + local tools_gate = gate_guard + tools_common + common.deps.eclipse + common.deps.jdt + { name: 'gate-tools-oraclejdk' + self.jdk_version + '-' + self.os + '-' + self.arch, run: [["mx", "--strict-compliance", "gate", "--strict-mode"]], targets: ["gate"], @@ -59,7 +57,7 @@ "com.oracle.truffle.tools" ], - local tools_coverage_weekly = tools_common + common.eclipse + common.jdt + { + local tools_coverage_weekly = tools_common + common.deps.eclipse + common.deps.jdt + { name: "weekly-tools-coverage", run: [ ["mx"] + coverage_whitelisting + [ diff --git a/truffle/ci/ci.jsonnet b/truffle/ci/ci.jsonnet index 12b764e76384..d9decc07d11d 100644 --- a/truffle/ci/ci.jsonnet +++ b/truffle/ci/ci.jsonnet @@ -1,16 +1,14 @@ { - local common_json = import '../../common.json', local common = import '../../ci/ci_common/common.jsonnet', local bench_hw = (import '../../ci/ci_common/bench-common.libsonnet').bench_hw, - local composable = (import '../../ci/ci_common/common-utils.libsonnet').composable, local top_level_ci = (import '../../ci/ci_common/common-utils.libsonnet').top_level_ci, - local devkits = composable(common_json.devkits), + local devkits = common.devkits, local darwin_amd64 = common.darwin_amd64, local linux_amd64 = common.linux_amd64, local windows_amd64 = common.windows_amd64, - local truffle_common = composable(common_json.deps.common) + common.mx + { + local truffle_common = { setup+: [ ["cd", "./truffle"], ], @@ -20,11 +18,11 @@ local guard = { guard: { - includes: ["/sdk/**", "/truffle/**", "**.jsonnet"] + top_level_ci, + includes: ["/sdk/**", "/truffle/**", "**.jsonnet"] + top_level_ci, } }, - local bench_common = composable(common_json.deps.common) + common.mx + { + local bench_common = { environment+: { BENCH_RESULTS_FILE_PATH: "bench-results.json", }, @@ -103,7 +101,7 @@ ], }, - local truffle_gate = truffle_common + common.eclipse + common.jdt { + local truffle_gate = truffle_common + common.deps.eclipse + common.deps.jdt { name: 'gate-truffle-oraclejdk-' + self.jdk_version, run: [["mx", "--strict-compliance", "gate", "--strict-mode"]], }, @@ -151,7 +149,7 @@ ], }, - truffle_common + linux_amd64 + common.oraclejdk17 + common.eclipse + common.jdt + guard + { + truffle_common + linux_amd64 + common.oraclejdk17 + common.deps.eclipse + common.deps.jdt + guard + { name: "weekly-truffle-coverage-17-linux-amd64", run: [ ["mx", "--strict-compliance", "gate", "--strict-mode", "--jacoco-relativize-paths", "--jacoco-omit-src-gen", "--jacocout", "coverage", "--jacoco-format", "lcov"], diff --git a/vm/ci/ci_common/common.jsonnet b/vm/ci/ci_common/common.jsonnet index 371ef3df8658..c69a52ee307e 100644 --- a/vm/ci/ci_common/common.jsonnet +++ b/vm/ci/ci_common/common.jsonnet @@ -1,9 +1,7 @@ -local composable = (import '../../../ci/ci_common/common-utils.libsonnet').composable; local vm = import '../ci_includes/vm.jsonnet'; local graal_common = import '../../../ci/ci_common/common.jsonnet'; local repo_config = import '../../../repo-configuration.libsonnet'; -local common_json = composable(import '../../../common.json'); -local devkits = common_json.devkits; +local devkits = graal_common.devkits; { verify_name(build): { @@ -63,29 +61,29 @@ local devkits = common_json.devkits; js_windows_jdk17: self.js_windows_common + { setup+: [ - # Keep in sync with the 'devkits' object defined in the top level common.json file. + # Keep in sync with the 'devkits' object defined in ci/common.jsonnet. ['set-export', 'DEVKIT_VERSION', '2019'], ], }, js_windows_jdk20: self.js_windows_common + { setup+: [ - # Keep in sync with the 'devkits' object defined in the top level common.json file. + # Keep in sync with the 'devkits' object defined in ci/common.jsonnet. ['set-export', 'DEVKIT_VERSION', '2022'], ], }, # SULONG - sulong_linux: common_json.sulong.deps.common + common_json.sulong.deps.linux, - sulong_darwin_amd64: common_json.sulong.deps.common + common_json.sulong.deps.darwin_amd64, - sulong_darwin_aarch64: common_json.sulong.deps.common + common_json.sulong.deps.darwin_aarch64, - sulong_windows: common_json.sulong.deps.common + common_json.sulong.deps.windows, + sulong_linux: graal_common.deps.sulong, + sulong_darwin_amd64: graal_common.deps.sulong, + sulong_darwin_aarch64: graal_common.deps.sulong, + sulong_windows: graal_common.deps.sulong, # TRUFFLERUBY, needs OpenSSL 1.0.2+, so OracleLinux 7+ - truffleruby_linux_amd64: self.sulong_linux + common_json.truffleruby.deps.common + common_json.truffleruby.deps.linux, - truffleruby_linux_aarch64: self.sulong_linux + common_json.truffleruby.deps.common + common_json.truffleruby.deps.linux, - truffleruby_darwin_amd64: self.sulong_darwin_amd64 + common_json.truffleruby.deps.common + common_json.truffleruby.deps.darwin, - truffleruby_darwin_aarch64: self.sulong_darwin_aarch64 + common_json.truffleruby.deps.common + common_json.truffleruby.deps.darwin, + truffleruby_linux_amd64: graal_common.deps.sulong + graal_common.deps.truffleruby, + truffleruby_linux_aarch64: graal_common.deps.sulong + graal_common.deps.truffleruby, + truffleruby_darwin_amd64: graal_common.deps.sulong + graal_common.deps.truffleruby, + truffleruby_darwin_aarch64: graal_common.deps.sulong + graal_common.deps.truffleruby, # FASTR # Note: On both Linux and MacOS, FastR depends on the gnur module and on gfortran @@ -164,7 +162,7 @@ local devkits = common_json.devkits; graalpython_darwin_aarch64: self.sulong_darwin_aarch64 + {}, - vm_linux_amd64: self.common_vm_linux + graal_common.linux_amd64 + graal_common.svm_deps.linux_amd64 { + vm_linux_amd64: self.common_vm_linux + graal_common.linux_amd64 + graal_common.deps.svm { capabilities+: ['manycores', 'ram16gb', 'fast'], }, @@ -363,11 +361,11 @@ local devkits = common_json.devkits; mx_vm_common: vm.mx_cmd_base_no_env + ['--env', '${VM_ENV}'] + self.mx_vm_cmd_suffix, mx_vm_installables: vm.mx_cmd_base_no_env + ['--env', '${VM_ENV}-complete'] + self.mx_vm_cmd_suffix, - svm_common_linux_amd64: { environment+: graal_common.svm_deps.common.environment, logs+: graal_common.svm_deps.common.logs} + graal_common.svm_deps.linux_amd64, - svm_common_linux_aarch64: { environment+: graal_common.svm_deps.common.environment, logs+: graal_common.svm_deps.common.logs} + graal_common.svm_deps.linux_aarch64, - svm_common_darwin_amd64: { environment+: graal_common.svm_deps.common.environment, logs+: graal_common.svm_deps.common.logs} + graal_common.svm_deps.darwin_amd64, - svm_common_darwin_aarch64: { environment+: graal_common.svm_deps.common.environment, logs+: graal_common.svm_deps.common.logs} + graal_common.svm_deps.darwin_aarch64, - svm_common_windows_amd64(jdk): { environment+: graal_common.svm_deps.common.environment, logs+: graal_common.svm_deps.common.logs} + graal_common.svm_deps.windows + common_json.devkits["windows-jdk" + jdk], + svm_common_linux_amd64: graal_common.deps.svm, + svm_common_linux_aarch64: graal_common.deps.svm, + svm_common_darwin_amd64: graal_common.deps.svm, + svm_common_darwin_aarch64: graal_common.deps.svm, + svm_common_windows_amd64(jdk): graal_common.deps.svm + graal_common.devkits["windows-jdk" + jdk], maven_deploy_sdk: ['--suite', 'sdk', 'maven-deploy', '--validate', 'none', '--all-distribution-types', '--with-suite-revisions-metadata'], deploy_artifacts_sdk(os, base_dist_name=null): (if base_dist_name != null then ['--base-dist-name=' + base_dist_name] else []) + ['--suite', 'sdk', 'deploy-artifacts', '--uploader', if os == 'windows' then 'artifact_uploader.cmd' else 'artifact_uploader'], @@ -635,7 +633,7 @@ local devkits = common_json.devkits; # # Gates # - vm.vm_java_20 + common_json.downloads.eclipse + common_json.downloads.jdt + self.gate_vm_linux_amd64 + { + vm.vm_java_20 + graal_common.deps.eclipse + graal_common.deps.jdt + self.gate_vm_linux_amd64 + { run: [ ['mx', 'gate', '-B=--force-deprecation-as-warning', '--tags', 'style,fullbuild'], ], diff --git a/vm/ci/ci_common/libgraal.jsonnet b/vm/ci/ci_common/libgraal.jsonnet index a56bee76752f..fe5b938a4226 100644 --- a/vm/ci/ci_common/libgraal.jsonnet +++ b/vm/ci/ci_common/libgraal.jsonnet @@ -1,9 +1,7 @@ -local composable = (import '../../../ci/ci_common/common-utils.libsonnet').composable; local vm = import '../ci_includes/vm.jsonnet'; local graal_common = import '../../../ci/ci_common/common.jsonnet'; local repo_config = import '../../../ci/repo-configuration.libsonnet'; -local common_json = composable(import '../../../common.json'); -local devkits = common_json.devkits; +local devkits = graal_common.devkits; local c = import 'common.jsonnet'; local g = vm.compiler_gate; local utils = import '../../../ci/ci_common/common-utils.libsonnet'; diff --git a/vm/ci/ci_includes/vm.jsonnet b/vm/ci/ci_includes/vm.jsonnet index a311622d49cb..6099cdafc83c 100644 --- a/vm/ci/ci_includes/vm.jsonnet +++ b/vm/ci/ci_includes/vm.jsonnet @@ -1,13 +1,10 @@ local utils = import '../../../ci/ci_common/common-utils.libsonnet'; -local composable = utils.composable; local vm_common = import '../ci_common/common.jsonnet'; local vm_common_bench = import '../ci_common/common-bench.jsonnet'; local vm = import 'vm.jsonnet'; local vm_bench = import 'vm-bench.jsonnet'; local vm_native = import 'vm-native.jsonnet'; local graal_common = import '../../../ci/ci_common/common.jsonnet'; -local common_json = composable(import '../../../common.json'); -local jdks = common_json.jdks; { vm_java_17:: graal_common.labsjdk17 + vm_common.vm_env_mixin('17'), @@ -46,8 +43,8 @@ local jdks = common_json.jdks; maven_17_20:: { downloads+: { - JAVA_HOME: jdks['labsjdk-ce-17'], - EXTRA_JAVA_HOMES: jdks['labsjdk-ce-20'], + JAVA_HOME: graal_common.jdks_data['labsjdk-ce-17'], + EXTRA_JAVA_HOMES: graal_common.jdks_data['labsjdk-ce-20'], }, mx_cmd_base:: ['mx', '--dynamicimports', '/tools,/compiler,/graal-js,/espresso,/substratevm', '--disable-installables=true', '--force-bash-launcher=true', '--skip-libraries=true'], build:: self.mx_cmd_base + ['build'], @@ -124,7 +121,7 @@ local jdks = common_json.jdks; ], name: 'gate-vm-unittest-linux-amd64', }, ['sdk', 'truffle', 'vm']), - utils.add_gate_predicate(self.vm_java_20 + common_json.devkits['windows-jdk20'] + vm_common.gate_vm_windows_amd64 + { + utils.add_gate_predicate(self.vm_java_20 + graal_common.devkits['windows-jdk20'] + vm_common.gate_vm_windows_amd64 + { run: [ ['mx', 'build'], ['mx', 'unittest', '--suite', 'vm'], diff --git a/wasm/ci/ci.jsonnet b/wasm/ci/ci.jsonnet index a2d6ded30104..d8e33f082ee4 100644 --- a/wasm/ci/ci.jsonnet +++ b/wasm/ci/ci.jsonnet @@ -7,14 +7,14 @@ builds: [ # Gates. - common.jdk17_gate_linux_eclipse_jdt + common.gate_graalwasm + common.amd64 + {environment+: {GATE_TAGS: 'style,fullbuild'}} + {name: 'gate-graalwasm-style-fullbuild-linux-amd64'}, - common.jdk17_gate_linux_wabt + common.gate_graalwasm_full + common.amd64 + {environment+: {GATE_TAGS: 'build,wasmtest'}} + {name: 'gate-graalwasm-unittest-linux-amd64'}, - common.jdk17_gate_linux_wabt_emsdk + common.gate_graalwasm_emsdk_full + common.amd64 + {environment+: {GATE_TAGS: 'buildall,wasmextratest'}} + {name: 'gate-graalwasm-extra-unittest-linux-amd64'}, - common.jdk17_gate_linux_wabt_emsdk + common.gate_graalwasm_emsdk_full + common.amd64 + {environment+: {GATE_TAGS: 'buildall,wasmbenchtest'}} + {name: 'gate-graalwasm-benchtest-linux-amd64'}, - common.jdk17_gate_windows_wabt + common.gate_graalwasm_full + common.amd64 + {environment+: {GATE_TAGS: 'build,wasmtest'}} + {name: 'gate-graalwasm-unittest-windows-amd64'} + common.devkits["windows-jdk17"], + common.jdk17_gate_linux_amd64_eclipse_jdt + common.gate_graalwasm + {environment+: {GATE_TAGS: 'style,fullbuild'}} {name: 'gate-graalwasm-style-fullbuild-linux-amd64'}, + common.jdk17_gate_linux_amd64_wabt + common.gate_graalwasm_full + {environment+: {GATE_TAGS: 'build,wasmtest'}} {name: 'gate-graalwasm-unittest-linux-amd64'}, + common.jdk17_gate_linux_amd64_wabt_emsdk + common.gate_graalwasm_emsdk_full + {environment+: {GATE_TAGS: 'buildall,wasmextratest'}} {name: 'gate-graalwasm-extra-unittest-linux-amd64'}, + common.jdk17_gate_linux_amd64_wabt_emsdk + common.gate_graalwasm_emsdk_full + {environment+: {GATE_TAGS: 'buildall,wasmbenchtest'}} {name: 'gate-graalwasm-benchtest-linux-amd64'}, + common.jdk17_gate_windows_amd64_wabt + common.gate_graalwasm_full + {environment+: {GATE_TAGS: 'build,wasmtest'}} {name: 'gate-graalwasm-unittest-windows-amd64'} + common.devkits["windows-jdk17"], # Benchmark jobs. - common.jdk17_bench_linux_wabt_emsdk + common.bench_graalwasm_emsdk_full + common.amd64 + { + common.jdk17_bench_linux_amd64_wabt_emsdk + common.bench_graalwasm_emsdk_full + { name: 'bench-graalwasm-c-micro-linux-amd64', environment+: { BENCH_RUNNER: 'run-c-micro-benchmarks', diff --git a/wasm/ci/ci_common/common.jsonnet b/wasm/ci/ci_common/common.jsonnet index 108b181dd045..67cc6483ca22 100644 --- a/wasm/ci/ci_common/common.jsonnet +++ b/wasm/ci/ci_common/common.jsonnet @@ -5,12 +5,9 @@ local wasm_suite_root = root_ci.wasm_suite_root; local graal_suite_root = root_ci.graal_suite_root; { - local mx = (import "../../../common.json").mx_version, local common = (import "../../../ci/ci_common/common.jsonnet"), - local common_json = (import "../../../common.json"), - local utils = (import "../../../ci/ci_common/common-utils.libsonnet"), - devkits: utils.composable(common_json.devkits), + devkits: common.devkits, gate: { targets+: ['gate'], @@ -20,60 +17,18 @@ local graal_suite_root = root_ci.graal_suite_root; targets+: ['bench', 'daily'], }, - common: { - environment+: { - MX_PYTHON: common_json.deps.common.environment["MX_PYTHON"], - }, - packages+: common_json.deps.common.packages + { - 'mx': mx, - 'pip:ninja_syntax': '==1.7.2', - }, - }, - - linux: self.common + { + linux_amd64: common.linux_amd64 + { packages+: { "01:binutils": '>=2.30', - git: '>=1.8.3', gcc: '==8.3.0', 'gcc-build-essentials': '==8.3.0', # GCC 4.9.0 fails on cluster make: '>=3.83', llvm: '==8.0.1', nodejs: '==8.9.4', }, - capabilities+: ['linux'], - docker: { - "image": "buildslave_ol7", - "mount_modules": true - }, - }, - - windows: self.common + { - capabilities+: ['windows'], - }, - - amd64: { - capabilities+: ['amd64'], - }, - - aarch64: { - capabilities+: ['aarch64'], - timelimit: '1:30:00' - }, - - eclipse: { - downloads+: { - ECLIPSE: {name: 'eclipse', version: '4.14.0', platformspecific: true}, - }, - environment+: { - ECLIPSE_EXE: '$ECLIPSE/eclipse', - }, }, - jdt: { - environment+: { - JDT: "builtin", - }, - }, + windows_amd64: common.windows_amd64, wabt: { downloads+: { @@ -171,13 +126,14 @@ local graal_suite_root = root_ci.graal_suite_root; ], logs: ['bench-results.json'], capabilities+: ['x52'], + timelimit: '1:00:00', }, - jdk17_gate_linux_eclipse_jdt : common.labsjdk17 + self.gate + self.linux + self.eclipse + self.jdt, - jdk17_gate_linux_wabt : common.labsjdk17 + self.gate + self.linux + self.wabt, - jdk17_gate_linux_wabt_emsdk : common.labsjdk17 + self.gate + self.linux + self.wabt + self.emsdk, - jdk17_gate_linux_wabt_emsdk_ocamlbuild : common.labsjdk17 + self.gate + self.linux + self.wabt + self.emsdk + self.ocamlbuild, - jdk17_bench_linux_wabt_emsdk : common.labsjdk17 + self.bench + self.linux + self.wabt + self.emsdk, - jdk17_bench_linux_wabt_emsdk_nodejs : common.labsjdk17 + self.bench + self.linux + self.wabt + self.emsdk + self.nodejs, - jdk17_gate_windows_wabt : common.labsjdk17 + self.gate + self.windows + self.wabt, + jdk17_gate_linux_amd64_eclipse_jdt : common.labsjdk17 + self.gate + self.linux_amd64 + common.deps.eclipse + common.deps.jdt, + jdk17_gate_linux_amd64_wabt : common.labsjdk17 + self.gate + self.linux_amd64 + self.wabt, + jdk17_gate_linux_amd64_wabt_emsdk : common.labsjdk17 + self.gate + self.linux_amd64 + self.wabt + self.emsdk, + jdk17_gate_linux_amd64_wabt_emsdk_ocamlbuild : common.labsjdk17 + self.gate + self.linux_amd64 + self.wabt + self.emsdk + self.ocamlbuild, + jdk17_bench_linux_amd64_wabt_emsdk : common.labsjdk17 + self.bench + self.linux_amd64 + self.wabt + self.emsdk, + jdk17_bench_linux_amd64_wabt_emsdk_nodejs : common.labsjdk17 + self.bench + self.linux_amd64 + self.wabt + self.emsdk + self.nodejs, + jdk17_gate_windows_amd64_wabt : common.labsjdk17 + self.gate + self.windows_amd64 + self.wabt, }