[GR-44299] Move global-common CI config from common.json to ci/common…

….jsonnet

.github/workflows/main.yml

@@ -161,17 +161,18 @@ jobs:
       if: ${{ env.MX_RUNS_STYLE == 'true' }}
       run: |
         sudo apt install python3-pip python-setuptools
-        jq -r '.deps.common.packages | to_entries[] | select(.key | startswith("pip:")) | (.key | split(":")[1]) + .value' common.json |
-          xargs sudo pip install
+        sudo pip install ninja_syntax $(jq -r '.pip.ninja_syntax' common.json)
+        sudo pip install lazy-object-proxy $(jq -r '.pip["lazy-object-proxy"]' common.json)
+        sudo pip install pylint $(jq -r '.pip.pylint' common.json)
     - name: Install additional pip packages
       if: ${{ matrix.env.PIP_PACKAGES != '' }}
       run: ${MX_PYTHON} -m pip install ${{ matrix.env.PIP_PACKAGES }}
     - name: Download Eclipse
       if: ${{ env.MX_RUNS_STYLE == 'true' }}
       run: |
         ECLIPSE_TAR=eclipse.tar.gz
-        ECLIPSE_ORG_VERSION=$(jq -r '.downloads.eclipse.eclipse_org.version' common.json)
-        ECLIPSE_ORG_TIMESTAMP=$(jq -r '.downloads.eclipse.eclipse_org.timestamp' common.json)
+        ECLIPSE_ORG_VERSION=$(jq -r '.eclipse.short_version' common.json)
+        ECLIPSE_ORG_TIMESTAMP=$(jq -r '.eclipse.timestamp' common.json)
         wget --no-verbose https://archive.eclipse.org/eclipse/downloads/drops4/R-${ECLIPSE_ORG_VERSION}-${ECLIPSE_ORG_TIMESTAMP}/eclipse-SDK-${ECLIPSE_ORG_VERSION}-linux-gtk-x86_64.tar.gz -O $ECLIPSE_TAR
         tar -xzf ${ECLIPSE_TAR}
         echo "ECLIPSE_EXE=${PWD}/eclipse/eclipse" >> $GITHUB_ENV

ci.jsonnet

@@ -41,8 +41,8 @@ local visualizer = import 'visualizer/ci/ci.jsonnet';
 local verify_ci = (import 'ci/ci_common/ci-check.libsonnet').verify_ci;
 
 {
-  # Ensure that entries in common.jsonnet can be resolved.
-  _checkCommon: (import 'ci/ci_common/common.jsonnet'),
+  # Ensure that non-hidden entries in ci/common.jsonnet and ci/ci_common/common.jsonnet can be resolved.
+  assert std.length(std.toString(import 'ci/ci_common/common.jsonnet')) > 0,
   ci_resources:: (import 'ci/ci_common/ci-resources.libsonnet'),
   overlay: graal_common.ci.overlay,
   specVersion: "3",

ci/README.md

@@ -0,0 +1,12 @@
+# CI Shared Files
+
+## Files shared between many repositories
+
+* `common.json`
+* `ci/common.jsonnet`
+
+## Files shared between graal and graal-enterprise
+
+* `graal-common.json`
+* `ci/ci_common`
+* `<suite>/ci/ci_common` in various suites

ci/ci_common/bench-common.libsonnet

@@ -1,5 +1,5 @@
 {
-  local common = import "ci/ci_common/common.jsonnet",
+  local common = import "../../ci/ci_common/common.jsonnet",
   local utils = import "common-utils.libsonnet",
 
   # benchmark job base with automatically generated name
@@ -43,26 +43,26 @@
       threads_per_node:: if self.is_numa then self.num_threads / std.length(self.numa_nodes) else self.num_threads,
     },
 
-    x52:: common.linux + common.amd64 + self._bench_machine + {
+    x52:: common.linux_amd64 + self._bench_machine + {
       machine_name:: "x52",
       capabilities+: ["no_frequency_scaling", "tmpfs25g"],
       numa_nodes:: [0, 1],
       default_numa_node:: 0,
       num_threads:: 72
     },
-    x82:: common.linux + common.amd64 + self._bench_machine + {
+    x82:: common.linux_amd64 + self._bench_machine + {
       machine_name:: "x82",
       capabilities+: ["no_frequency_scaling", "tmpfs25g"],
       numa_nodes:: [0, 1],
       default_numa_node:: 0,
       num_threads:: 96
     },
-    xgene3:: common.linux + common.aarch64 + self._bench_machine + {
+    xgene3:: common.linux_aarch64 + self._bench_machine + {
       machine_name:: "xgene3",
       capabilities+: [],
       num_threads:: 32
     },
-    a12c:: common.linux + common.aarch64 + self._bench_machine + {
+    a12c:: common.linux_aarch64 + self._bench_machine + {
       machine_name:: "a12c",
       capabilities+: ["no_frequency_scaling", "tmpfs25g"],
       numa_nodes:: [0, 1],

ci/ci_common/common-utils.libsonnet

@@ -1,69 +1,4 @@
 {
-  # composable: make an object composable
-  #
-  # When composing objects with `+`, the RHS overrides the LHS when fields collide,
-  # unless the RHS defines fields with `+:`. This is especially an issue when
-  # importing nested objects from JSON, where all fields are implicitly defined
-  # with `:`. To solve this, `composable` essentially turns `:` fields into `+:`.
-  #
-  # See the following example:
-  #
-  # input.jsonnet
-  # <code>
-  # {
-  #   composable:: ...,
-  #   obj1:: {
-  #     key1: {
-  #       foo: "foo",
-  #     },
-  #     key2: "key2",
-  #   },
-  #   obj2:: {
-  #     key1: {
-  #       baz : "baz",
-  #     },
-  #     key3: "key3",
-  #   },
-  #   res1: self.obj1 + self.obj2,
-  #   res2: self.composable(self.obj1) + self.composable(self.obj2),
-  # }
-  # </code>
-  #
-  # Output of jsonnet input.jsonnet
-  # <code>
-  # {
-  #    "res1": {
-  #       "key1": {
-  #          "baz": "baz"
-  #       },
-  #       "key2": "key2",
-  #       "key3": "key3"
-  #    },
-  #    "res2": {
-  #       "key1": {
-  #          "baz": "baz",
-  #          "foo": "foo"
-  #       },
-  #       "key2": "key2",
-  #       "key3": "key3"
-  #    }
-  # }
-  # </code>
-  # Note the missing `res1.key1.foo`!
-  #
-  local _composable(o) =
-    std.foldl(function(obj, key)
-      obj +
-       if std.type(o[key]) == "object" then
-         { [key] +: _composable(o[key]) }
-       else
-         { [key] : o[key] },
-      std.objectFields(o),
-      {}
-    ),
-  # exported name
-  composable(o) :: _composable(o),
-
   # prefixes the given number with 'jdk'
   prefixed_jdk(jdk_version)::
     if jdk_version == null || std.length(std.toString(jdk_version)) == 0 then

ci/ci_common/common.jsonnet

@@ -1,77 +1,14 @@
-local composable = (import "common-utils.libsonnet").composable;
+# This file is only shared between the graal and graal-enterprise repositories.
 
-local mx_version = (import "../../common.json").mx_version;
-local common_json = composable(import "../../common.json");
+local common = import "../common.jsonnet";
 local repo_config = import '../repo-configuration.libsonnet';
-local jdks = common_json.jdks;
-local deps = common_json.deps;
-local downloads = common_json.downloads;
-
-# Finds the first integer in a string and returns it as an integer.
-local find_first_integer(versionString) =
-  local charToInt(c) =
-    std.codepoint(c) - std.codepoint("0");
-  local firstNum(s, i) =
-    assert std.length(s) > i : "No number found in string " + s;
-    local n = charToInt(s[i]);
-    if n >=0 && n < 10 then i else firstNum(s, i + 1);
-  local lastNum(s, i) =
-    if i >= std.length(s) then
-      i
-    else
-      local n = charToInt(s[i]);
-      if n < 0 || n > 9 then i else lastNum(s, i + 1);
-  local versionIndexStart = firstNum(versionString, 0);
-  local versionIndexEnd = lastNum(versionString, versionIndexStart);
-  std.parseInt(versionString[versionIndexStart:versionIndexEnd])
-;
-# jdk_version is an hidden field that can be used to generate job names
-local add_jdk_version(name) =
-  local jdk = jdks[name];
-  // this assumes that the version is the first number in the jdk.version string
-  local version = find_first_integer(jdk.version);
-  // santity check that the parsed version is also included in the name
-  assert std.length(std.findSubstr(std.toString(version), name)) == 1 : "Cannot find version %d in name %s" % [version, name];
-  { jdk_version:: version }
-;
-
-{
-
-  mx:: {
-    packages+: {
-      mx: mx_version
-    }
-  },
-
-  eclipse:: downloads.eclipse,
-  jdt:: downloads.jdt,
-  devkits:: common_json.devkits,
-
-  svm_deps:: common_json.svm.deps + repo_config.native_image.extra_deps,
 
+common + common.frequencies + {
   build_base:: {
     // holds location of CI resources that can easily be overwritten in an overlay
     ci_resources:: (import "ci/ci_common/ci-resources.libsonnet"),
   },
 
-  // Job frequencies
-  // ***************
-  on_demand:: {
-    targets+: ["ondemand"],
-  },
-  post_merge:: {
-    targets+: ["post-merge"],
-  },
-  daily:: {
-    targets+: ["daily"],
-  },
-  weekly:: {
-    targets+: ["weekly"],
-  },
-  monthly:: {
-    targets+: ["monthly"],
-  },
-
   # Add a guard to `build` that prevents it from running in the gate
   # for a PR that only touches *.md files, the docs, are config files for GitHub
   add_excludes_guard(build):: build + {
@@ -111,13 +48,7 @@ local add_jdk_version(name) =
     }
   },
 
-} + {
-  // JDK definitions
-  // ***************
-  // this adds all jdks from common.json
-  [name]: add_jdk_version(name) + { downloads+: { [if std.endsWith(name, "llvm") then "LLVM_JAVA_HOME" else "JAVA_HOME"] : jdks[name] }},
-  for name in std.objectFieldsAll(jdks)
-} + {
+} + common.jdks + {
   # Aliases to edition specific labsjdks
   labsjdk17::            self["labsjdk-" + repo_config.graalvm_edition + "-17"],
   labsjdk19::            self["labsjdk-" + repo_config.graalvm_edition + "-19"],
@@ -130,52 +61,28 @@ local add_jdk_version(name) =
   labsjdk20Debug::       self["labsjdk-" + repo_config.graalvm_edition + "-20Debug"],
   labsjdk20LLVM::        self["labsjdk-" + repo_config.graalvm_edition + "-20-llvm"],
 
-
   // Hardware definitions
   // ********************
-  common:: deps.common + self.mx + {
-    local where = if std.objectHas(self, "name") then " in " + self.name else "",
-    # enforce self.os (useful for generating job names)
-    os:: error "self.os not set" + where,
-    # enforce self.arch (useful for generating job names)
-    arch:: error "self.arch not set" + where,
-    capabilities +: [],
-    catch_files +: common_json.catch_files,
-    logs +: [
+  local graal_common_extras = common.deps.pylint + {
+    logs+: [
       "*.bgv",
-      "./" + repo_config.compiler.compiler_suite + "/graal_dumps/*/*"
-    ]
+      "./" + repo_config.compiler.compiler_suite + "/graal_dumps/*/*",
+    ],
+    timelimit: "30:00",
   },
-
-  ol7:: {
-    docker+: {
-      image: "buildslave_ol7",
-      mount_modules: true,
+  local linux_deps_extras = {
+    packages+: {
+      "apache/ant": ">=1.9.4",
     },
   },
 
-  linux::   deps.linux   + self.common + {os::"linux",   capabilities+: [self.os]},
-  darwin::  deps.darwin  + self.common + {os::"darwin",  capabilities+: [self.os]},
-  windows:: deps.windows + self.common + {os::"windows", capabilities+: [self.os]},
-  windows_server_2016:: self.windows + {capabilities+: ["windows_server_2016"]},
-
-  amd64::   { arch::"amd64",   capabilities+: [self.arch]},
-  aarch64:: { arch::"aarch64", capabilities+: [self.arch]},
-
-  linux_amd64::               self.linux               + self.amd64   + self.ol7,
-  darwin_amd64::              self.darwin              + self.amd64,
-  darwin_aarch64::            self.darwin              + self.aarch64 + {
-      # only needed until GR-22580 is resolved?
-      python_version: 3,
-  },
-  windows_amd64::             self.windows             + self.amd64,
-  windows_server_2016_amd64:: self.windows_server_2016 + self.amd64,
-  linux_aarch64::             self.linux               + self.aarch64,
+  linux_amd64: linux_deps_extras + common.linux_amd64 + graal_common_extras,
+  linux_aarch64: linux_deps_extras + common.linux_aarch64 + graal_common_extras,
+  darwin_amd64: common.darwin_amd64 + graal_common_extras,
+  darwin_aarch64: common.darwin_aarch64 + graal_common_extras,
+  windows_amd64: common.windows_amd64 + graal_common_extras,
+  windows_server_2016_amd64: common.windows_server_2016_amd64 + graal_common_extras,
 
+  // Other
   mach5_target:: {targets+: ["mach5"]},
-
-  // Utils
-  disable_proxies:: {
-    setup+: [["unset", "HTTP_PROXY", "HTTPS_PROXY", "FTP_PROXY", "NO_PROXY", "http_proxy", "https_proxy", "ftp_proxy", "no_proxy"]],
-  },
 }

ci/ci_common/run-spec-tools.libsonnet

@@ -55,7 +55,7 @@ local std_get = (import "../../ci/ci_common/common-utils.libsonnet").std_get;
     table
   ,
   // Removes the 'timelimit' property from an object.
-  // Usually, this is used to remove hard-coded (default) timelimits defined in `common.json`.
+  // Usually, this is used to remove hard-coded (default) timelimits defined in `ci/ci_common/common.jsonnet`.
   // These definitions assume that the os/arch definition comes first and will be refined later.
   // With run-spec, however, this is not true in general because the os/arch is only fixed later
   // in the pipeline. Thus, hard-coded timelimits would override any previous settings. To resolve