[GR-49556] Deploy a reduced Maven bundle.

PullRequest: graal/15854
oracle · Nov 6, 2023 · 2bef78b · 2bef78b
2 parents 30a8e53 + 8c4644b
commit 2bef78b
Show file tree

Hide file tree

Showing 3 changed files with 118 additions and 58 deletions.
diff --git a/vm/ci/ci_common/common.jsonnet b/vm/ci/ci_common/common.jsonnet
@@ -567,14 +567,14 @@ local devkits = graal_common.devkits;
       ce_licenses,
 
     legacy_mx_args:: ['--disable-installables=true'],  # `['--force-bash-launcher=true', '--skip-libraries=true']` have been replaced by arguments from `vm.maven_deploy_base_functions.mx_args(os, arch)`
-    mx_args(os, arch):: self.legacy_mx_args + vm.maven_deploy_base_functions.mx_args(os, arch),
-    mx_cmd_base(os, arch):: ['mx'] + vm.maven_deploy_base_functions.dynamic_imports(os, arch) + self.mx_args(os, arch),
-    mx_cmd_base_only_native(os, arch):: ['mx', '--dynamicimports', '/substratevm'] + self.mx_args(os, arch) + ['--native-images=false'],  # `--native-images=false` takes precedence over `self.mx_args(os, arch)`
+    mx_args(os, arch, reduced):: self.legacy_mx_args + vm.maven_deploy_base_functions.mx_args(os, arch, reduced),
+    mx_cmd_base(os, arch, reduced):: ['mx'] + vm.maven_deploy_base_functions.dynamic_imports(os, arch) + self.mx_args(os, arch, reduced),
+    mx_cmd_base_only_native(os, arch, reduced):: ['mx', '--dynamicimports', '/substratevm'] + self.mx_args(os, arch, reduced) + ['--native-images=false'],  # `--native-images=false` takes precedence over `self.mx_args(os, arch)`
 
     only_native_dists:: 'TRUFFLE_NFI_NATIVE,SVM_HOSTED_NATIVE',
 
-    build(os, arch, mx_args=[], build_args=[]):: [
-      self.mx_cmd_base(os, arch) + mx_args + ['build'] + build_args,
+    build(os, arch, reduced, mx_args=[], build_args=[]):: [
+      self.mx_cmd_base(os, arch, reduced) + mx_args + ['build'] + build_args,
     ],
 
     pd_layouts_archive_name(platform):: 'pd-layouts-' + platform + '.tgz',
@@ -584,12 +584,10 @@ local devkits = graal_common.devkits;
     mvn_args: ['maven-deploy', '--tags=public', '--all-distribution-types', '--validate=full', '--version-suite=vm'],
     mvn_args_only_native: self.mvn_args + ['--all-suites', '--only', self.only_native_dists],
 
-    main_platform:: 'linux-amd64',
-    other_platforms:: ['linux-aarch64', 'darwin-amd64', 'darwin-aarch64', 'windows-amd64'],
-    is_main_platform(os, arch):: os + '-' + arch == self.main_platform,
+    compose_platform(os, arch):: os + '-' + arch,
 
-    deploy_ce(os, arch, dry_run, extra_args, extra_mx_args=[]):: [
-      self.mx_cmd_base(os, arch)
+    deploy_ce(os, arch, reduced, dry_run, extra_args, extra_mx_args=[]):: [
+      self.mx_cmd_base(os, arch, reduced)
       + $.maven_deploy_base_functions.ce_suites(os,arch)
       + extra_mx_args
       + self.mvn_args
@@ -598,8 +596,8 @@ local devkits = graal_common.devkits;
       + extra_args,
     ],
 
-    deploy_ee(os, arch, dry_run, extra_args, extra_mx_args=[]):: [
-      self.mx_cmd_base(os, arch)
+    deploy_ee(os, arch, reduced, dry_run, extra_args, extra_mx_args=[]):: [
+      self.mx_cmd_base(os, arch, reduced)
       + vm.maven_deploy_base_functions.ee_suites(os, arch)
       + extra_mx_args
       + self.mvn_args
@@ -608,73 +606,131 @@ local devkits = graal_common.devkits;
       + extra_args,
     ],
 
-    deploy_only_native(os, arch, dry_run, extra_args, extra_mx_args=[]):: [
-      self.mx_cmd_base_only_native(os, arch)
+    deploy_only_native(os, arch, reduced, dry_run, extra_args, extra_mx_args=[]):: [
+      self.mx_cmd_base_only_native(os, arch, reduced)
       + extra_mx_args
       + self.mvn_args_only_native
       + ['--licenses', $.maven_deploy_base_functions.ce_licenses()]
       + (if dry_run then ['--dry-run'] else [])
       + extra_args,
     ],
 
-    run_block(os, arch, dry_run, remote_mvn_repo, remote_non_mvn_repo, local_repo)::
-      if (self.is_main_platform(os, arch)) then (
+    run_block(os, arch, dry_run, remote_mvn_repo, remote_non_mvn_repo, local_repo, main_platform, other_platforms, mvn_artifacts=true, mvn_bundle=true, mvn_reduced_bundle=true)::
+      local multiplatform_build(reduced) = self.build(os, arch, reduced, mx_args=['--multi-platform-layout-directories=' + std.join(',', [main_platform] + other_platforms)], build_args=['--targets={MAVEN_TAG_DISTRIBUTIONS:public}']);  # `self.only_native_dists` are in `{MAVEN_TAG_DISTRIBUTIONS:public}`
+
+      local mvn_artifacts_snippet =
+        # remotely deploy only the suites that are defined in the current repository, to avoid duplicated deployments
+        if (vm.maven_deploy_base_functions.edition == 'ce') then
+          self.deploy_ce(os, arch, false, dry_run, [remote_mvn_repo])
+        else
+          self.deploy_ee(os, arch, false, dry_run, ['--dummy-javadoc', remote_mvn_repo])
+          + self.deploy_ee(os, arch, false, dry_run, ['--dummy-javadoc', '--only', 'JS_ISOLATE,JS_ISOLATE_RESOURCES,TOOLS_COMMUNITY,LANGUAGES_COMMUNITY', remote_mvn_repo], extra_mx_args=['--suite', 'graal-js']);
+
+      local mvn_bundle_snippet =
         [
-          self.mx_cmd_base(os, arch) + ['restore-pd-layouts', self.pd_layouts_archive_name(platform)] for platform in self.other_platforms
-        ]
-        + self.build(os, arch, mx_args=['--multi-platform-layout-directories=' + std.join(',', [self.main_platform] + self.other_platforms)], build_args=['--targets={MAVEN_TAG_DISTRIBUTIONS:public}'])  # `self.only_native_dists` are in `{MAVEN_TAG_DISTRIBUTIONS:public}`
-        + (
-          # remotely deploy only the suites that are defined in the current repository, to avoid duplicated deployments
-          if (vm.maven_deploy_base_functions.edition == 'ce') then
-            self.deploy_ce(os, arch, dry_run, [remote_mvn_repo])
-          else
-            self.deploy_ee(os, arch, dry_run, ['--dummy-javadoc', remote_mvn_repo])
-            + self.deploy_ee(os, arch, dry_run, ['--dummy-javadoc', '--only', 'JS_ISOLATE,JS_ISOLATE_RESOURCES,TOOLS_COMMUNITY,LANGUAGES_COMMUNITY', remote_mvn_repo], extra_mx_args=['--suite', 'graal-js'])
-        )
-        + [
-          # resource bundle
-          ['set-export', 'VERSION_STRING', self.mx_cmd_base(os, arch) + ['--quiet', 'graalvm-version']],
-          ['set-export', 'LOCAL_MAVEN_REPO_REL_PATH', 'maven-resource-bundle-' + vm.maven_deploy_base_functions.edition + '-${VERSION_STRING}'],
+          # Set env vars
+          ['set-export', 'VERSION_STRING', self.mx_cmd_base(os, arch, reduced=false) + ['--quiet', 'graalvm-version']],
+          ['set-export', 'LOCAL_MAVEN_REPO_REL_PATH', 'maven-bundle-' + vm.maven_deploy_base_functions.edition + '-${VERSION_STRING}'],
           ['set-export', 'LOCAL_MAVEN_REPO_URL', ['mx', '--quiet', 'local-path-to-url', '${LOCAL_MAVEN_REPO_REL_PATH}']],
         ]
         + (
-          # locally deploy all relevant suites
+          # Locally deploy all relevant suites
           if (vm.maven_deploy_base_functions.edition == 'ce') then
-            self.deploy_ce(os, arch, dry_run, [local_repo, '${LOCAL_MAVEN_REPO_URL}'])
+            self.deploy_ce(os, arch, false, dry_run, [local_repo, '${LOCAL_MAVEN_REPO_URL}'])
           else
-            self.deploy_ce(os, arch, dry_run, ['--dummy-javadoc', '--skip', 'JS_ISOLATE,JS_ISOLATE_RESOURCES,TOOLS_COMMUNITY,LANGUAGES_COMMUNITY', local_repo, '${LOCAL_MAVEN_REPO_URL}'])
-            + self.deploy_ee(os, arch, dry_run, ['--dummy-javadoc', '--only', 'JS_ISOLATE,JS_ISOLATE_RESOURCES,TOOLS_COMMUNITY,LANGUAGES_COMMUNITY', local_repo, '${LOCAL_MAVEN_REPO_URL}'], extra_mx_args=['--suite', 'graal-js'])
-            + self.deploy_ee(os, arch, dry_run, ['--dummy-javadoc', local_repo, '${LOCAL_MAVEN_REPO_URL}'])
+            self.deploy_ce(os, arch, false, dry_run, ['--dummy-javadoc', '--skip', 'JS_ISOLATE,JS_ISOLATE_RESOURCES,TOOLS_COMMUNITY,LANGUAGES_COMMUNITY', local_repo, '${LOCAL_MAVEN_REPO_URL}'])
+            + self.deploy_ee(os, arch, false, dry_run, ['--dummy-javadoc', '--only', 'JS_ISOLATE,JS_ISOLATE_RESOURCES,TOOLS_COMMUNITY,LANGUAGES_COMMUNITY', local_repo, '${LOCAL_MAVEN_REPO_URL}'], extra_mx_args=['--suite', 'graal-js'])
+            + self.deploy_ee(os, arch, false, dry_run, ['--dummy-javadoc', local_repo, '${LOCAL_MAVEN_REPO_URL}'])
         )
         + (
+          # Archive and deploy
           if (dry_run) then
-            [['echo', 'Skipping the archiving and the final maven deployment']]
+            [['echo', 'Skipping the archiving and the final deployment of the Maven bundle']]
           else [
-            ['set-export', 'MAVEN_RESOURCE_BUNDLE', '${LOCAL_MAVEN_REPO_REL_PATH}'],
-            ['mx', 'build', '--targets', 'MAVEN_RESOURCE_BUNDLE'],
+            ['set-export', 'MAVEN_BUNDLE_PATH', '${LOCAL_MAVEN_REPO_REL_PATH}'],
+            ['set-export', 'MAVEN_BUNDLE_ARTIFACT_ID', 'maven-bundle-' + vm.maven_deploy_base_functions.edition],
+            ['mx', 'build', '--targets', 'MAVEN_BUNDLE'],
             ['mx', '--suite', 'vm', 'maven-deploy', '--tags=resource-bundle', '--all-distribution-types', '--validate=none', '--with-suite-revisions-metadata', remote_non_mvn_repo],
           ]
+        );
+
+      local mvn_reduced_bundle_snippet =
+        if (vm.maven_deploy_base_functions.edition == 'ce') then
+          [['echo', 'Skipping the archiving and the final deployment of the reduced Maven bundle']]
+        else (
+          [
+            # Set env vars
+            ['set-export', 'VERSION_STRING', self.mx_cmd_base(os, arch, reduced=true) + ['--quiet', 'graalvm-version']],
+            ['set-export', 'LOCAL_MAVEN_REDUCED_REPO_REL_PATH', 'maven-reduced-bundle-' + vm.maven_deploy_base_functions.edition + '-${VERSION_STRING}'],
+            ['set-export', 'LOCAL_MAVEN_REDUCED_REPO_URL', ['mx', '--quiet', 'local-path-to-url', '${LOCAL_MAVEN_REDUCED_REPO_REL_PATH}']],
+          ]
+          + (
+            multiplatform_build(reduced=true)
+          )
+          + (
+            # Locally deploy all relevant suites
+            self.deploy_ce(os, arch, true, dry_run, ['--dummy-javadoc', '--only', vm.maven_deploy_base_functions.reduced_ce_dists, local_repo, '${LOCAL_MAVEN_REDUCED_REPO_URL}'])
+            + self.deploy_ee(os, arch, true, dry_run, ['--dummy-javadoc', '--only', vm.maven_deploy_base_functions.reduced_ee_dists, local_repo, '${LOCAL_MAVEN_REDUCED_REPO_URL}'], extra_mx_args=['--suite', 'graal-js'])
+          )
+          + (
+            # Archive and deploy
+            if (dry_run) then
+              [['echo', 'Skipping the archiving and the final deployment of the reduced Maven bundle']]
+            else [
+              ['set-export', 'MAVEN_BUNDLE_PATH', '${LOCAL_MAVEN_REDUCED_REPO_REL_PATH}'],
+              ['set-export', 'MAVEN_BUNDLE_ARTIFACT_ID', 'maven-reduced-bundle-' + vm.maven_deploy_base_functions.edition + (if (std.length(other_platforms) == 0) then '-' + main_platform else '')],
+              ['mx', 'build', '--targets', 'MAVEN_BUNDLE'],
+              ['mx', '--suite', 'vm', 'maven-deploy', '--tags=resource-bundle', '--all-distribution-types', '--validate=none', '--with-suite-revisions-metadata', remote_non_mvn_repo],
+            ]
+          )
+        );
+
+      if (self.compose_platform(os, arch) == main_platform) then (
+        [self.mx_cmd_base(os, arch, reduced=false) + ['restore-pd-layouts', self.pd_layouts_archive_name(platform)] for platform in other_platforms]
+        + (
+          if (mvn_artifacts || mvn_bundle) then
+            multiplatform_build(reduced=false)
+          else
+            [['echo', 'Skipping the full build']]
+        )
+        + (
+          if (mvn_artifacts) then
+            mvn_artifacts_snippet
+          else
+            [['echo', 'Skipping Maven artifacts']]
+        )
+        + (
+          if (mvn_bundle) then
+            mvn_bundle_snippet
+          else
+            [['echo', 'Skipping Maven bundle']]
+        )
+        + (
+          if (mvn_reduced_bundle) then
+            mvn_reduced_bundle_snippet
+          else
+          [['echo', 'Skipping reduced Maven bundle']]
         )
       ) else (
-        self.build(os, arch, build_args=['--targets=' + self.only_native_dists + ',{PLATFORM_DEPENDENT_LAYOUT_DIR_DISTRIBUTIONS}'])
+        self.build(os, arch, reduced=false, build_args=['--targets=' + self.only_native_dists + ',{PLATFORM_DEPENDENT_LAYOUT_DIR_DISTRIBUTIONS}'])
         + (
           if (vm.maven_deploy_base_functions.edition == 'ce') then
-            self.deploy_only_native(os, arch, dry_run, [remote_mvn_repo])
+            self.deploy_only_native(os, arch, reduced=false, dry_run=dry_run, extra_args=[remote_mvn_repo])
           else
             [['echo', 'Skipping the deployment of ' + self.only_native_dists + ': It is already deployed by the ce job']]
         )
-        + [self.mx_cmd_base(os, arch) + ['archive-pd-layouts', self.pd_layouts_archive_name(os + '-' + arch)]]
+        + [self.mx_cmd_base(os, arch, reduced=false) + ['archive-pd-layouts', self.pd_layouts_archive_name(os + '-' + arch)]]
       ),
 
-    base_object(os, arch, dry_run, remote_mvn_repo, remote_non_mvn_repo, local_repo):: {
-      run: $.maven_deploy_base_functions.run_block(os, arch, dry_run, remote_mvn_repo, remote_non_mvn_repo, local_repo),
-    } + if (self.is_main_platform(os, arch)) then {
+    base_object(os, arch, dry_run, remote_mvn_repo, remote_non_mvn_repo, local_repo, main_platform='linux-amd64', other_platforms=['linux-aarch64', 'darwin-amd64', 'darwin-aarch64', 'windows-amd64'],):: {
+      run: $.maven_deploy_base_functions.run_block(os, arch, dry_run, remote_mvn_repo, remote_non_mvn_repo, local_repo, main_platform, other_platforms),
+    } + if (self.compose_platform(os, arch) == main_platform) then {
        requireArtifacts+: [
          {
            name: $.maven_deploy_base_functions.pd_layouts_artifact_name(platform, dry_run),
            dir: vm.vm_dir,
            autoExtract: true,
-         } for platform in $.maven_deploy_base_functions.other_platforms
+         } for platform in other_platforms
        ],
      }
     else {

diff --git a/vm/ci/ci_includes/vm.jsonnet b/vm/ci/ci_includes/vm.jsonnet
@@ -16,6 +16,7 @@ local graal_common = import '../../../ci/ci_common/common.jsonnet';
 
   binaries_repository: 'lafo',
   maven_deploy_repository: 'lafo-maven',
+  edition:: 'ce',
   vm_dir:: 'vm',
   svm_suite:: '/substratevm',
   libgraal_env: 'libgraal',
@@ -32,9 +33,9 @@ local graal_common = import '../../../ci/ci_common/common.jsonnet';
   check_graalvm_base_build(path, os, arch, java_version): [],
 
   vm_setup:: {
-    short_name:: 'ce',
+    short_name:: $.edition,
     setup+: [
-      ['set-export', 'VM_ENV', 'ce'],
+      ['set-export', 'VM_ENV', self.short_name],
       ['set-export', 'RELEASE_CATALOG', 'https://www.graalvm.org/component-catalog/v2/graal-updater-component-catalog-java${BASE_JDK_SHORT_VERSION}.properties|{ee=GraalVM Enterprise Edition}rest://gds.oracle.com/api/20220101/'],
       ['set-export', 'RELEASE_PRODUCT_ID', 'D53FAE8052773FFAE0530F15000AA6C6'],
       ['set-export', 'SNAPSHOT_CATALOG', ['mx', 'urlrewrite', 'http://www.graalvm.org/catalog/ce/java${BASE_JDK_SHORT_VERSION}']],
@@ -78,10 +79,9 @@ local graal_common = import '../../../ci/ci_common/common.jsonnet';
   },
 
   maven_deploy_base_functions: {
-    edition:: 'ce',
+    edition:: vm.edition,
 
-    mx_args(os, arch)::
-      ['--native-images=false'],
+    mx_args(os, arch, reduced=false):: ['--native-images=false'],
 
     dynamic_imports(os, arch)::
       ['--dynamicimports', vm_common.maven_deploy_base_functions.dynamic_ce_imports(os, arch)],
@@ -91,6 +91,9 @@ local graal_common = import '../../../ci/ci_common/common.jsonnet';
 
     ee_licenses()::
       error 'The vm suite does not define ee licenses',
+
+    reduced_ce_dists:: error 'The vm suite does not define reduced dists',
+    reduced_ee_dists:: error 'The vm suite does not define reduced dists',
   },
 
   local builds = [

diff --git a/vm/mx.vm/mx_vm.py b/vm/mx.vm/mx_vm.py
@@ -468,15 +468,16 @@ def isBenchmarkProject(self):
         register_community_tools_distribution(_suite, register_distribution)
         register_community_languages_distribution(_suite, register_distribution)
 
-    maven_resource_bundle = mx.get_env('MAVEN_RESOURCE_BUNDLE')
-    if register_distribution and maven_resource_bundle is not None:
-        register_distribution(mx.LayoutTARDistribution(_suite, 'MAVEN_RESOURCE_BUNDLE', [], {
-            './': 'file:' + os.path.realpath(maven_resource_bundle)
+    maven_bundle_path = mx.get_env('MAVEN_BUNDLE_PATH')
+    maven_bundle_artifact_id = mx.get_env('MAVEN_BUNDLE_ARTIFACT_ID')
+    if bool(maven_bundle_path) != bool(maven_bundle_artifact_id):
+        mx.abort(f"Both $MAVEN_BUNDLE_PATH and $MAVEN_BUNDLE_ARTIFACT_ID must be either set or not set. Got:\n$MAVEN_BUNDLE_PATH={'' if maven_bundle_path is None else maven_bundle_path}\n$MAVEN_BUNDLE_ARTIFACT_ID={'' if maven_bundle_artifact_id is None else maven_bundle_artifact_id}")
+    if register_distribution and maven_bundle_path is not None:
+        register_distribution(mx.LayoutTARDistribution(_suite, 'MAVEN_BUNDLE', [], {
+            './': 'file:' + os.path.realpath(maven_bundle_path)
         }, None, True, None, maven={
             'groupId': 'org.graalvm.polyglot',
-            'artifactId': 'maven-{edition}-resource-bundle'.format(
-                edition='ee' if mx.suite('vm-enterprise', fatalIfMissing=False) else 'ce',
-            ),
+            'artifactId': maven_bundle_artifact_id,
             'version': mx_sdk_vm_impl.graalvm_version('graalvm'),
             'tag': 'resource-bundle',
         }))