From e52918b2940ecef26087ea313cbffa4d4fc7c514 Mon Sep 17 00:00:00 2001 From: Danilo Ansaloni Date: Fri, 20 Oct 2023 13:39:25 +0200 Subject: [PATCH 1/9] The maven bundle dist does not have a fixed name. Also rename `resource-bundle` to `bundle`. --- vm/ci/ci_common/common.jsonnet | 9 +++++---- vm/ci/ci_includes/vm.jsonnet | 7 ++++--- vm/mx.vm/mx_vm.py | 15 ++++++++------- 3 files changed, 17 insertions(+), 14 deletions(-) diff --git a/vm/ci/ci_common/common.jsonnet b/vm/ci/ci_common/common.jsonnet index 9dc6d7f375fe..5670f0b59155 100644 --- a/vm/ci/ci_common/common.jsonnet +++ b/vm/ci/ci_common/common.jsonnet @@ -631,9 +631,9 @@ local devkits = graal_common.devkits; + self.deploy_ee(os, arch, dry_run, ['--dummy-javadoc', '--only', 'JS_ISOLATE,JS_ISOLATE_RESOURCES,TOOLS_COMMUNITY,LANGUAGES_COMMUNITY', remote_mvn_repo], extra_mx_args=['--suite', 'graal-js']) ) + [ - # resource bundle + # Maven bundle ['set-export', 'VERSION_STRING', self.mx_cmd_base(os, arch) + ['--quiet', 'graalvm-version']], - ['set-export', 'LOCAL_MAVEN_REPO_REL_PATH', 'maven-resource-bundle-' + vm.maven_deploy_base_functions.edition + '-${VERSION_STRING}'], + ['set-export', 'LOCAL_MAVEN_REPO_REL_PATH', 'maven-bundle-' + vm.maven_deploy_base_functions.edition + '-${VERSION_STRING}'], ['set-export', 'LOCAL_MAVEN_REPO_URL', ['mx', '--quiet', 'local-path-to-url', '${LOCAL_MAVEN_REPO_REL_PATH}']], ] + ( @@ -649,8 +649,9 @@ local devkits = graal_common.devkits; if (dry_run) then [['echo', 'Skipping the archiving and the final maven deployment']] else [ - ['set-export', 'MAVEN_RESOURCE_BUNDLE', '${LOCAL_MAVEN_REPO_REL_PATH}'], - ['mx', 'build', '--targets', 'MAVEN_RESOURCE_BUNDLE'], + ['set-export', 'MAVEN_BUNDLE_PATH', '${LOCAL_MAVEN_REPO_REL_PATH}'], + ['set-export', 'MAVEN_BUNDLE_ARTIFACT_ID', 'maven-bundle-' + vm.maven_deploy_base_functions.edition], + ['mx', 'build', '--targets', 'MAVEN_BUNDLE'], ['mx', '--suite', 'vm', 'maven-deploy', '--tags=resource-bundle', '--all-distribution-types', '--validate=none', '--with-suite-revisions-metadata', remote_non_mvn_repo], ] ) diff --git a/vm/ci/ci_includes/vm.jsonnet b/vm/ci/ci_includes/vm.jsonnet index 810ad59fc9d4..2bcdccd8191e 100644 --- a/vm/ci/ci_includes/vm.jsonnet +++ b/vm/ci/ci_includes/vm.jsonnet @@ -16,6 +16,7 @@ local graal_common = import '../../../ci/ci_common/common.jsonnet'; binaries_repository: 'lafo', maven_deploy_repository: 'lafo-maven', + edition:: 'ce', vm_dir:: 'vm', svm_suite:: '/substratevm', libgraal_env: 'libgraal', @@ -32,9 +33,9 @@ local graal_common = import '../../../ci/ci_common/common.jsonnet'; check_graalvm_base_build(path, os, arch, java_version): [], vm_setup:: { - short_name:: 'ce', + short_name:: $.edition, setup+: [ - ['set-export', 'VM_ENV', 'ce'], + ['set-export', 'VM_ENV', self.short_name], ['set-export', 'RELEASE_CATALOG', 'https://www.graalvm.org/component-catalog/v2/graal-updater-component-catalog-java${BASE_JDK_SHORT_VERSION}.properties|{ee=GraalVM Enterprise Edition}rest://gds.oracle.com/api/20220101/'], ['set-export', 'RELEASE_PRODUCT_ID', 'D53FAE8052773FFAE0530F15000AA6C6'], ['set-export', 'SNAPSHOT_CATALOG', ['mx', 'urlrewrite', 'http://www.graalvm.org/catalog/ce/java${BASE_JDK_SHORT_VERSION}']], @@ -78,7 +79,7 @@ local graal_common = import '../../../ci/ci_common/common.jsonnet'; }, maven_deploy_base_functions: { - edition:: 'ce', + edition:: vm.edition, mx_args(os, arch):: ['--native-images=false'], diff --git a/vm/mx.vm/mx_vm.py b/vm/mx.vm/mx_vm.py index 1af49161485f..db4b0894d1a1 100644 --- a/vm/mx.vm/mx_vm.py +++ b/vm/mx.vm/mx_vm.py @@ -465,15 +465,16 @@ def isBenchmarkProject(self): register_community_tools_distribution(_suite, register_distribution) register_community_languages_distribution(_suite, register_distribution) - maven_resource_bundle = mx.get_env('MAVEN_RESOURCE_BUNDLE') - if register_distribution and maven_resource_bundle is not None: - register_distribution(mx.LayoutTARDistribution(_suite, 'MAVEN_RESOURCE_BUNDLE', [], { - './': 'file:' + os.path.realpath(maven_resource_bundle) + maven_bundle_path = mx.get_env('MAVEN_BUNDLE_PATH') + maven_bundle_artifact_id = mx.get_env('MAVEN_BUNDLE_ARTIFACT_ID') + if bool(maven_bundle_path) != bool(maven_bundle_artifact_id): + mx.abort(f"Both $MAVEN_BUNDLE_PATH and $MAVEN_BUNDLE_ARTIFACT_ID must be either set or not set. Got:\n$MAVEN_BUNDLE_PATH={'' if maven_bundle_path is None else maven_bundle_path}\n$MAVEN_BUNDLE_ARTIFACT_ID={'' if maven_bundle_artifact_id is None else maven_bundle_artifact_id}") + if register_distribution and maven_bundle_path is not None: + register_distribution(mx.LayoutTARDistribution(_suite, 'MAVEN_BUNDLE', [], { + './': 'file:' + os.path.realpath(maven_bundle_path) }, None, True, None, maven={ 'groupId': 'org.graalvm.polyglot', - 'artifactId': 'maven-{edition}-resource-bundle'.format( - edition='ee' if mx.suite('vm-enterprise', fatalIfMissing=False) else 'ce', - ), + 'artifactId': maven_bundle_artifact_id, 'version': mx_sdk_vm_impl.graalvm_version('graalvm'), 'tag': 'resource-bundle', })) From 4d8f25129fe7f191555fb90e3ebdb674c287ba79 Mon Sep 17 00:00:00 2001 From: Danilo Ansaloni Date: Fri, 20 Oct 2023 15:20:57 +0200 Subject: [PATCH 2/9] Deploy also a reduced maven bundle. --- vm/ci/ci_common/common.jsonnet | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/vm/ci/ci_common/common.jsonnet b/vm/ci/ci_common/common.jsonnet index 5670f0b59155..bfc7546be3eb 100644 --- a/vm/ci/ci_common/common.jsonnet +++ b/vm/ci/ci_common/common.jsonnet @@ -637,23 +637,47 @@ local devkits = graal_common.devkits; ['set-export', 'LOCAL_MAVEN_REPO_URL', ['mx', '--quiet', 'local-path-to-url', '${LOCAL_MAVEN_REPO_REL_PATH}']], ] + ( - # locally deploy all relevant suites + if (vm.maven_deploy_base_functions.edition == 'ce') then + [['echo', 'Skipping the setting of the env variables for the Maven reduced bundle']] + else [ + # Maven reduced bundle + ['set-export', 'LOCAL_MAVEN_REDUCED_REPO_REL_PATH', 'maven-reduced-bundle-' + vm.maven_deploy_base_functions.edition + '-${VERSION_STRING}'], + ['set-export', 'LOCAL_MAVEN_REDUCED_REPO_URL', ['mx', '--quiet', 'local-path-to-url', '${LOCAL_MAVEN_REDUCED_REPO_REL_PATH}']], + ] + ) + + ( + # Maven bundle - Locally deploy all relevant suites if (vm.maven_deploy_base_functions.edition == 'ce') then self.deploy_ce(os, arch, dry_run, [local_repo, '${LOCAL_MAVEN_REPO_URL}']) else self.deploy_ce(os, arch, dry_run, ['--dummy-javadoc', '--skip', 'JS_ISOLATE,JS_ISOLATE_RESOURCES,TOOLS_COMMUNITY,LANGUAGES_COMMUNITY', local_repo, '${LOCAL_MAVEN_REPO_URL}']) + self.deploy_ee(os, arch, dry_run, ['--dummy-javadoc', '--only', 'JS_ISOLATE,JS_ISOLATE_RESOURCES,TOOLS_COMMUNITY,LANGUAGES_COMMUNITY', local_repo, '${LOCAL_MAVEN_REPO_URL}'], extra_mx_args=['--suite', 'graal-js']) + self.deploy_ee(os, arch, dry_run, ['--dummy-javadoc', local_repo, '${LOCAL_MAVEN_REPO_URL}']) + # Maven reduced bundle + + self.deploy_ce(os, arch, dry_run, ['--dummy-javadoc', '--only', 'TRUFFLE_API,POLYGLOT,COLLECTIONS,NATIVEIMAGE,WORD,JS_COMMUNITY,GRAALJS,TREGEX,TRUFFLE_ICU4J,TRUFFLE_RUNTIME,JNIUTILS,TRUFFLE_COMPILER,NATIVEBRIDGE,INSIGHT_HEAP,INSIGHT,TRUFFLE_DSL_PROCESSOR', local_repo, '${LOCAL_MAVEN_REDUCED_REPO_URL}']) + + self.deploy_ee(os, arch, dry_run, ['--dummy-javadoc', '--only', 'JS_ENTERPRISE,TRUFFLE_ENTERPRISE,HEAP_LANGUAGE', local_repo, '${LOCAL_MAVEN_REDUCED_REPO_URL}'], extra_mx_args=['--suite', 'graal-js']) ) + ( + # Maven bundle if (dry_run) then - [['echo', 'Skipping the archiving and the final maven deployment']] + [['echo', 'Skipping the archiving and the final deployment of the Maven bundle']] else [ ['set-export', 'MAVEN_BUNDLE_PATH', '${LOCAL_MAVEN_REPO_REL_PATH}'], ['set-export', 'MAVEN_BUNDLE_ARTIFACT_ID', 'maven-bundle-' + vm.maven_deploy_base_functions.edition], ['mx', 'build', '--targets', 'MAVEN_BUNDLE'], ['mx', '--suite', 'vm', 'maven-deploy', '--tags=resource-bundle', '--all-distribution-types', '--validate=none', '--with-suite-revisions-metadata', remote_non_mvn_repo], ] + + ( + # Maven reduced bundled + if (vm.maven_deploy_base_functions.edition == 'ce') then + [['echo', 'Skipping the archiving and the final deployment of the reduced Maven bundle']] + else [ + ['set-export', 'MAVEN_BUNDLE_PATH', '${LOCAL_MAVEN_REDUCED_REPO_REL_PATH}'], + ['set-export', 'MAVEN_BUNDLE_ARTIFACT_ID', 'maven-reduced-bundle-' + vm.maven_deploy_base_functions.edition], + ['mx', 'build', '--targets', 'MAVEN_BUNDLE'], + ['mx', '--suite', 'vm', 'maven-deploy', '--tags=resource-bundle', '--all-distribution-types', '--validate=none', '--with-suite-revisions-metadata', remote_non_mvn_repo], + ] + ) ) ) else ( self.build(os, arch, build_args=['--targets=' + self.only_native_dists + ',{PLATFORM_DEPENDENT_LAYOUT_DIR_DISTRIBUTIONS}']) From e00e31a879e113c9c8226db92ef0dec79d3c91e0 Mon Sep 17 00:00:00 2001 From: Danilo Ansaloni Date: Mon, 23 Oct 2023 13:48:40 +0200 Subject: [PATCH 3/9] Main and other platforms are configurable. No changes in the expanded JSON. --- vm/ci/ci_common/common.jsonnet | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/vm/ci/ci_common/common.jsonnet b/vm/ci/ci_common/common.jsonnet index bfc7546be3eb..9a7093af6d66 100644 --- a/vm/ci/ci_common/common.jsonnet +++ b/vm/ci/ci_common/common.jsonnet @@ -583,9 +583,7 @@ local devkits = graal_common.devkits; mvn_args: ['maven-deploy', '--tags=public', '--all-distribution-types', '--validate=full', '--version-suite=vm'], mvn_args_only_native: self.mvn_args + ['--all-suites', '--only', self.only_native_dists], - main_platform:: 'linux-amd64', - other_platforms:: ['linux-aarch64', 'darwin-amd64', 'darwin-aarch64', 'windows-amd64'], - is_main_platform(os, arch):: os + '-' + arch == self.main_platform, + compose_platform(os, arch):: os + '-' + arch, deploy_ce(os, arch, dry_run, extra_args, extra_mx_args=[]):: [ self.mx_cmd_base(os, arch) @@ -616,12 +614,12 @@ local devkits = graal_common.devkits; + extra_args, ], - run_block(os, arch, dry_run, remote_mvn_repo, remote_non_mvn_repo, local_repo):: - if (self.is_main_platform(os, arch)) then ( + run_block(os, arch, dry_run, remote_mvn_repo, remote_non_mvn_repo, local_repo, main_platform, other_platforms):: + if (self.compose_platform(os, arch) == main_platform) then ( [ - self.mx_cmd_base(os, arch) + ['restore-pd-layouts', self.pd_layouts_archive_name(platform)] for platform in self.other_platforms + self.mx_cmd_base(os, arch) + ['restore-pd-layouts', self.pd_layouts_archive_name(platform)] for platform in other_platforms ] - + self.build(os, arch, mx_args=['--multi-platform-layout-directories=' + std.join(',', [self.main_platform] + self.other_platforms)], build_args=['--targets={MAVEN_TAG_DISTRIBUTIONS:public}']) # `self.only_native_dists` are in `{MAVEN_TAG_DISTRIBUTIONS:public}` + + self.build(os, arch, mx_args=['--multi-platform-layout-directories=' + std.join(',', [main_platform] + other_platforms)], build_args=['--targets={MAVEN_TAG_DISTRIBUTIONS:public}']) # `self.only_native_dists` are in `{MAVEN_TAG_DISTRIBUTIONS:public}` + ( # remotely deploy only the suites that are defined in the current repository, to avoid duplicated deployments if (vm.maven_deploy_base_functions.edition == 'ce') then @@ -690,15 +688,15 @@ local devkits = graal_common.devkits; + [self.mx_cmd_base(os, arch) + ['archive-pd-layouts', self.pd_layouts_archive_name(os + '-' + arch)]] ), - base_object(os, arch, dry_run, remote_mvn_repo, remote_non_mvn_repo, local_repo):: { - run: $.maven_deploy_base_functions.run_block(os, arch, dry_run, remote_mvn_repo, remote_non_mvn_repo, local_repo), - } + if (self.is_main_platform(os, arch)) then { + base_object(os, arch, dry_run, remote_mvn_repo, remote_non_mvn_repo, local_repo, main_platform='linux-amd64', other_platforms=['linux-aarch64', 'darwin-amd64', 'darwin-aarch64', 'windows-amd64'],):: { + run: $.maven_deploy_base_functions.run_block(os, arch, dry_run, remote_mvn_repo, remote_non_mvn_repo, local_repo, main_platform, other_platforms), + } + if (self.compose_platform(os, arch) == main_platform) then { requireArtifacts+: [ { name: $.maven_deploy_base_functions.pd_layouts_artifact_name(platform, dry_run), dir: vm.vm_dir, autoExtract: true, - } for platform in $.maven_deploy_base_functions.other_platforms + } for platform in other_platforms ], } else { From fa65365ae0b3b0bb97bb4b9b31db0677e7bf0515 Mon Sep 17 00:00:00 2001 From: Danilo Ansaloni Date: Mon, 23 Oct 2023 16:14:47 +0200 Subject: [PATCH 4/9] Group commands that deploy specific artifacts in dedicated snippets. Negligible changes in the expanded JSON. --- vm/ci/ci_common/common.jsonnet | 73 +++++++++++++++++++--------------- 1 file changed, 42 insertions(+), 31 deletions(-) diff --git a/vm/ci/ci_common/common.jsonnet b/vm/ci/ci_common/common.jsonnet index 9a7093af6d66..18f5a8fca20f 100644 --- a/vm/ci/ci_common/common.jsonnet +++ b/vm/ci/ci_common/common.jsonnet @@ -615,48 +615,36 @@ local devkits = graal_common.devkits; ], run_block(os, arch, dry_run, remote_mvn_repo, remote_non_mvn_repo, local_repo, main_platform, other_platforms):: - if (self.compose_platform(os, arch) == main_platform) then ( + local restore_layouts_snippet = + [self.mx_cmd_base(os, arch) + ['restore-pd-layouts', self.pd_layouts_archive_name(platform)] for platform in other_platforms] + + self.build(os, arch, mx_args=['--multi-platform-layout-directories=' + std.join(',', [main_platform] + other_platforms)], build_args=['--targets={MAVEN_TAG_DISTRIBUTIONS:public}']); # `self.only_native_dists` are in `{MAVEN_TAG_DISTRIBUTIONS:public}` + + local mvn_artifacts_snippet = + # remotely deploy only the suites that are defined in the current repository, to avoid duplicated deployments + if (vm.maven_deploy_base_functions.edition == 'ce') then + self.deploy_ce(os, arch, dry_run, [remote_mvn_repo]) + else + self.deploy_ee(os, arch, dry_run, ['--dummy-javadoc', remote_mvn_repo]) + + self.deploy_ee(os, arch, dry_run, ['--dummy-javadoc', '--only', 'JS_ISOLATE,JS_ISOLATE_RESOURCES,TOOLS_COMMUNITY,LANGUAGES_COMMUNITY', remote_mvn_repo], extra_mx_args=['--suite', 'graal-js']); + + local mvn_bundle_snippet = [ - self.mx_cmd_base(os, arch) + ['restore-pd-layouts', self.pd_layouts_archive_name(platform)] for platform in other_platforms - ] - + self.build(os, arch, mx_args=['--multi-platform-layout-directories=' + std.join(',', [main_platform] + other_platforms)], build_args=['--targets={MAVEN_TAG_DISTRIBUTIONS:public}']) # `self.only_native_dists` are in `{MAVEN_TAG_DISTRIBUTIONS:public}` - + ( - # remotely deploy only the suites that are defined in the current repository, to avoid duplicated deployments - if (vm.maven_deploy_base_functions.edition == 'ce') then - self.deploy_ce(os, arch, dry_run, [remote_mvn_repo]) - else - self.deploy_ee(os, arch, dry_run, ['--dummy-javadoc', remote_mvn_repo]) - + self.deploy_ee(os, arch, dry_run, ['--dummy-javadoc', '--only', 'JS_ISOLATE,JS_ISOLATE_RESOURCES,TOOLS_COMMUNITY,LANGUAGES_COMMUNITY', remote_mvn_repo], extra_mx_args=['--suite', 'graal-js']) - ) - + [ - # Maven bundle + # Set env vars ['set-export', 'VERSION_STRING', self.mx_cmd_base(os, arch) + ['--quiet', 'graalvm-version']], ['set-export', 'LOCAL_MAVEN_REPO_REL_PATH', 'maven-bundle-' + vm.maven_deploy_base_functions.edition + '-${VERSION_STRING}'], ['set-export', 'LOCAL_MAVEN_REPO_URL', ['mx', '--quiet', 'local-path-to-url', '${LOCAL_MAVEN_REPO_REL_PATH}']], ] + ( - if (vm.maven_deploy_base_functions.edition == 'ce') then - [['echo', 'Skipping the setting of the env variables for the Maven reduced bundle']] - else [ - # Maven reduced bundle - ['set-export', 'LOCAL_MAVEN_REDUCED_REPO_REL_PATH', 'maven-reduced-bundle-' + vm.maven_deploy_base_functions.edition + '-${VERSION_STRING}'], - ['set-export', 'LOCAL_MAVEN_REDUCED_REPO_URL', ['mx', '--quiet', 'local-path-to-url', '${LOCAL_MAVEN_REDUCED_REPO_REL_PATH}']], - ] - ) - + ( - # Maven bundle - Locally deploy all relevant suites + # Locally deploy all relevant suites if (vm.maven_deploy_base_functions.edition == 'ce') then self.deploy_ce(os, arch, dry_run, [local_repo, '${LOCAL_MAVEN_REPO_URL}']) else self.deploy_ce(os, arch, dry_run, ['--dummy-javadoc', '--skip', 'JS_ISOLATE,JS_ISOLATE_RESOURCES,TOOLS_COMMUNITY,LANGUAGES_COMMUNITY', local_repo, '${LOCAL_MAVEN_REPO_URL}']) + self.deploy_ee(os, arch, dry_run, ['--dummy-javadoc', '--only', 'JS_ISOLATE,JS_ISOLATE_RESOURCES,TOOLS_COMMUNITY,LANGUAGES_COMMUNITY', local_repo, '${LOCAL_MAVEN_REPO_URL}'], extra_mx_args=['--suite', 'graal-js']) + self.deploy_ee(os, arch, dry_run, ['--dummy-javadoc', local_repo, '${LOCAL_MAVEN_REPO_URL}']) - # Maven reduced bundle - + self.deploy_ce(os, arch, dry_run, ['--dummy-javadoc', '--only', 'TRUFFLE_API,POLYGLOT,COLLECTIONS,NATIVEIMAGE,WORD,JS_COMMUNITY,GRAALJS,TREGEX,TRUFFLE_ICU4J,TRUFFLE_RUNTIME,JNIUTILS,TRUFFLE_COMPILER,NATIVEBRIDGE,INSIGHT_HEAP,INSIGHT,TRUFFLE_DSL_PROCESSOR', local_repo, '${LOCAL_MAVEN_REDUCED_REPO_URL}']) - + self.deploy_ee(os, arch, dry_run, ['--dummy-javadoc', '--only', 'JS_ENTERPRISE,TRUFFLE_ENTERPRISE,HEAP_LANGUAGE', local_repo, '${LOCAL_MAVEN_REDUCED_REPO_URL}'], extra_mx_args=['--suite', 'graal-js']) ) + ( - # Maven bundle + # Archive and deploy if (dry_run) then [['echo', 'Skipping the archiving and the final deployment of the Maven bundle']] else [ @@ -665,9 +653,26 @@ local devkits = graal_common.devkits; ['mx', 'build', '--targets', 'MAVEN_BUNDLE'], ['mx', '--suite', 'vm', 'maven-deploy', '--tags=resource-bundle', '--all-distribution-types', '--validate=none', '--with-suite-revisions-metadata', remote_non_mvn_repo], ] + ); + + local mvn_reduced_bundle_snippet = + if (vm.maven_deploy_base_functions.edition == 'ce') then + [['echo', 'Skipping the archiving and the final deployment of the reduced Maven bundle']] + else ( + [ + # Set env vars + ['set-export', 'VERSION_STRING', self.mx_cmd_base(os, arch) + ['--quiet', 'graalvm-version']], + ['set-export', 'LOCAL_MAVEN_REDUCED_REPO_REL_PATH', 'maven-reduced-bundle-' + vm.maven_deploy_base_functions.edition + '-${VERSION_STRING}'], + ['set-export', 'LOCAL_MAVEN_REDUCED_REPO_URL', ['mx', '--quiet', 'local-path-to-url', '${LOCAL_MAVEN_REDUCED_REPO_REL_PATH}']], + ] + ( - # Maven reduced bundled - if (vm.maven_deploy_base_functions.edition == 'ce') then + # Locally deploy all relevant suites + self.deploy_ce(os, arch, dry_run, ['--dummy-javadoc', '--only', 'TRUFFLE_API,POLYGLOT,COLLECTIONS,NATIVEIMAGE,WORD,JS_COMMUNITY,GRAALJS,TREGEX,TRUFFLE_ICU4J,TRUFFLE_RUNTIME,JNIUTILS,TRUFFLE_COMPILER,NATIVEBRIDGE,INSIGHT_HEAP,INSIGHT,TRUFFLE_DSL_PROCESSOR', local_repo, '${LOCAL_MAVEN_REDUCED_REPO_URL}']) + + self.deploy_ee(os, arch, dry_run, ['--dummy-javadoc', '--only', 'JS_ENTERPRISE,TRUFFLE_ENTERPRISE,HEAP_LANGUAGE', local_repo, '${LOCAL_MAVEN_REDUCED_REPO_URL}'], extra_mx_args=['--suite', 'graal-js']) + ) + + ( + # Archive and deploy + if (dry_run) then [['echo', 'Skipping the archiving and the final deployment of the reduced Maven bundle']] else [ ['set-export', 'MAVEN_BUNDLE_PATH', '${LOCAL_MAVEN_REDUCED_REPO_REL_PATH}'], @@ -676,7 +681,13 @@ local devkits = graal_common.devkits; ['mx', '--suite', 'vm', 'maven-deploy', '--tags=resource-bundle', '--all-distribution-types', '--validate=none', '--with-suite-revisions-metadata', remote_non_mvn_repo], ] ) - ) + ); + + if (self.compose_platform(os, arch) == main_platform) then ( + restore_layouts_snippet + + mvn_artifacts_snippet + + mvn_bundle_snippet + + mvn_reduced_bundle_snippet ) else ( self.build(os, arch, build_args=['--targets=' + self.only_native_dists + ',{PLATFORM_DEPENDENT_LAYOUT_DIR_DISTRIBUTIONS}']) + ( From 4429d1f6c13f73b570f6ef680bcb758bef182284 Mon Sep 17 00:00:00 2001 From: Danilo Ansaloni Date: Mon, 23 Oct 2023 16:23:03 +0200 Subject: [PATCH 5/9] Allow choosing what to deploy. No changes in the expanded JSON. --- vm/ci/ci_common/common.jsonnet | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/vm/ci/ci_common/common.jsonnet b/vm/ci/ci_common/common.jsonnet index 18f5a8fca20f..1f086992df0e 100644 --- a/vm/ci/ci_common/common.jsonnet +++ b/vm/ci/ci_common/common.jsonnet @@ -614,7 +614,7 @@ local devkits = graal_common.devkits; + extra_args, ], - run_block(os, arch, dry_run, remote_mvn_repo, remote_non_mvn_repo, local_repo, main_platform, other_platforms):: + run_block(os, arch, dry_run, remote_mvn_repo, remote_non_mvn_repo, local_repo, main_platform, other_platforms, mvn_artifacts=true, mvn_bundle=true, mvn_reduced_bundle=true):: local restore_layouts_snippet = [self.mx_cmd_base(os, arch) + ['restore-pd-layouts', self.pd_layouts_archive_name(platform)] for platform in other_platforms] + self.build(os, arch, mx_args=['--multi-platform-layout-directories=' + std.join(',', [main_platform] + other_platforms)], build_args=['--targets={MAVEN_TAG_DISTRIBUTIONS:public}']); # `self.only_native_dists` are in `{MAVEN_TAG_DISTRIBUTIONS:public}` @@ -685,9 +685,24 @@ local devkits = graal_common.devkits; if (self.compose_platform(os, arch) == main_platform) then ( restore_layouts_snippet - + mvn_artifacts_snippet - + mvn_bundle_snippet - + mvn_reduced_bundle_snippet + + ( + if (mvn_artifacts) then + mvn_artifacts_snippet + else + [['echo', 'Skipping Maven artifacts']] + ) + + ( + if (mvn_bundle) then + mvn_bundle_snippet + else + [['echo', 'Skipping Maven bundle']] + ) + + ( + if (mvn_reduced_bundle) then + mvn_reduced_bundle_snippet + else + [['echo', 'Skipping reduced Maven bundle']] + ) ) else ( self.build(os, arch, build_args=['--targets=' + self.only_native_dists + ',{PLATFORM_DEPENDENT_LAYOUT_DIR_DISTRIBUTIONS}']) + ( From 5805293e56b18929d8a32b56ba10563f9f105fa5 Mon Sep 17 00:00:00 2001 From: Danilo Ansaloni Date: Tue, 24 Oct 2023 12:23:26 +0200 Subject: [PATCH 6/9] Bundles for a single platform have a different name. --- vm/ci/ci_common/common.jsonnet | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vm/ci/ci_common/common.jsonnet b/vm/ci/ci_common/common.jsonnet index 1f086992df0e..738faa0497fc 100644 --- a/vm/ci/ci_common/common.jsonnet +++ b/vm/ci/ci_common/common.jsonnet @@ -676,7 +676,7 @@ local devkits = graal_common.devkits; [['echo', 'Skipping the archiving and the final deployment of the reduced Maven bundle']] else [ ['set-export', 'MAVEN_BUNDLE_PATH', '${LOCAL_MAVEN_REDUCED_REPO_REL_PATH}'], - ['set-export', 'MAVEN_BUNDLE_ARTIFACT_ID', 'maven-reduced-bundle-' + vm.maven_deploy_base_functions.edition], + ['set-export', 'MAVEN_BUNDLE_ARTIFACT_ID', 'maven-reduced-bundle-' + vm.maven_deploy_base_functions.edition + (if (std.length(other_platforms) == 0) then '-' + main_platform else '')], ['mx', 'build', '--targets', 'MAVEN_BUNDLE'], ['mx', '--suite', 'vm', 'maven-deploy', '--tags=resource-bundle', '--all-distribution-types', '--validate=none', '--with-suite-revisions-metadata', remote_non_mvn_repo], ] From ffaba6415d6de4131253e9cd90cb058fbfb0c883 Mon Sep 17 00:00:00 2001 From: Danilo Ansaloni Date: Wed, 25 Oct 2023 16:17:06 +0200 Subject: [PATCH 7/9] Move the list of reduced dists to the vm suites. --- vm/ci/ci_common/common.jsonnet | 4 ++-- vm/ci/ci_includes/vm.jsonnet | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/vm/ci/ci_common/common.jsonnet b/vm/ci/ci_common/common.jsonnet index 738faa0497fc..a7c2faeae01d 100644 --- a/vm/ci/ci_common/common.jsonnet +++ b/vm/ci/ci_common/common.jsonnet @@ -667,8 +667,8 @@ local devkits = graal_common.devkits; ] + ( # Locally deploy all relevant suites - self.deploy_ce(os, arch, dry_run, ['--dummy-javadoc', '--only', 'TRUFFLE_API,POLYGLOT,COLLECTIONS,NATIVEIMAGE,WORD,JS_COMMUNITY,GRAALJS,TREGEX,TRUFFLE_ICU4J,TRUFFLE_RUNTIME,JNIUTILS,TRUFFLE_COMPILER,NATIVEBRIDGE,INSIGHT_HEAP,INSIGHT,TRUFFLE_DSL_PROCESSOR', local_repo, '${LOCAL_MAVEN_REDUCED_REPO_URL}']) - + self.deploy_ee(os, arch, dry_run, ['--dummy-javadoc', '--only', 'JS_ENTERPRISE,TRUFFLE_ENTERPRISE,HEAP_LANGUAGE', local_repo, '${LOCAL_MAVEN_REDUCED_REPO_URL}'], extra_mx_args=['--suite', 'graal-js']) + self.deploy_ce(os, arch, dry_run, ['--dummy-javadoc', '--only', vm.maven_deploy_base_functions.reduced_ce_dists, local_repo, '${LOCAL_MAVEN_REDUCED_REPO_URL}']) + + self.deploy_ee(os, arch, dry_run, ['--dummy-javadoc', '--only', vm.maven_deploy_base_functions.reduced_ee_dists, local_repo, '${LOCAL_MAVEN_REDUCED_REPO_URL}'], extra_mx_args=['--suite', 'graal-js']) ) + ( # Archive and deploy diff --git a/vm/ci/ci_includes/vm.jsonnet b/vm/ci/ci_includes/vm.jsonnet index 2bcdccd8191e..89ec1203151e 100644 --- a/vm/ci/ci_includes/vm.jsonnet +++ b/vm/ci/ci_includes/vm.jsonnet @@ -92,6 +92,9 @@ local graal_common = import '../../../ci/ci_common/common.jsonnet'; ee_licenses():: error 'The vm suite does not define ee licenses', + + reduced_ce_dists:: error 'The vm suite does not define reduced dists', + reduced_ee_dists:: error 'The vm suite does not define reduced dists', }, local builds = [ From 21573b486ae516859859ddd47cd2aa5682a8ebaa Mon Sep 17 00:00:00 2001 From: Danilo Ansaloni Date: Wed, 1 Nov 2023 18:05:44 +0100 Subject: [PATCH 8/9] Do not obfuscate reduced Maven bundles. Does not change the expanded JSON. --- vm/ci/ci_common/common.jsonnet | 32 +++++++++++++++++++------------- vm/ci/ci_includes/vm.jsonnet | 3 ++- 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/vm/ci/ci_common/common.jsonnet b/vm/ci/ci_common/common.jsonnet index a7c2faeae01d..608ae4ff9267 100644 --- a/vm/ci/ci_common/common.jsonnet +++ b/vm/ci/ci_common/common.jsonnet @@ -566,14 +566,14 @@ local devkits = graal_common.devkits; ce_licenses, legacy_mx_args:: ['--disable-installables=true'], # `['--force-bash-launcher=true', '--skip-libraries=true']` have been replaced by arguments from `vm.maven_deploy_base_functions.mx_args(os, arch)` - mx_args(os, arch):: self.legacy_mx_args + vm.maven_deploy_base_functions.mx_args(os, arch), - mx_cmd_base(os, arch):: ['mx'] + vm.maven_deploy_base_functions.dynamic_imports(os, arch) + self.mx_args(os, arch), + mx_args(os, arch, strip=true):: self.legacy_mx_args + vm.maven_deploy_base_functions.mx_args(os, arch, strip), + mx_cmd_base(os, arch, strip=true):: ['mx'] + vm.maven_deploy_base_functions.dynamic_imports(os, arch) + self.mx_args(os, arch, strip), mx_cmd_base_only_native(os, arch):: ['mx', '--dynamicimports', '/substratevm'] + self.mx_args(os, arch) + ['--native-images=false'], # `--native-images=false` takes precedence over `self.mx_args(os, arch)` only_native_dists:: 'TRUFFLE_NFI_NATIVE,SVM_HOSTED_NATIVE', - build(os, arch, mx_args=[], build_args=[]):: [ - self.mx_cmd_base(os, arch) + mx_args + ['build'] + build_args, + build(os, arch, mx_args=[], build_args=[], strip=true):: [ + self.mx_cmd_base(os, arch, strip) + mx_args + ['build'] + build_args, ], pd_layouts_archive_name(platform):: 'pd-layouts-' + platform + '.tgz', @@ -585,8 +585,8 @@ local devkits = graal_common.devkits; compose_platform(os, arch):: os + '-' + arch, - deploy_ce(os, arch, dry_run, extra_args, extra_mx_args=[]):: [ - self.mx_cmd_base(os, arch) + deploy_ce(os, arch, dry_run, extra_args, extra_mx_args=[], strip=true):: [ + self.mx_cmd_base(os, arch, strip) + $.maven_deploy_base_functions.ce_suites(os,arch) + extra_mx_args + self.mvn_args @@ -595,8 +595,8 @@ local devkits = graal_common.devkits; + extra_args, ], - deploy_ee(os, arch, dry_run, extra_args, extra_mx_args=[]):: [ - self.mx_cmd_base(os, arch) + deploy_ee(os, arch, dry_run, extra_args, extra_mx_args=[], strip=true):: [ + self.mx_cmd_base(os, arch, strip) + vm.maven_deploy_base_functions.ee_suites(os, arch) + extra_mx_args + self.mvn_args @@ -615,9 +615,11 @@ local devkits = graal_common.devkits; ], run_block(os, arch, dry_run, remote_mvn_repo, remote_non_mvn_repo, local_repo, main_platform, other_platforms, mvn_artifacts=true, mvn_bundle=true, mvn_reduced_bundle=true):: - local restore_layouts_snippet = + local multiplatform_build(strip=true) = self.build(os, arch, mx_args=['--multi-platform-layout-directories=' + std.join(',', [main_platform] + other_platforms)], build_args=['--targets={MAVEN_TAG_DISTRIBUTIONS:public}'], strip=strip); # `self.only_native_dists` are in `{MAVEN_TAG_DISTRIBUTIONS:public}` + + local restore_layouts_snippet_and_build = [self.mx_cmd_base(os, arch) + ['restore-pd-layouts', self.pd_layouts_archive_name(platform)] for platform in other_platforms] - + self.build(os, arch, mx_args=['--multi-platform-layout-directories=' + std.join(',', [main_platform] + other_platforms)], build_args=['--targets={MAVEN_TAG_DISTRIBUTIONS:public}']); # `self.only_native_dists` are in `{MAVEN_TAG_DISTRIBUTIONS:public}` + + multiplatform_build(); local mvn_artifacts_snippet = # remotely deploy only the suites that are defined in the current repository, to avoid duplicated deployments @@ -665,10 +667,14 @@ local devkits = graal_common.devkits; ['set-export', 'LOCAL_MAVEN_REDUCED_REPO_REL_PATH', 'maven-reduced-bundle-' + vm.maven_deploy_base_functions.edition + '-${VERSION_STRING}'], ['set-export', 'LOCAL_MAVEN_REDUCED_REPO_URL', ['mx', '--quiet', 'local-path-to-url', '${LOCAL_MAVEN_REDUCED_REPO_REL_PATH}']], ] + + ( + # Build without stripping. Note that restored platform-dependendent layouts might be stripped. + multiplatform_build(strip=false) + ) + ( # Locally deploy all relevant suites - self.deploy_ce(os, arch, dry_run, ['--dummy-javadoc', '--only', vm.maven_deploy_base_functions.reduced_ce_dists, local_repo, '${LOCAL_MAVEN_REDUCED_REPO_URL}']) - + self.deploy_ee(os, arch, dry_run, ['--dummy-javadoc', '--only', vm.maven_deploy_base_functions.reduced_ee_dists, local_repo, '${LOCAL_MAVEN_REDUCED_REPO_URL}'], extra_mx_args=['--suite', 'graal-js']) + self.deploy_ce(os, arch, dry_run, ['--dummy-javadoc', '--only', vm.maven_deploy_base_functions.reduced_ce_dists, local_repo, '${LOCAL_MAVEN_REDUCED_REPO_URL}'], strip=false) + + self.deploy_ee(os, arch, dry_run, ['--dummy-javadoc', '--only', vm.maven_deploy_base_functions.reduced_ee_dists, local_repo, '${LOCAL_MAVEN_REDUCED_REPO_URL}'], extra_mx_args=['--suite', 'graal-js'], strip=false) ) + ( # Archive and deploy @@ -684,7 +690,7 @@ local devkits = graal_common.devkits; ); if (self.compose_platform(os, arch) == main_platform) then ( - restore_layouts_snippet + restore_layouts_snippet_and_build + ( if (mvn_artifacts) then mvn_artifacts_snippet diff --git a/vm/ci/ci_includes/vm.jsonnet b/vm/ci/ci_includes/vm.jsonnet index 89ec1203151e..d27fd2add128 100644 --- a/vm/ci/ci_includes/vm.jsonnet +++ b/vm/ci/ci_includes/vm.jsonnet @@ -81,7 +81,8 @@ local graal_common = import '../../../ci/ci_common/common.jsonnet'; maven_deploy_base_functions: { edition:: vm.edition, - mx_args(os, arch):: + mx_args(os, arch, strip=true):: + # the `vm` suite never strips artifacts ['--native-images=false'], dynamic_imports(os, arch):: From 8c4644b40f63d11a30ffadf8df0fe6b2d3a44b58 Mon Sep 17 00:00:00 2001 From: Danilo Ansaloni Date: Fri, 3 Nov 2023 14:39:11 +0100 Subject: [PATCH 9/9] Avoid building twice when deploying only the reduced bundles. --- vm/ci/ci_common/common.jsonnet | 67 +++++++++++++++++----------------- vm/ci/ci_includes/vm.jsonnet | 4 +- 2 files changed, 35 insertions(+), 36 deletions(-) diff --git a/vm/ci/ci_common/common.jsonnet b/vm/ci/ci_common/common.jsonnet index 608ae4ff9267..a0009cfc97a2 100644 --- a/vm/ci/ci_common/common.jsonnet +++ b/vm/ci/ci_common/common.jsonnet @@ -566,14 +566,14 @@ local devkits = graal_common.devkits; ce_licenses, legacy_mx_args:: ['--disable-installables=true'], # `['--force-bash-launcher=true', '--skip-libraries=true']` have been replaced by arguments from `vm.maven_deploy_base_functions.mx_args(os, arch)` - mx_args(os, arch, strip=true):: self.legacy_mx_args + vm.maven_deploy_base_functions.mx_args(os, arch, strip), - mx_cmd_base(os, arch, strip=true):: ['mx'] + vm.maven_deploy_base_functions.dynamic_imports(os, arch) + self.mx_args(os, arch, strip), - mx_cmd_base_only_native(os, arch):: ['mx', '--dynamicimports', '/substratevm'] + self.mx_args(os, arch) + ['--native-images=false'], # `--native-images=false` takes precedence over `self.mx_args(os, arch)` + mx_args(os, arch, reduced):: self.legacy_mx_args + vm.maven_deploy_base_functions.mx_args(os, arch, reduced), + mx_cmd_base(os, arch, reduced):: ['mx'] + vm.maven_deploy_base_functions.dynamic_imports(os, arch) + self.mx_args(os, arch, reduced), + mx_cmd_base_only_native(os, arch, reduced):: ['mx', '--dynamicimports', '/substratevm'] + self.mx_args(os, arch, reduced) + ['--native-images=false'], # `--native-images=false` takes precedence over `self.mx_args(os, arch)` only_native_dists:: 'TRUFFLE_NFI_NATIVE,SVM_HOSTED_NATIVE', - build(os, arch, mx_args=[], build_args=[], strip=true):: [ - self.mx_cmd_base(os, arch, strip) + mx_args + ['build'] + build_args, + build(os, arch, reduced, mx_args=[], build_args=[]):: [ + self.mx_cmd_base(os, arch, reduced) + mx_args + ['build'] + build_args, ], pd_layouts_archive_name(platform):: 'pd-layouts-' + platform + '.tgz', @@ -585,8 +585,8 @@ local devkits = graal_common.devkits; compose_platform(os, arch):: os + '-' + arch, - deploy_ce(os, arch, dry_run, extra_args, extra_mx_args=[], strip=true):: [ - self.mx_cmd_base(os, arch, strip) + deploy_ce(os, arch, reduced, dry_run, extra_args, extra_mx_args=[]):: [ + self.mx_cmd_base(os, arch, reduced) + $.maven_deploy_base_functions.ce_suites(os,arch) + extra_mx_args + self.mvn_args @@ -595,8 +595,8 @@ local devkits = graal_common.devkits; + extra_args, ], - deploy_ee(os, arch, dry_run, extra_args, extra_mx_args=[], strip=true):: [ - self.mx_cmd_base(os, arch, strip) + deploy_ee(os, arch, reduced, dry_run, extra_args, extra_mx_args=[]):: [ + self.mx_cmd_base(os, arch, reduced) + vm.maven_deploy_base_functions.ee_suites(os, arch) + extra_mx_args + self.mvn_args @@ -605,8 +605,8 @@ local devkits = graal_common.devkits; + extra_args, ], - deploy_only_native(os, arch, dry_run, extra_args, extra_mx_args=[]):: [ - self.mx_cmd_base_only_native(os, arch) + deploy_only_native(os, arch, reduced, dry_run, extra_args, extra_mx_args=[]):: [ + self.mx_cmd_base_only_native(os, arch, reduced) + extra_mx_args + self.mvn_args_only_native + ['--licenses', $.maven_deploy_base_functions.ce_licenses()] @@ -615,35 +615,31 @@ local devkits = graal_common.devkits; ], run_block(os, arch, dry_run, remote_mvn_repo, remote_non_mvn_repo, local_repo, main_platform, other_platforms, mvn_artifacts=true, mvn_bundle=true, mvn_reduced_bundle=true):: - local multiplatform_build(strip=true) = self.build(os, arch, mx_args=['--multi-platform-layout-directories=' + std.join(',', [main_platform] + other_platforms)], build_args=['--targets={MAVEN_TAG_DISTRIBUTIONS:public}'], strip=strip); # `self.only_native_dists` are in `{MAVEN_TAG_DISTRIBUTIONS:public}` - - local restore_layouts_snippet_and_build = - [self.mx_cmd_base(os, arch) + ['restore-pd-layouts', self.pd_layouts_archive_name(platform)] for platform in other_platforms] - + multiplatform_build(); + local multiplatform_build(reduced) = self.build(os, arch, reduced, mx_args=['--multi-platform-layout-directories=' + std.join(',', [main_platform] + other_platforms)], build_args=['--targets={MAVEN_TAG_DISTRIBUTIONS:public}']); # `self.only_native_dists` are in `{MAVEN_TAG_DISTRIBUTIONS:public}` local mvn_artifacts_snippet = # remotely deploy only the suites that are defined in the current repository, to avoid duplicated deployments if (vm.maven_deploy_base_functions.edition == 'ce') then - self.deploy_ce(os, arch, dry_run, [remote_mvn_repo]) + self.deploy_ce(os, arch, false, dry_run, [remote_mvn_repo]) else - self.deploy_ee(os, arch, dry_run, ['--dummy-javadoc', remote_mvn_repo]) - + self.deploy_ee(os, arch, dry_run, ['--dummy-javadoc', '--only', 'JS_ISOLATE,JS_ISOLATE_RESOURCES,TOOLS_COMMUNITY,LANGUAGES_COMMUNITY', remote_mvn_repo], extra_mx_args=['--suite', 'graal-js']); + self.deploy_ee(os, arch, false, dry_run, ['--dummy-javadoc', remote_mvn_repo]) + + self.deploy_ee(os, arch, false, dry_run, ['--dummy-javadoc', '--only', 'JS_ISOLATE,JS_ISOLATE_RESOURCES,TOOLS_COMMUNITY,LANGUAGES_COMMUNITY', remote_mvn_repo], extra_mx_args=['--suite', 'graal-js']); local mvn_bundle_snippet = [ # Set env vars - ['set-export', 'VERSION_STRING', self.mx_cmd_base(os, arch) + ['--quiet', 'graalvm-version']], + ['set-export', 'VERSION_STRING', self.mx_cmd_base(os, arch, reduced=false) + ['--quiet', 'graalvm-version']], ['set-export', 'LOCAL_MAVEN_REPO_REL_PATH', 'maven-bundle-' + vm.maven_deploy_base_functions.edition + '-${VERSION_STRING}'], ['set-export', 'LOCAL_MAVEN_REPO_URL', ['mx', '--quiet', 'local-path-to-url', '${LOCAL_MAVEN_REPO_REL_PATH}']], ] + ( # Locally deploy all relevant suites if (vm.maven_deploy_base_functions.edition == 'ce') then - self.deploy_ce(os, arch, dry_run, [local_repo, '${LOCAL_MAVEN_REPO_URL}']) + self.deploy_ce(os, arch, false, dry_run, [local_repo, '${LOCAL_MAVEN_REPO_URL}']) else - self.deploy_ce(os, arch, dry_run, ['--dummy-javadoc', '--skip', 'JS_ISOLATE,JS_ISOLATE_RESOURCES,TOOLS_COMMUNITY,LANGUAGES_COMMUNITY', local_repo, '${LOCAL_MAVEN_REPO_URL}']) - + self.deploy_ee(os, arch, dry_run, ['--dummy-javadoc', '--only', 'JS_ISOLATE,JS_ISOLATE_RESOURCES,TOOLS_COMMUNITY,LANGUAGES_COMMUNITY', local_repo, '${LOCAL_MAVEN_REPO_URL}'], extra_mx_args=['--suite', 'graal-js']) - + self.deploy_ee(os, arch, dry_run, ['--dummy-javadoc', local_repo, '${LOCAL_MAVEN_REPO_URL}']) + self.deploy_ce(os, arch, false, dry_run, ['--dummy-javadoc', '--skip', 'JS_ISOLATE,JS_ISOLATE_RESOURCES,TOOLS_COMMUNITY,LANGUAGES_COMMUNITY', local_repo, '${LOCAL_MAVEN_REPO_URL}']) + + self.deploy_ee(os, arch, false, dry_run, ['--dummy-javadoc', '--only', 'JS_ISOLATE,JS_ISOLATE_RESOURCES,TOOLS_COMMUNITY,LANGUAGES_COMMUNITY', local_repo, '${LOCAL_MAVEN_REPO_URL}'], extra_mx_args=['--suite', 'graal-js']) + + self.deploy_ee(os, arch, false, dry_run, ['--dummy-javadoc', local_repo, '${LOCAL_MAVEN_REPO_URL}']) ) + ( # Archive and deploy @@ -663,18 +659,17 @@ local devkits = graal_common.devkits; else ( [ # Set env vars - ['set-export', 'VERSION_STRING', self.mx_cmd_base(os, arch) + ['--quiet', 'graalvm-version']], + ['set-export', 'VERSION_STRING', self.mx_cmd_base(os, arch, reduced=true) + ['--quiet', 'graalvm-version']], ['set-export', 'LOCAL_MAVEN_REDUCED_REPO_REL_PATH', 'maven-reduced-bundle-' + vm.maven_deploy_base_functions.edition + '-${VERSION_STRING}'], ['set-export', 'LOCAL_MAVEN_REDUCED_REPO_URL', ['mx', '--quiet', 'local-path-to-url', '${LOCAL_MAVEN_REDUCED_REPO_REL_PATH}']], ] + ( - # Build without stripping. Note that restored platform-dependendent layouts might be stripped. - multiplatform_build(strip=false) + multiplatform_build(reduced=true) ) + ( # Locally deploy all relevant suites - self.deploy_ce(os, arch, dry_run, ['--dummy-javadoc', '--only', vm.maven_deploy_base_functions.reduced_ce_dists, local_repo, '${LOCAL_MAVEN_REDUCED_REPO_URL}'], strip=false) - + self.deploy_ee(os, arch, dry_run, ['--dummy-javadoc', '--only', vm.maven_deploy_base_functions.reduced_ee_dists, local_repo, '${LOCAL_MAVEN_REDUCED_REPO_URL}'], extra_mx_args=['--suite', 'graal-js'], strip=false) + self.deploy_ce(os, arch, true, dry_run, ['--dummy-javadoc', '--only', vm.maven_deploy_base_functions.reduced_ce_dists, local_repo, '${LOCAL_MAVEN_REDUCED_REPO_URL}']) + + self.deploy_ee(os, arch, true, dry_run, ['--dummy-javadoc', '--only', vm.maven_deploy_base_functions.reduced_ee_dists, local_repo, '${LOCAL_MAVEN_REDUCED_REPO_URL}'], extra_mx_args=['--suite', 'graal-js']) ) + ( # Archive and deploy @@ -690,7 +685,13 @@ local devkits = graal_common.devkits; ); if (self.compose_platform(os, arch) == main_platform) then ( - restore_layouts_snippet_and_build + [self.mx_cmd_base(os, arch, reduced=false) + ['restore-pd-layouts', self.pd_layouts_archive_name(platform)] for platform in other_platforms] + + ( + if (mvn_artifacts || mvn_bundle) then + multiplatform_build(reduced=false) + else + [['echo', 'Skipping the full build']] + ) + ( if (mvn_artifacts) then mvn_artifacts_snippet @@ -710,14 +711,14 @@ local devkits = graal_common.devkits; [['echo', 'Skipping reduced Maven bundle']] ) ) else ( - self.build(os, arch, build_args=['--targets=' + self.only_native_dists + ',{PLATFORM_DEPENDENT_LAYOUT_DIR_DISTRIBUTIONS}']) + self.build(os, arch, reduced=false, build_args=['--targets=' + self.only_native_dists + ',{PLATFORM_DEPENDENT_LAYOUT_DIR_DISTRIBUTIONS}']) + ( if (vm.maven_deploy_base_functions.edition == 'ce') then - self.deploy_only_native(os, arch, dry_run, [remote_mvn_repo]) + self.deploy_only_native(os, arch, reduced=false, dry_run=dry_run, extra_args=[remote_mvn_repo]) else [['echo', 'Skipping the deployment of ' + self.only_native_dists + ': It is already deployed by the ce job']] ) - + [self.mx_cmd_base(os, arch) + ['archive-pd-layouts', self.pd_layouts_archive_name(os + '-' + arch)]] + + [self.mx_cmd_base(os, arch, reduced=false) + ['archive-pd-layouts', self.pd_layouts_archive_name(os + '-' + arch)]] ), base_object(os, arch, dry_run, remote_mvn_repo, remote_non_mvn_repo, local_repo, main_platform='linux-amd64', other_platforms=['linux-aarch64', 'darwin-amd64', 'darwin-aarch64', 'windows-amd64'],):: { diff --git a/vm/ci/ci_includes/vm.jsonnet b/vm/ci/ci_includes/vm.jsonnet index d27fd2add128..2e98e3002f64 100644 --- a/vm/ci/ci_includes/vm.jsonnet +++ b/vm/ci/ci_includes/vm.jsonnet @@ -81,9 +81,7 @@ local graal_common = import '../../../ci/ci_common/common.jsonnet'; maven_deploy_base_functions: { edition:: vm.edition, - mx_args(os, arch, strip=true):: - # the `vm` suite never strips artifacts - ['--native-images=false'], + mx_args(os, arch, reduced=false):: ['--native-images=false'], dynamic_imports(os, arch):: ['--dynamicimports', vm_common.maven_deploy_base_functions.dynamic_ce_imports(os, arch)],