From dba8c348ab3254cdb671fc58a021d4fe80fc52a8 Mon Sep 17 00:00:00 2001 From: Fabio Niephaus Date: Fri, 28 Oct 2022 11:31:22 +0200 Subject: [PATCH] Build TruffleSqueak standalone instead of shipping TruffleSqueak Native with installables. --- .github/workflows/ci.yml | 106 +++++++++++++------ mx.trufflesqueak/launchers/trufflesqueak | 26 +++-- mx.trufflesqueak/launchers/trufflesqueak.cmd | 22 ++-- mx.trufflesqueak/mx_trufflesqueak.py | 18 ++-- mx.trufflesqueak/suite.py | 13 ++- mx.trufflesqueak/trufflesqueak-jar | 3 + mx.trufflesqueak/trufflesqueak-jvm | 2 - mx.trufflesqueak/trufflesqueak-standalone | 4 + mx.trufflesqueak/trufflesqueak-svm | 2 - mx.trufflesqueak/utils.sh | 91 +++++++++++----- 10 files changed, 184 insertions(+), 103 deletions(-) create mode 100644 mx.trufflesqueak/trufflesqueak-jar delete mode 100644 mx.trufflesqueak/trufflesqueak-jvm create mode 100644 mx.trufflesqueak/trufflesqueak-standalone delete mode 100644 mx.trufflesqueak/trufflesqueak-svm diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 36556268b..903c4441e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,9 +23,9 @@ jobs: runs-on: ubuntu-20.04 steps: - name: Clone TruffleSqueak repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: '3.8' - name: Set up dependencies @@ -46,7 +46,7 @@ jobs: - name: Check style and perform full build run: mx gate --strict-mode --tags style,fullbuild - builds: + build-jar: strategy: fail-fast: false matrix: @@ -55,39 +55,31 @@ jobs: env: RUN_TESTS: "${{ matrix.os == 'ubuntu-20.04' || matrix.java == 17 }}" RUN_WITH_COVERAGE: "${{ matrix.os == 'ubuntu-20.04' && matrix.java == 17 }}" - TRUFFLESQUEAK_EXEC: "${{ matrix.os == 'windows-2019' && 'trufflesqueak.cmd' || 'trufflesqueak' }}" - TS_INFIX: "${{ matrix.java == 11 && '_SVM' || '' }}" - MX_ENV: "${{ matrix.java == 11 && 'trufflesqueak-svm' || 'trufflesqueak-jvm' }}" + MX_ENV: "trufflesqueak-jar" VERBOSE_GRAALVM_LAUNCHERS: true - name: ${{ matrix.os }} + JDK${{ matrix.java }} + name: JAR ${{ matrix.os }} + JDK${{ matrix.java }} timeout-minutes: 60 runs-on: ${{ matrix.os }} steps: - name: Clone TruffleSqueak repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: submodules: true - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: '3.8' - name: Set up dependencies shell: bash - run: mx.trufflesqueak/utils.sh set-up-dependencies java${{ matrix.java }} - - name: Build TruffleSqueak component - run: mx.trufflesqueak/utils.sh build-component ${{ env.MX_ENV }} ${{ matrix.java }} ${{ env.INSTALLABLE_TARGET }} + run: mx.trufflesqueak/utils.sh set-up-dependencies java${{ matrix.java }} false + - name: Build TruffleSqueak installable + run: mx.trufflesqueak/utils.sh build-installable ${{ matrix.java }} if: ${{ matrix.os != 'windows-2019' }} - - name: Configure pagefile # this reduces OOM kills on Windows - uses: al-cheb/configure-pagefile-action@v1.2 - with: - minimum-size: 8GB - disk-root: "C:" - if: ${{ matrix.os == 'windows-2019' }} - - name: Build TruffleSqueak component via cmd.exe + - name: Build TruffleSqueak installable via cmd.exe shell: cmd run: | call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64 - call mx --env ${{ env.MX_ENV }} --no-download-progress build --dependencies SMALLTALK_INSTALLABLE${{ env.TS_INFIX }}_JAVA${{ matrix.java }},GRAALVM_TRUFFLESQUEAK${{ env.TS_INFIX }}_JAVA${{ matrix.java }} + call mx --env ${{ env.MX_ENV }} --no-download-progress build --dependencies SMALLTALK_INSTALLABLE_JAVA${{ matrix.java }},GRAALVM_TRUFFLESQUEAK_JAR_JAVA${{ matrix.java }} call mx --env ${{ env.MX_ENV }} graalvm-home > graalvm-home-with-forward-slashes.txt set /p GRAALVM_HOME=>%GITHUB_PATH% echo GRAALVM_HOME=%GRAALVM_HOME%>>%GITHUB_ENV% echo [%GRAALVM_HOME% set as $GRAALVM_HOME] - call mx --env ${{ env.MX_ENV }} paths SMALLTALK_INSTALLABLE${{ env.TS_INFIX }}_JAVA${{ matrix.java }} > installable-path-with-forward-slashes.txt + call mx --env ${{ env.MX_ENV }} paths SMALLTALK_INSTALLABLE_JAVA${{ matrix.java }} > installable-path-with-forward-slashes.txt set /p INSTALLABLE_PATH= standalone-home-with-forward-slashes.txt + set /p STANDALONE_HOME=>%GITHUB_PATH% + echo [%STANDALONE_HOME%\bin added to $PATH] + call mx --env ${{ env.MX_ENV }} paths SMALLTALK_STANDALONE_SVM_JAVA${{ env.JAVA_VERSION }} > standalone-path-with-forward-slashes.txt + set /p STANDALONE_PATH=nul || ( - @REM Default to polyglot launcher in JVM mode - set extra_args=%extra_args% --jvm --polyglot - @REM Make ReflectionUtils work - set extra_args=%extra_args% --vm.-add-exports=java.base/jdk.internal.module=ALL-UNNAMED - @REM Make Truffle.getRuntime() accessible for VM introspection - set extra_args=%extra_args% --vm.-add-opens=jdk.internal.vm.compiler/org.graalvm.compiler.truffle.runtime=ALL-UNNAMED - @REM Enable access to HostObject and others - set extra_args=%extra_args% --vm.-add-opens=org.graalvm.truffle/com.oracle.truffle.host=ALL-UNNAMED - @REM Enable access to Truffle's SourceSection (for retrieving sources through interop) - set extra_args=%extra_args% --vm.-add-opens=org.graalvm.truffle/com.oracle.truffle.api.source=ALL-UNNAMED -) +@REM Default to polyglot launcher in JVM mode +set extra_args=%extra_args% --jvm --polyglot +@REM Make ReflectionUtils work +set extra_args=%extra_args% --vm.-add-exports=java.base/jdk.internal.module=ALL-UNNAMED +@REM Make Truffle.getRuntime() accessible for VM introspection +set extra_args=%extra_args% --vm.-add-opens=jdk.internal.vm.compiler/org.graalvm.compiler.truffle.runtime=ALL-UNNAMED +@REM Enable access to HostObject and others +set extra_args=%extra_args% --vm.-add-opens=org.graalvm.truffle/com.oracle.truffle.host=ALL-UNNAMED +@REM Enable access to Truffle's SourceSection (for retrieving sources through interop) +set extra_args=%extra_args% --vm.-add-opens=org.graalvm.truffle/com.oracle.truffle.api.source=ALL-UNNAMED if "%VERBOSE_GRAALVM_LAUNCHERS%"=="true" echo on diff --git a/mx.trufflesqueak/mx_trufflesqueak.py b/mx.trufflesqueak/mx_trufflesqueak.py index 387c70379..215471df2 100644 --- a/mx.trufflesqueak/mx_trufflesqueak.py +++ b/mx.trufflesqueak/mx_trufflesqueak.py @@ -180,32 +180,32 @@ def patched_init(self, *args, **kw_args): _enable_local_compression() -mx_sdk_vm.register_vm_config('trufflesqueak', ['nfi', 'nfi-libffi', 'sdk', 'st', 'tfl'], - _SUITE, env_file='trufflesqueak-jvm') -mx_sdk_vm.register_vm_config('trufflesqueak-svm', ['cmp', 'nfi', 'nfi-libffi', 'sdk', 'st', 'svm', 'svmnfi', 'svmsl', 'tfl', 'tflm'], - _SUITE, env_file='trufflesqueak-svm') +mx_sdk_vm.register_vm_config('trufflesqueak-jar', ['sdk', 'st', 'tfl'], + _SUITE, env_file='trufflesqueak-jar') +mx_sdk_vm.register_vm_config('trufflesqueak-standalone', ['cmp', 'nfi', 'nfi-libffi', 'sdk', 'st', 'tfl'], + _SUITE, env_file='trufflesqueak-standalone') mx_sdk.register_graalvm_component(mx_sdk.GraalVmLanguage( suite=_SUITE, name='TruffleSqueak', short_name='st', dir_name=LANGUAGE_ID, + standalone_dir_name='trufflesqueak---', license_files=[], # already included in `TRUFFLESQUEAK_HOME`. third_party_license_files=[], dependencies=['Truffle'], + standalone_dependencies={}, truffle_jars=[ 'trufflesqueak:TRUFFLESQUEAK', 'trufflesqueak:TRUFFLESQUEAK_SHARED', ], - support_distributions=['trufflesqueak:TRUFFLESQUEAK_HOME'], - provided_executables=[ - 'bin/', - ], + support_distributions=['trufflesqueak:TRUFFLESQUEAK_HOME'] + [] if _SVM else ['trufflesqueak:TRUFFLESQUEAK_LAUNCHER_SCRIPTS'], + provided_executables=[] if _SVM else ['bin/'], library_configs=[ mx_sdk.LanguageLibraryConfig( language=LANGUAGE_ID, destination='lib/' % LANGUAGE_ID, - launchers=['bin/'], + launchers=['bin/'] if _SVM else ['bin/'], jar_distributions=['trufflesqueak:TRUFFLESQUEAK_LAUNCHER'], main_class='%s.launcher.TruffleSqueakLauncher' % PACKAGE_NAME, build_args=[ diff --git a/mx.trufflesqueak/suite.py b/mx.trufflesqueak/suite.py index 53745c994..60e59a42d 100644 --- a/mx.trufflesqueak/suite.py +++ b/mx.trufflesqueak/suite.py @@ -222,9 +222,6 @@ "README_TRUFFLESQUEAK.md": "file:README.md", "lib/": "dependency:de.hpi.swa.trufflesqueak.ffi.native", "native-image.properties": "file:mx.trufflesqueak/native-image.properties", - "bin/" : [ - "file:mx.trufflesqueak/launchers/", - ], }, "maven": False, }, @@ -243,6 +240,16 @@ ], }, + "TRUFFLESQUEAK_LAUNCHER_SCRIPTS": { + "native": True, + "platformDependent": True, + "description": "TruffleSqueak home distribution", + "layout": { + "bin/": ["file:mx.trufflesqueak/launchers/"], + }, + "maven": False, + }, + "TRUFFLESQUEAK_SHARED": { "description": "TruffleSqueak shared distribution", "moduleInfo": { diff --git a/mx.trufflesqueak/trufflesqueak-jar b/mx.trufflesqueak/trufflesqueak-jar new file mode 100644 index 000000000..eb19462fd --- /dev/null +++ b/mx.trufflesqueak/trufflesqueak-jar @@ -0,0 +1,3 @@ +COMPONENTS=TruffleSqueak +DEFAULT_DYNAMIC_IMPORTS=/trufflesqueak,/vm +DISABLE_INSTALLABLES=false diff --git a/mx.trufflesqueak/trufflesqueak-jvm b/mx.trufflesqueak/trufflesqueak-jvm deleted file mode 100644 index 605f08613..000000000 --- a/mx.trufflesqueak/trufflesqueak-jvm +++ /dev/null @@ -1,2 +0,0 @@ -DEFAULT_DYNAMIC_IMPORTS=/trufflesqueak,/vm -COMPONENTS=nfi-libffi,sdk,st,tfl diff --git a/mx.trufflesqueak/trufflesqueak-standalone b/mx.trufflesqueak/trufflesqueak-standalone new file mode 100644 index 000000000..a726e4b17 --- /dev/null +++ b/mx.trufflesqueak/trufflesqueak-standalone @@ -0,0 +1,4 @@ +COMPONENTS=TruffleSqueak,GraalVM compiler,Truffle NFI LIBFFI +DISABLE_INSTALLABLES=false +DYNAMIC_IMPORTS=/compiler,/substratevm,/vm +NATIVE_IMAGES=lib:smalltalkvm diff --git a/mx.trufflesqueak/trufflesqueak-svm b/mx.trufflesqueak/trufflesqueak-svm deleted file mode 100644 index 3850ef321..000000000 --- a/mx.trufflesqueak/trufflesqueak-svm +++ /dev/null @@ -1,2 +0,0 @@ -DEFAULT_DYNAMIC_IMPORTS=/trufflesqueak,/substratevm,/vm -COMPONENTS=cmp,nfi-libffi,sdk,st,svm,svmnfi,tfl,tflm diff --git a/mx.trufflesqueak/utils.sh b/mx.trufflesqueak/utils.sh index 165d71430..a3a030817 100755 --- a/mx.trufflesqueak/utils.sh +++ b/mx.trufflesqueak/utils.sh @@ -41,23 +41,33 @@ add-path() { echo "$(resolve-path "$1")" >> $GITHUB_PATH } -build-component() { - local env_name=$1 - local java_version=$2 - local target=$3 - local graalvm_home="$(mx --env "${env_name}" graalvm-home)" - - local distro_name="GRAALVM_TRUFFLESQUEAK${TS_INFIX}_JAVA${java_version}" - local component_name="SMALLTALK_INSTALLABLE${TS_INFIX}_JAVA${java_version}" +build-installable() { + local java_version=$1 + local graalvm_home="$(mx --env trufflesqueak-jar graalvm-home)" + local distro_name="GRAALVM_TRUFFLESQUEAK_JAR_JAVA${java_version}" + local component_name="SMALLTALK_INSTALLABLE_JAVA${java_version}" - mx --env "${env_name}" --no-download-progress build --dependencies "${component_name},${distro_name}" - cp $(mx --env "${env_name}" paths "${component_name}") "${target}" + mx --env trufflesqueak-jar --no-download-progress build --dependencies "${component_name},${distro_name}" + cp $(mx --env trufflesqueak-jar paths "${component_name}") "${INSTALLABLE_TARGET}" add-path "${graalvm_home}/bin" set-env "GRAALVM_HOME" "$(resolve-path "${graalvm_home}")" echo "[${graalvm_home} set as \$GRAALVM_HOME]" } +build-standalone() { + local java_version=$1 + local distro_name="GRAALVM_TRUFFLESQUEAK_STANDALONE_JAVA${java_version}" + local component_name="SMALLTALK_STANDALONE_SVM_JAVA${java_version}" + + mx --env trufflesqueak-standalone --no-download-progress build --dependencies "${component_name},${distro_name}" + cp "$(mx --env trufflesqueak-standalone paths "${component_name}")" "${STANDALONE_TARGET}" + + local standalone_home="$(mx --env trufflesqueak-standalone standalone-home smalltalk)" + add-path "${standalone_home}/bin" + echo "[${standalone_home}/bin added to \$PATH]" +} + deploy-asset() { local git_tag=$(git tag --points-at HEAD) if [[ -z "${git_tag}" ]]; then @@ -68,6 +78,20 @@ deploy-asset() { exit 0 fi local filename=$1 + if [[ "${filename}" == *.tar ]]; then + echo "Compressing tarball..." + gzip "${filename}" + filename="${filename}.gz" + elif [[ "${filename}" == *.zip ]]; then + echo "Compressing zip..." + tmp_dir="$(mktemp -d)" + unzip "${filename}" -d "${tmp_dir}" + rm "${filename}" + pushd "${tmp_dir}" > /dev/null + zip -r "${filename}" ./* + popd > /dev/null + fi + local auth="Authorization: token $2" local release_id @@ -170,14 +194,7 @@ download-cuis-test-image() { echo "[Cuis test image (${DEP_CUIS_TEST_IMAGE_TAG}) downloaded successfully]" } -format-native-image-config() { - readonly NI_CONFIG_BASE="${BASE_DIRECTORY}/src/de.hpi.swa.trufflesqueak/src/META-INF/native-image" - for f in "${NI_CONFIG_BASE}/"*.json; do - underscore --in "${f}" --out "${f}" --wrapwidth 80 print - done -} - -installable-filename() { +filename-installable() { local java_version=$1 local git_describe=$(git describe --tags --always) local git_short_commit=$(git log -1 --format="%h") @@ -185,6 +202,21 @@ installable-filename() { echo "trufflesqueak-installable-${java_version}-${OS_NAME}-${OS_ARCH}-${git_description}.jar" } +filename-standalone() { + local git_describe=$(git describe --tags --always) + local git_short_commit=$(git log -1 --format="%h") + local git_description="${git_describe:-${git_short_commit}}" + local file_extension="tar" && [[ "${OS_NAME}" == "windows" ]] && file_extension="zip" + echo "trufflesqueak-${git_description}-${OS_NAME}-${OS_ARCH}.${file_extension}" +} + +format-native-image-config() { + readonly NI_CONFIG_BASE="${BASE_DIRECTORY}/src/de.hpi.swa.trufflesqueak/src/META-INF/native-image" + for f in "${NI_CONFIG_BASE}/"*.json; do + underscore --in "${f}" --out "${f}" --wrapwidth 80 print + done +} + resolve-path() { if [[ "${OS_NAME}" == "windows" ]]; then # Convert Unix path to Windows path @@ -201,12 +233,15 @@ set-env() { set-up-dependencies() { local java_version=$1 - - case "$(uname -s)" in - "Linux") - sudo apt-get update --quiet --yes && sudo apt-get install --quiet --yes libz-dev libxi-dev libxtst-dev libxrender-dev libfreetype6-dev - ;; - esac + local is_standalone=$2 + + if [[ "${is_standalone}" == "true" ]]; then + case "$(uname -s)" in + "Linux") + sudo apt-get update --quiet --yes && sudo apt-get install --quiet --yes libz-dev libxi-dev libxtst-dev libxrender-dev libfreetype6-dev + ;; + esac + fi # Repository was shallow copied and Git did not fetch tags, so fetch the tag # of the commit (if any) to make it available for other Git operations. @@ -215,10 +250,14 @@ set-up-dependencies() { set-up-mx shallow-clone-graal + set-up-labsjdk "labsjdk-ce-${java_version:4}" download-trufflesqueak-icon download-trufflesqueak-test-image - set-up-labsjdk "labsjdk-ce-${java_version:4}" - set-env "INSTALLABLE_TARGET" "$(installable-filename "${java_version}")" + if [[ "${is_standalone}" == "true" ]]; then + set-env "STANDALONE_TARGET" "$(filename-standalone "${java_version}")" + else + set-env "INSTALLABLE_TARGET" "$(filename-installable "${java_version}")" + fi } set-up-labsjdk() {