Skip to content

Commit

Permalink
Merge pull request #2 from Shopify/sulong-fix-and-revert
Browse files Browse the repository at this point in the history 
Pull upstream to get Sulong fix. Revert workaround
  • Loading branch information
@XrXr
XrXr authored Oct 12, 2019
2 parents 23b0c3c + 8fa9b34 commit 0d5a26b
Show file tree
Hide file tree
Showing 482 changed files with 11,996 additions and 4,891 deletions.
11 changes: 7 additions & 4 deletions common.hocon
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,8 @@ jdks: {
oraclejdk8Debug: {name : oraclejdk, version : "8u221-jvmci-19.3-b03-fastdebug", platformspecific: true}
oraclejdk11: {name : oraclejdk, version : "11.0.3+12", platformspecific: true}
openjdk11: {name : openjdk, version : "11.0.3+7", platformspecific: true}
labsjdk11: {name : labsjdk, version : "ce-11-20190917-093434", platformspecific: true}
labsjdk-ce-11: {name : labsjdk, version : "ce-11.0.5+9-jvmci-19.3-b03", platformspecific: true}
labsjdk-ee-11: {name : labsjdk, version : "ee-11.0.5+1-jvmci-19.3-b03", platformspecific: true}
}

# This must always point to HEAD in the master branch but can be used to point
Expand All @@ -28,15 +29,17 @@ download-hsdis : {
]
}

oraclejdk8 : { downloads : { JAVA_HOME : ${jdks.oraclejdk8}, EXTRA_JAVA_HOMES : { pathlist :[ ${jdks.oraclejdk11} ]} }}
oraclejdk8 : { downloads : { JAVA_HOME : ${jdks.oraclejdk8}, EXTRA_JAVA_HOMES : { pathlist :[ ${jdks.labsjdk-ee-11} ]} }}
oraclejdk8Only : { downloads : { JAVA_HOME : ${jdks.oraclejdk8} }}
oraclejdk8Debug : { downloads : { JAVA_HOME : ${jdks.oraclejdk8Debug}, EXTRA_JAVA_HOMES : { pathlist :[ ${jdks.oraclejdk11} ]} }}
oraclejdk8Debug : { downloads : { JAVA_HOME : ${jdks.oraclejdk8Debug}, EXTRA_JAVA_HOMES : { pathlist :[ ${jdks.labsjdk-ee-11} ]} }}

openjdk8 : { downloads : { JAVA_HOME : ${jdks.openjdk8} }}

oraclejdk11 : { downloads : { JAVA_HOME : ${jdks.oraclejdk11} }}
openjdk11 : { downloads : { JAVA_HOME : ${jdks.openjdk11} }}
labsjdk11 : { downloads : { JAVA_HOME : ${jdks.labsjdk11} }}

labsjdk-ce-11 : { downloads : { JAVA_HOME : ${jdks.labsjdk-ce-11} }}
labsjdk-ee-11 : { downloads : { JAVA_HOME : ${jdks.labsjdk-ee-11} }}

common : ${mx} {
timelimit : "30:00"
Expand Down
46 changes: 23 additions & 23 deletions compiler/ci_common/gate_tasks.hocon
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,39 +2,39 @@

builds += [
# Darwin AMD64
${gateTest} ${oraclejdk8} ${gateDarwinAMD64} {name: "gate-compiler-test-8-darwin-amd64", timelimit: "40:00"}
${gateTest} ${openjdk8} ${gateDarwinAMD64} {name: "weekly-test-compiler-test-openjdk8-darwin-amd64"} ${graalWeekly}
${gateTest} ${labsjdk11} ${gateDarwinAMD64} {name: "weekly-test-compiler-test-11-darwin-amd64"} ${graalWeekly}
# ${gateTestBenchmark} ${oraclejdk8} ${gateDarwinAMD64} {name: "weekly-test-compiler-benchmarktest-8-darwin-amd64"} ${graalWeekly}
${gateTest} ${oraclejdk8} ${gateDarwinAMD64} {name: "gate-compiler-test-8-darwin-amd64", timelimit: "40:00"}
${gateTest} ${openjdk8} ${gateDarwinAMD64} {name: "weekly-test-compiler-test-openjdk8-darwin-amd64"} ${graalWeekly}
${gateTest} ${labsjdk-ee-11} ${gateDarwinAMD64} {name: "weekly-test-compiler-test-labsjdk-ee-11-darwin-amd64"} ${graalWeekly}
# ${gateTestBenchmark} ${oraclejdk8} ${gateDarwinAMD64} {name: "weekly-test-compiler-benchmarktest-8-darwin-amd64"} ${graalWeekly}

# Windows AMD64
${gateTest} ${oraclejdk8} ${gateWindowsAMD64} {name: "gate-compiler-test-8-windows-amd64", timelimit: "50:00", packages: {msvc: "==10.0" }}

# Linux AMD64
${gateTest} ${oraclejdk8} ${gateLinuxAMD64} {name: "gate-compiler-test-8-linux-amd64"}
${gateTest} ${openjdk8} ${gateLinuxAMD64} {name: "weekly-test-compiler-test-openjdk8-linux-amd64"} ${graalWeekly}
${gateTestMaxVS} ${oraclejdk8} ${gateLinuxAMD64} {name: "weekly-test-compiler-test-8-linux-amd64-maxvectorsize"} ${graalWeekly}
${gateTestAVX0} ${oraclejdk8} ${gateLinuxAMD64} {name: "weekly-test-compiler-test-8-linux-amd64-avx0"} ${graalWeekly}
${gateTestAVX1} ${oraclejdk8} ${gateLinuxAMD64} {name: "weekly-test-compiler-test-8-linux-amd64-avx1"} ${graalWeekly}
${gateTest} ${labsjdk11} ${gateLinuxAMD64} {name: "gate-compiler-test-11-linux-amd64"}
${gateJavaBaseTest} ${labsjdk11} ${gateLinuxAMD64} {name: "weekly-test-compiler-javabasetest-11-linux-amd64"} ${graalWeekly}
${gateTestCTW} ${oraclejdk8} ${gateLinuxAMD64} {name: "gate-compiler-ctw-8-linux-amd64"}
${gateTestCTWWeekly} ${openjdk8} ${gateLinuxAMD64} {name: "weekly-test-compiler-ctw-openjdk8-linux-amd64", timelimit: "2:00:00"} ${graalWeekly}
${gateTestCTWWeekly} ${labsjdk11} ${gateLinuxAMD64} {name: "weekly-test-compiler-ctw-11-linux-amd64"} ${graalWeekly}
${gateTest} ${oraclejdk8Debug} ${gateLinuxAMD64} {name: "weekly-test-compiler-test-8-linux-amd64-fastdebug", timelimit: "1:50:00"} ${graalWeekly}
${gateTestBenchmark} ${oraclejdk8} ${gateLinuxAMD64} {name: "gate-compiler-benchmarktest-8-linux-amd64"}
${gateTestBenchmark} ${oraclejdk8Debug} ${gateLinuxAMD64} {name: "weekly-test-compiler-benchmarktest-8-linux-amd64-fastdebug"} ${graalWeekly}
${gateStyle} ${oraclejdk8} ${gateLinuxAMD64} {name: "gate-compiler-style-linux-amd64"}
${gateCoverage} ${oraclejdk8} ${gateLinuxAMD64} {name: "weekly-compiler-coverage-8-linux-amd64", timelimit: "1:50:00"} ${graalWeekly}
${gateTest} ${oraclejdk8} ${gateLinuxAMD64} {name: "gate-compiler-test-8-linux-amd64"}
${gateTest} ${openjdk8} ${gateLinuxAMD64} {name: "weekly-test-compiler-test-openjdk8-linux-amd64"} ${graalWeekly}
${gateTestMaxVS} ${oraclejdk8} ${gateLinuxAMD64} {name: "weekly-test-compiler-test-8-linux-amd64-maxvectorsize"} ${graalWeekly}
${gateTestAVX0} ${oraclejdk8} ${gateLinuxAMD64} {name: "weekly-test-compiler-test-8-linux-amd64-avx0"} ${graalWeekly}
${gateTestAVX1} ${oraclejdk8} ${gateLinuxAMD64} {name: "weekly-test-compiler-test-8-linux-amd64-avx1"} ${graalWeekly}
${gateTest} ${labsjdk-ee-11} ${gateLinuxAMD64} {name: "gate-compiler-test-labsjdk-ee-11-linux-amd64"}
${gateJavaBaseTest} ${labsjdk-ee-11} ${gateLinuxAMD64} {name: "weekly-test-compiler-javabasetest-labsjdk-ee-11-linux-amd64"} ${graalWeekly}
${gateTestCTW} ${oraclejdk8} ${gateLinuxAMD64} {name: "gate-compiler-ctw-8-linux-amd64"}
${gateTestCTWWeekly} ${openjdk8} ${gateLinuxAMD64} {name: "weekly-test-compiler-ctw-openjdk8-linux-amd64", timelimit: "2:00:00"} ${graalWeekly}
${gateTestCTWWeekly} ${labsjdk-ee-11} ${gateLinuxAMD64} {name: "weekly-test-compiler-ctw-labsjdk-ee-11-linux-amd64"} ${graalWeekly}
${gateTest} ${oraclejdk8Debug} ${gateLinuxAMD64} {name: "weekly-test-compiler-test-8-linux-amd64-fastdebug", timelimit: "1:50:00"} ${graalWeekly}
${gateTestBenchmark} ${oraclejdk8} ${gateLinuxAMD64} {name: "gate-compiler-benchmarktest-8-linux-amd64"}
${gateTestBenchmark} ${oraclejdk8Debug} ${gateLinuxAMD64} {name: "weekly-test-compiler-benchmarktest-8-linux-amd64-fastdebug"} ${graalWeekly}
${gateStyle} ${oraclejdk8} ${gateLinuxAMD64} {name: "gate-compiler-style-linux-amd64"}
${gateCoverage} ${oraclejdk8} ${gateLinuxAMD64} {name: "weekly-compiler-coverage-8-linux-amd64", timelimit: "1:50:00"} ${graalWeekly}

${gateTestCompileImmediately} ${oraclejdk8} ${gateLinuxAMD64} {name: "gate-compiler-test-compile-immediately-8-linux-amd64"}
${gateMathStubsListener} {name: "daily-hotspot-mathstubs-listener"}

# Solaris SPARC
${gateTest} ${oraclejdk8} ${gateSolarisSPARC} {name: "gate-compiler-test-8-solaris-sparcv9", timelimit: "1:00:00"}
${gateTest} ${labsjdk11} ${gateSolarisSPARC} {name: "gate-compiler-test-11-solaris-sparcv9", timelimit: "1:00:00"}
${gateTest} ${labsjdk-ee-11} ${gateSolarisSPARC} {name: "gate-compiler-test-labsjdk-ee-11-solaris-sparcv9", timelimit: "1:00:00"}
${gateTestCTW} ${oraclejdk8} ${gateSolarisSPARC} {name: "gate-compiler-ctw-8-solaris-sparcv9", timelimit: "1:00:00"}
${gateTestCTWWeekly} ${labsjdk11} ${gateSolarisSPARC} {name: "weekly-test-compiler-ctw-11-solaris-sparcv9", timelimit: "1:50:00"} ${graalWeekly}
${gateTestCTWWeekly} ${labsjdk-ee-11} ${gateSolarisSPARC} {name: "weekly-test-compiler-ctw-labsjdk-ee-11-solaris-sparcv9", timelimit: "1:50:00"} ${graalWeekly}
${gateTest} ${oraclejdk8Debug} ${gateSolarisSPARC} {name: "weekly-test-compiler-test-8-solaris-sparcv9-fastdebug", timelimit: "1:50:00"} ${graalWeekly}
${gateTestBenchmark} ${oraclejdk8} ${gateSolarisSPARC} {name: "gate-compiler-benchmarktest-8-solaris-sparcv9", timelimit: "1:00:00"}
# ${gateTestBenchmark} ${oraclejdk8Debug} ${gateSolarisSPARC} {name: "gate-compiler-benchmarktest-8-solaris-sparcv9-fastdebug", timelimit: "1:50:00"}
Expand All @@ -43,6 +43,6 @@ builds += [
#${gateTest} ${oraclejdk8} ${gateLinuxSPARC} {name: "weekly-test-compiler-test-8-linux-sparcv9", timelimit: "1:00:00"} ${graalWeekly}

# Linux AArch64
${gateTest} ${labsjdk11} ${gateLinuxAArch64} {name: "gate-compiler-test-11-linux-aarch64", timelimit: "1:50:00"}
${gateTestCTW} ${labsjdk11} ${gateLinuxAArch64} {name: "gate-compiler-ctw-11-linux-aarch64", timelimit: "1:50:00"}
${gateTest} ${labsjdk-ce-11} ${gateLinuxAArch64} {name: "gate-compiler-test-labsjdk-ce-11-linux-aarch64", timelimit: "1:50:00"}
${gateTestCTW} ${labsjdk-ce-11} ${gateLinuxAArch64} {name: "gate-compiler-ctw-labsjdk-ce-11-linux-aarch64", timelimit: "1:50:00"}
]
14 changes: 7 additions & 7 deletions compiler/ci_common/gate_tasks_bootstrap.hocon
View file
Original file line number Diff line number Diff line change
@@ -1,15 +1,15 @@

builds += [
# Darwin AMD64
${gateBootstrapLite} ${oraclejdk8} ${gateDarwinAMD64Bootstrap} {name: "gate-compiler-bootstraplite-8-darwin-amd64"}
${gateBootstrapLite} ${labsjdk11} ${gateDarwinAMD64Bootstrap} {name: "weekly-test-compiler-bootstraplite-11-darwin-amd64"} ${graalWeekly}
${gateBootstrapLite} ${oraclejdk8} ${gateDarwinAMD64Bootstrap} {name: "gate-compiler-bootstraplite-8-darwin-amd64"}
${gateBootstrapLite} ${labsjdk-ee-11} ${gateDarwinAMD64Bootstrap} {name: "weekly-test-compiler-bootstraplite-labsjdk-ee-11-darwin-amd64"} ${graalWeekly}

# Linux AMD64
${gateBootstrap} ${oraclejdk8} ${gateLinuxAMD64Bootstrap} {name: "gate-compiler-bootstrap-8-linux-amd64"}
${gateBootstrapFullVerify} ${oraclejdk8} ${gateLinuxAMD64Bootstrap} {name: "gate-compiler-bootstrapfullverify-8-linux-amd64"}
${gateBootstrap} ${labsjdk11} ${gateLinuxAMD64Bootstrap} {name: "gate-compiler-bootstrap-11-linux-amd64"}
${gateBootstrap} ${oraclejdk8} ${gateLinuxAMD64Bootstrap} {name: "gate-compiler-bootstrap-8-linux-amd64"}
${gateBootstrapFullVerify} ${oraclejdk8} ${gateLinuxAMD64Bootstrap} {name: "gate-compiler-bootstrapfullverify-8-linux-amd64"}
${gateBootstrap} ${labsjdk-ee-11} ${gateLinuxAMD64Bootstrap} {name: "gate-compiler-bootstrap-labsjdk-ee-11-linux-amd64"}

# Solaris SPARC
${gateBootstrap} ${oraclejdk8} ${gateSolarisSPARCBootstrap} {name: "gate-compiler-bootstrap-8-solaris-sparcv9", timelimit: "1:00:00"}
${gateBootstrap} ${labsjdk11} ${gateSolarisSPARCBootstrap} {name: "weekly-test-compiler-bootstrap-11-solaris-sparcv9", timelimit: "1:00:00"} ${graalWeekly}
${gateBootstrap} ${oraclejdk8} ${gateSolarisSPARCBootstrap} {name: "gate-compiler-bootstrap-8-solaris-sparcv9", timelimit: "1:00:00"}
${gateBootstrap} ${labsjdk-ee-11} ${gateSolarisSPARCBootstrap} {name: "weekly-test-compiler-bootstrap-labsjdk-ee-11-solaris-sparcv9", timelimit: "1:00:00"} ${graalWeekly}
]
4 changes: 2 additions & 2 deletions compiler/ci_includes/gate_tasks.hocon
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# Build targets

builds += [
${gateJAOTC} ${labsjdk11} ${gateLinuxAMD64} {name: "gate-compiler-jaotc-linux-amd64"}
${gateJAOTC} ${labsjdk11} ${gateLinuxAArch64} {name: "gate-compiler-jaotc-linux-aarch64", timelimit: "1:30:00"}
${gateJAOTC} ${labsjdk-ce-11} ${gateLinuxAMD64} {name: "gate-compiler-jaotc-labsjdk-ce-11-linux-amd64"}
${gateJAOTC} ${labsjdk-ce-11} ${gateLinuxAArch64} {name: "gate-compiler-jaotc-labsjdk-ce-11-linux-aarch64", timelimit: "1:30:00"}
]
141 changes: 93 additions & 48 deletions compiler/mx.compiler/mx_compiler.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -106,29 +106,104 @@ def get_vm_prefix(asList=True):
#: JDK9 or later is being used (checked above).
isJDK8 = jdk.javaCompliance < '1.9'

class SafeDirectoryUpdater(object):
"""
Multi-thread safe context manager for creating/updating a directory.
:Example:
# Compiles `sources` into `dst` with javac. If multiple threads/processes are
# performing this compilation concurrently, the contents of `dst`
# will reflect the complete results of one of the compilations
# from the perspective of other threads/processes.
with SafeDirectoryUpdater(dst) as sdu:
mx.run([jdk.javac, '-d', sdu.directory, sources])
"""
def __init__(self, directory, create=False):
"""
:param directory: the target directory that will be created/updated within the context.
The working copy of the directory is accessed via `self.directory`
within the context.
"""

self.target = directory
self.workspace = None
self.directory = None
self.create = create

def __enter__(self):
parent = dirname(self.target)
self.workspace = tempfile.mkdtemp(dir=parent)
self.directory = join(self.workspace, basename(self.target))
if self.create:
mx.ensure_dir_exists(self.directory)
self.target_timestamp = mx.TimeStampFile(self.target)
return self

def __exit__(self, exc_type, exc_value, traceback):
if exc_type is not None:
shutil.rmtree(self.workspace)
raise

# Try delete the target directory if it existed prior to creating
# self.workspace and has not been modified in between.
if self.target_timestamp.timestamp is not None and self.target_timestamp.timestamp == mx.TimeStampFile(self.target).timestamp:
old_target = join(self.workspace, 'to_deleted_' + basename(self.target))
try:
os.rename(self.target, old_target)
except:
# Silently assume another process won the race to rename dst_jdk_dir
pass

# Try atomically move self.directory to self.target
try:
os.rename(self.directory, self.target)
except:
if not exists(self.target):
raise
else:
# Silently assume another process won the race to create self.target
pass

shutil.rmtree(self.workspace)

def _check_jvmci_version(jdk):
"""
Runs a Java utility to check that `jdk` supports the minimum JVMCI API required by Graal.
This runs a version of org.graalvm.compiler.hotspot.JVMCIVersionCheck that is "moved"
to the unnamed package. Without this, on JDK 10+, the class in the jdk.internal.vm.compiler
module will be run instead of the version on the class path.
"""
simplename = 'JVMCIVersionCheck'
name = 'org.graalvm.compiler.hotspot.' + simplename
unqualified_name = 'JVMCIVersionCheck'
qualified_name = 'org.graalvm.compiler.hotspot.' + unqualified_name
binDir = mx.ensure_dir_exists(join(_suite.get_output_root(), '.jdk' + str(jdk.version)))

if isinstance(_suite, mx.BinarySuite):
javaSource = join(binDir, simplename + '.java')
if not exists(javaSource):
dists = [d for d in _suite.dists if d.name == 'GRAAL_HOTSPOT']
assert len(dists) == 1, 'could not find GRAAL_HOTSPOT distribution'
d = dists[0]
assert exists(d.sourcesPath), 'missing expected file: ' + d.sourcesPath
dists = [d for d in _suite.dists if d.name == 'GRAAL_HOTSPOT']
assert len(dists) == 1, 'could not find GRAAL_HOTSPOT distribution'
d = dists[0]
assert exists(d.sourcesPath), 'missing expected file: ' + d.sourcesPath
source_timestamp = getmtime(d.sourcesPath)
def source_supplier():
with zipfile.ZipFile(d.sourcesPath, 'r') as zf:
with open(javaSource, 'w') as fp:
fp.write(zf.read(name.replace('.', '/') + '.java'))
return zf.read(qualified_name.replace('.', '/') + '.java')
else:
javaSource = join(_suite.dir, 'src', 'org.graalvm.compiler.hotspot', 'src', name.replace('.', '/') + '.java')
javaClass = join(binDir, name.replace('.', '/') + '.class')
if not exists(javaClass) or getmtime(javaClass) < getmtime(javaSource):
mx.run([jdk.javac, '-d', binDir, javaSource])
mx.run([jdk.java, '-cp', binDir, name])
source_path = join(_suite.dir, 'src', 'org.graalvm.compiler.hotspot', 'src', qualified_name.replace('.', '/') + '.java')
source_timestamp = getmtime(source_path)
def source_supplier():
with open(source_path, 'r') as fp:
return fp.read()

unqualified_class_file = join(binDir, unqualified_name + '.class')
if not exists(unqualified_class_file) or getmtime(unqualified_class_file) < source_timestamp:
with SafeDirectoryUpdater(binDir, create=True) as sdu:
unqualified_source_path = join(sdu.directory, unqualified_name + '.java')
with open(unqualified_source_path, 'w') as fp:
fp.write(source_supplier().replace('package org.graalvm.compiler.hotspot;', ''))
mx.run([jdk.javac, '-d', sdu.directory, unqualified_source_path])
mx.run([jdk.java, '-cp', binDir, unqualified_name])

if os.environ.get('JVMCI_VERSION_CHECK', None) != 'ignore':
_check_jvmci_version(jdk)
Expand Down Expand Up @@ -256,7 +331,7 @@ def ctw(args, extraVMarguments=None):

if not _get_XX_option_value(vmargs, 'UseJVMCINativeLibrary', False):
if _get_XX_option_value(vmargs, 'UseJVMCICompiler', False):
if _get_XX_option_value(vmargs, 'BootstrapJVMCI', False) == None:
if _get_XX_option_value(vmargs, 'BootstrapJVMCI', False) is None:
vmargs.append('-XX:+BootstrapJVMCI')

mainClassAndArgs = []
Expand Down Expand Up @@ -1133,13 +1208,8 @@ def _update_graaljdk(src_jdk, dst_jdk_dir=None, root_module_names=None, export_t

mx.log('Updating/creating {} from {} since {}'.format(dst_jdk_dir, src_jdk.home, update_reason))

# The GraalJDK is created in a temporary directory since multiple processes
# may be racing to do the creation.
tmp_dir = tempfile.mkdtemp(prefix=basename(dst_jdk_dir), dir=dirname(dst_jdk_dir))
tmp_dst_jdk_dir = join(tmp_dir, 'jdk')
dst_jdk_dir_timestamp = mx.TimeStampFile(dst_jdk_dir)
try:

with SafeDirectoryUpdater(dst_jdk_dir) as sdu:
tmp_dst_jdk_dir = sdu.directory
def _copy_file(src, dst):
mx.log('Copying {} to {}'.format(src, dst))
shutil.copyfile(src, dst)
Expand Down Expand Up @@ -1236,31 +1306,6 @@ def _copy_file(src, dst):
with open(join(tmp_dst_jdk_dir, 'proguard.map'), 'w') as fp:
fp.write(unstrip_map)

except:
shutil.rmtree(tmp_dir)
raise

# Try delete the current dst_jdk_dir if it existed prior to creating
# tmp_dst_jdk_dir and has not be modified in between.
if dst_jdk_dir_timestamp.timestamp is not None and dst_jdk_dir_timestamp.timestamp == mx.TimeStampFile(dst_jdk_dir).timestamp:
old_dst_jdk_dir = join(tmp_dir, 'old_jdk')
try:
os.rename(dst_jdk_dir, old_dst_jdk_dir)
except:
# Silently assume another process won the race to rename dst_jdk_dir
pass

# Try atomically move tmp_dst_jdk_dir to dst_jdk_dir
try:
os.rename(tmp_dst_jdk_dir, dst_jdk_dir)
except:
if not exists(dst_jdk_dir):
raise
else:
# Silently assume another process won the race to create dst_jdk_dir
pass

shutil.rmtree(tmp_dir)
return dst_jdk_dir, True

__graal_config = None
Expand Down
Loading

0 comments on commit 0d5a26b

Please sign in to comment.