From 9f3cf28e0a81917c082b199dc716ea9854413c30 Mon Sep 17 00:00:00 2001 From: Yudi Zheng Date: Tue, 14 May 2024 09:58:22 +0200 Subject: [PATCH] Use hashed lookup in the secondary supers array. --- .../test/SecondarySupersLookupTest.java | 276 ++++++++++++++++++ .../compiler/asm/amd64/AMD64Assembler.java | 8 + .../amd64/AMD64ArithmeticLIRGenerator.java | 13 +- .../core/amd64/AMD64NodeMatchRules.java | 16 + .../hotspot/GraalHotSpotVMConfig.java | 2 + .../meta/HotSpotHostForeignCallsProvider.java | 5 + .../replacements/HotSpotReplacementsUtil.java | 14 + .../replacements/InstanceOfSnippets.java | 24 +- .../replacements/TypeCheckSnippetUtils.java | 112 ++++++- .../stubs/LookUpSecondarySupersTableStub.java | 128 ++++++++ .../compiler/lir/amd64/AMD64BitCountOp.java | 129 ++++++++ .../compiler/lir/amd64/AMD64ControlFlow.java | 19 ++ .../StandardGraphBuilderPlugins.java | 8 + .../aarch64/AArch64BitCountNode.java | 44 --- .../aarch64/AArch64GraphBuilderPlugins.java | 7 - .../amd64/AMD64GraphBuilderPlugins.java | 10 - .../replacements/nodes/BitCountNode.java | 6 +- 17 files changed, 728 insertions(+), 93 deletions(-) create mode 100644 compiler/src/jdk.graal.compiler.test/src/jdk/graal/compiler/hotspot/test/SecondarySupersLookupTest.java create mode 100644 compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/stubs/LookUpSecondarySupersTableStub.java create mode 100644 compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/lir/amd64/AMD64BitCountOp.java delete mode 100644 compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/replacements/aarch64/AArch64BitCountNode.java diff --git a/compiler/src/jdk.graal.compiler.test/src/jdk/graal/compiler/hotspot/test/SecondarySupersLookupTest.java b/compiler/src/jdk.graal.compiler.test/src/jdk/graal/compiler/hotspot/test/SecondarySupersLookupTest.java new file mode 100644 index 000000000000..926957f8f3ef --- /dev/null +++ b/compiler/src/jdk.graal.compiler.test/src/jdk/graal/compiler/hotspot/test/SecondarySupersLookupTest.java @@ -0,0 +1,276 @@ +/* + * Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package jdk.graal.compiler.hotspot.test; + +import org.junit.Test; + +public class SecondarySupersLookupTest extends HotSpotGraalCompilerTest { + + interface I01 { + } + + interface I02 extends I01 { + } + + interface I03 extends I02 { + } + + interface I04 extends I03 { + } + + interface I05 extends I04 { + } + + interface I06 extends I05 { + } + + interface I07 extends I06 { + } + + interface I08 extends I07 { + } + + interface I09 extends I08 { + } + + interface I10 extends I09 { + } + + interface I11 extends I10 { + } + + interface I12 extends I11 { + } + + interface I13 extends I12 { + } + + interface I14 extends I13 { + } + + interface I15 extends I14 { + } + + interface I16 extends I15 { + } + + interface I17 extends I16 { + } + + interface I18 extends I17 { + } + + interface I19 extends I18 { + } + + interface I20 extends I19 { + } + + interface I21 extends I20 { + } + + interface I22 extends I21 { + } + + interface I23 extends I22 { + } + + interface I24 extends I23 { + } + + interface I25 extends I24 { + } + + interface I26 extends I25 { + } + + interface I27 extends I26 { + } + + interface I28 extends I27 { + } + + interface I29 extends I28 { + } + + interface I30 extends I29 { + } + + interface I31 extends I30 { + } + + interface I32 extends I31 { + } + + interface I33 extends I32 { + } + + interface I34 extends I33 { + } + + interface I35 extends I34 { + } + + interface I36 extends I35 { + } + + interface I37 extends I36 { + } + + interface I38 extends I37 { + } + + interface I39 extends I38 { + } + + interface I40 extends I39 { + } + + interface I41 extends I40 { + } + + interface I42 extends I41 { + } + + interface I43 extends I42 { + } + + interface I44 extends I43 { + } + + interface I45 extends I44 { + } + + interface I46 extends I45 { + } + + interface I47 extends I46 { + } + + interface I48 extends I47 { + } + + interface I49 extends I48 { + } + + interface I50 extends I49 { + } + + interface I51 extends I50 { + } + + interface I52 extends I51 { + } + + interface I53 extends I52 { + } + + interface I54 extends I53 { + } + + interface I55 extends I54 { + } + + interface I56 extends I55 { + } + + interface I57 extends I56 { + } + + interface I58 extends I57 { + } + + interface I59 extends I58 { + } + + interface I60 extends I59 { + } + + interface I61 extends I60 { + } + + interface I62 extends I61 { + } + + interface I63 extends I62 { + } + + interface I64 extends I63 { + } + + final Object obj01 = new I01() { + }; + final Object obj08 = new I08() { + }; + final Object obj34 = new I34() { + }; + final Object obj60 = new I60() { + }; + final Object obj64 = new I64() { + }; + + public static boolean instanceOfI01(Object o) { + return o instanceof I01; + } + + @Test + public void testHashHitMiss() { + // hash miss + test("instanceOfI01", new Object()); + // hash hit + test("instanceOfI01", obj01); + } + + public static boolean instanceOfI08(Object o) { + return o instanceof I08; + } + + public static boolean instanceOfI34(Object o) { + return o instanceof I34; + } + + public static boolean instanceOfI60(Object o) { + return o instanceof I60; + } + + @Test + public void testHashCollision() { + // hash collision, call into slow path, more than 64 secondary supers + test("instanceOfI01", obj64); + // I08, I34 and I60 have the same hash value because + // the hashing algorithm depends on the type name. + // hash collision, call into slow path, next slot hit + test("instanceOfI08", obj34); + // hash collision, next slot empty + test("instanceOfI34", obj08); + // hash collision, call into slow path, next slot miss + test("instanceOfI60", obj34); + + // other tests for completion + test("instanceOfI08", obj60); + test("instanceOfI34", obj60); + test("instanceOfI60", obj08); + } +} diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/asm/amd64/AMD64Assembler.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/asm/amd64/AMD64Assembler.java index 48447123620a..b2cccdd7e866 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/asm/amd64/AMD64Assembler.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/asm/amd64/AMD64Assembler.java @@ -4837,6 +4837,14 @@ public final void testq(Register dst, AMD64Address src) { AMD64RMOp.TEST.emit(this, OperandSize.QWORD, dst, src); } + public final void btq(Register src, int imm8) { + prefixq(src); + emitByte(0x0F); + emitByte(0xBA); + emitModRM(4, src); + emitByte(imm8); + } + public final void btrq(Register src, int imm8) { prefixq(src); emitByte(0x0F); diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/core/amd64/AMD64ArithmeticLIRGenerator.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/core/amd64/AMD64ArithmeticLIRGenerator.java index ce512d835d06..6061ab39c8eb 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/core/amd64/AMD64ArithmeticLIRGenerator.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/core/amd64/AMD64ArithmeticLIRGenerator.java @@ -121,6 +121,7 @@ import jdk.graal.compiler.lir.amd64.AMD64ArithmeticLIRGeneratorTool; import jdk.graal.compiler.lir.amd64.AMD64Binary; import jdk.graal.compiler.lir.amd64.AMD64BinaryConsumer; +import jdk.graal.compiler.lir.amd64.AMD64BitCountOp; import jdk.graal.compiler.lir.amd64.AMD64BitSwapOp; import jdk.graal.compiler.lir.amd64.AMD64ClearRegisterOp; import jdk.graal.compiler.lir.amd64.AMD64ConvertFloatToIntegerOp; @@ -1059,11 +1060,15 @@ public Value emitZeroExtend(Value inputVal, int fromBits, int toBits, boolean re @Override public Variable emitBitCount(Value value) { Variable result = getLIRGen().newVariable(LIRKind.combine(value).changeType(AMD64Kind.DWORD)); - assert ((AMD64Kind) value.getPlatformKind()).isInteger(); - if (value.getPlatformKind() == AMD64Kind.QWORD) { - getLIRGen().append(new AMD64Unary.RMOp(POPCNT, QWORD, result, asAllocatable(value))); + if (getLIRGen().target().arch.getFeatures().contains(CPUFeature.POPCNT)) { + assert ((AMD64Kind) value.getPlatformKind()).isInteger(); + if (value.getPlatformKind() == AMD64Kind.QWORD) { + getLIRGen().append(new AMD64Unary.RMOp(POPCNT, QWORD, result, asAllocatable(value))); + } else { + getLIRGen().append(new AMD64Unary.RMOp(POPCNT, DWORD, result, asAllocatable(value))); + } } else { - getLIRGen().append(new AMD64Unary.RMOp(POPCNT, DWORD, result, asAllocatable(value))); + getLIRGen().append(new AMD64BitCountOp(getLIRGen(), result, asAllocatable(value))); } return result; } diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/core/amd64/AMD64NodeMatchRules.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/core/amd64/AMD64NodeMatchRules.java index d31251bf4a9e..fd9b3132ad29 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/core/amd64/AMD64NodeMatchRules.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/core/amd64/AMD64NodeMatchRules.java @@ -69,6 +69,7 @@ import jdk.graal.compiler.lir.LabelRef; import jdk.graal.compiler.lir.amd64.AMD64AddressValue; import jdk.graal.compiler.lir.amd64.AMD64BinaryConsumer; +import jdk.graal.compiler.lir.amd64.AMD64ControlFlow; import jdk.graal.compiler.lir.amd64.AMD64ControlFlow.TestBranchOp; import jdk.graal.compiler.lir.amd64.AMD64ControlFlow.TestConstBranchOp; import jdk.graal.compiler.lir.amd64.AMD64UnaryConsumer; @@ -373,6 +374,21 @@ public ComplexMatchResult resetLowestSetBit(ValueNode a, ValueNode b) { } } + @MatchRule("(If (IntegerTest value Constant=a))") + public ComplexMatchResult testBitAndBranch(IfNode root, ValueNode value, ConstantNode a) { + long constant = a.asJavaConstant().asLong(); + if (Long.bitCount(constant) == 1) { + return builder -> { + LabelRef trueDestination = getLIRBlock(root.trueSuccessor()); + LabelRef falseDestination = getLIRBlock(root.falseSuccessor()); + gen.append(new AMD64ControlFlow.BitTestAndBranchOp(trueDestination, falseDestination, gen.asAllocatable(operand(value)), + root.getTrueSuccessorProbability(), Long.numberOfTrailingZeros(constant))); + return null; + }; + } + return null; + } + @MatchRule("(If (IntegerTest Read=access value))") @MatchRule("(If (IntegerTest FloatingRead=access value))") public ComplexMatchResult integerTestBranchMemory(IfNode root, LIRLowerableAccess access, ValueNode value) { diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/GraalHotSpotVMConfig.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/GraalHotSpotVMConfig.java index 1737a1e87803..32dd86399af2 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/GraalHotSpotVMConfig.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/GraalHotSpotVMConfig.java @@ -212,6 +212,8 @@ public final int logMinObjAlignment() { public final int superCheckOffsetOffset = getFieldOffset("Klass::_super_check_offset", Integer.class, "juint"); public final int secondarySuperCacheOffset = getFieldOffset("Klass::_secondary_super_cache", Integer.class, "Klass*"); public final int secondarySupersOffset = getFieldOffset("Klass::_secondary_supers", Integer.class, "Array*"); + public final int klassHashSlotOffset = getFieldOffset("Klass::_hash_slot", Integer.class, "uint8_t", 0, JDK >= 23); + public final int klassBitmapOffset = getFieldOffset("Klass::_bitmap", Integer.class, "uintx", 0, JDK >= 23); // JDK-8186777 public final int classMirrorOffset = getFieldOffset("Klass::_java_mirror", Integer.class, "OopHandle"); diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/meta/HotSpotHostForeignCallsProvider.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/meta/HotSpotHostForeignCallsProvider.java index 5a1807f98771..c98e16c7f6c2 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/meta/HotSpotHostForeignCallsProvider.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/meta/HotSpotHostForeignCallsProvider.java @@ -82,6 +82,7 @@ import static jdk.graal.compiler.hotspot.replacements.MonitorSnippets.MONITORENTER; import static jdk.graal.compiler.hotspot.replacements.MonitorSnippets.MONITOREXIT; import static jdk.graal.compiler.hotspot.stubs.ExceptionHandlerStub.EXCEPTION_HANDLER_FOR_PC; +import static jdk.graal.compiler.hotspot.stubs.LookUpSecondarySupersTableStub.LOOKUP_SECONDARY_SUPERS_TABLE_SLOW_PATH; import static jdk.graal.compiler.hotspot.stubs.StubUtil.VM_MESSAGE_C; import static jdk.graal.compiler.hotspot.stubs.UnwindExceptionToCallerStub.EXCEPTION_HANDLER_FOR_RETURN_ADDRESS; import static jdk.graal.compiler.nodes.java.ForeignCallDescriptors.REGISTER_FINALIZER; @@ -120,6 +121,7 @@ import jdk.graal.compiler.hotspot.stubs.IntegerExactOverflowExceptionStub; import jdk.graal.compiler.hotspot.stubs.IntrinsicStubsGen; import jdk.graal.compiler.hotspot.stubs.LongExactOverflowExceptionStub; +import jdk.graal.compiler.hotspot.stubs.LookUpSecondarySupersTableStub; import jdk.graal.compiler.hotspot.stubs.NegativeArraySizeExceptionStub; import jdk.graal.compiler.hotspot.stubs.NullPointerExceptionStub; import jdk.graal.compiler.hotspot.stubs.OutOfBoundsExceptionStub; @@ -511,6 +513,9 @@ public void initialize(HotSpotProviders providers, OptionValues options) { registerStubCall(exceptionRuntimeCalls.get(BytecodeExceptionKind.ILLEGAL_ARGUMENT_EXCEPTION_ARGUMENT_IS_NOT_AN_ARRAY), SAFEPOINT, HAS_SIDE_EFFECT, DESTROYS_ALL_CALLER_SAVE_REGISTERS, any()))); + link(new LookUpSecondarySupersTableStub(options, providers, + registerStubCall(LOOKUP_SECONDARY_SUPERS_TABLE_SLOW_PATH, DESTROYS_ALL_CALLER_SAVE_REGISTERS))); + linkForeignCall(options, providers, IDENTITY_HASHCODE, c.identityHashCodeAddress, PREPEND_THREAD); linkForeignCall(options, providers, createDescriptor(REGISTER_FINALIZER, SAFEPOINT, HAS_SIDE_EFFECT, any()), c.registerFinalizerAddress, PREPEND_THREAD); linkForeignCall(options, providers, MONITORENTER, c.monitorenterAddress, PREPEND_THREAD); diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/replacements/HotSpotReplacementsUtil.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/replacements/HotSpotReplacementsUtil.java index a3e87c56255d..821617be5cc9 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/replacements/HotSpotReplacementsUtil.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/replacements/HotSpotReplacementsUtil.java @@ -634,6 +634,20 @@ public static int secondarySuperCacheOffset(@InjectedParameter GraalHotSpotVMCon public static final LocationIdentity SECONDARY_SUPERS_LOCATION = NamedLocationIdentity.immutable("SecondarySupers"); + public static final LocationIdentity KLASS_HASH_SLOT_LOCATION = NamedLocationIdentity.immutable("Klass::_hash_slot"); + + public static final LocationIdentity KLASS_BITMAP_LOCATION = NamedLocationIdentity.immutable("Klass::_bitmap"); + + @Fold + public static int klassHashSlotOffset(@InjectedParameter GraalHotSpotVMConfig config) { + return config.klassHashSlotOffset; + } + + @Fold + public static int klassBitmapOffset(@InjectedParameter GraalHotSpotVMConfig config) { + return config.klassBitmapOffset; + } + @Fold public static int secondarySupersOffset(@InjectedParameter GraalHotSpotVMConfig config) { return config.secondarySupersOffset; diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/replacements/InstanceOfSnippets.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/replacements/InstanceOfSnippets.java index d09b86ac29e8..f0f1bbddc2bb 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/replacements/InstanceOfSnippets.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/replacements/InstanceOfSnippets.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2012, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -24,8 +24,6 @@ */ package jdk.graal.compiler.hotspot.replacements; -import static jdk.vm.ci.meta.DeoptimizationAction.InvalidateReprofile; -import static jdk.vm.ci.meta.DeoptimizationReason.OptimizedTypeCheckViolated; import static jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil.PRIMARY_SUPERS_LOCATION; import static jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil.SECONDARY_SUPER_CACHE_LOCATION; import static jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil.loadHubIntrinsic; @@ -41,6 +39,8 @@ import static jdk.graal.compiler.nodes.extended.BranchProbabilityNode.NOT_LIKELY_PROBABILITY; import static jdk.graal.compiler.nodes.extended.BranchProbabilityNode.probability; import static jdk.graal.compiler.nodes.extended.BranchProbabilityNode.unknownProbability; +import static jdk.vm.ci.meta.DeoptimizationAction.InvalidateReprofile; +import static jdk.vm.ci.meta.DeoptimizationReason.OptimizedTypeCheckViolated; import jdk.graal.compiler.api.replacements.Snippet; import jdk.graal.compiler.api.replacements.Snippet.ConstantParameter; @@ -74,7 +74,7 @@ import jdk.graal.compiler.replacements.SnippetTemplate.SnippetInfo; import jdk.graal.compiler.replacements.Snippets; import jdk.graal.compiler.replacements.nodes.ExplodeLoopNode; - +import jdk.graal.compiler.serviceprovider.JavaVersionUtil; import jdk.vm.ci.hotspot.HotSpotResolvedObjectType; import jdk.vm.ci.meta.Assumptions; import jdk.vm.ci.meta.DeoptimizationAction; @@ -163,7 +163,7 @@ public static Object instanceofPrimary(KlassPointer hub, Object object, @Constan } /** - * A test against a restricted secondary type type. + * A test against a restricted secondary type. */ @Snippet(allowMissingProbabilities = true) public static Object instanceofSecondary(KlassPointer hub, Object object, @VarargsParameter KlassPointer[] hints, @VarargsParameter boolean[] hintIsPositive, Object trueValue, Object falseValue, @@ -237,7 +237,7 @@ public static Object isAssignableFrom(@NonNullParameter Class thisClassNonNul if (probability(FAST_PATH_PROBABILITY, !otherHub.isNull())) { GuardingNode guardNonNull = SnippetAnchorNode.anchor(); KlassPointer nonNullOtherHub = ClassGetHubNode.piCastNonNull(otherHub, guardNonNull); - if (TypeCheckSnippetUtils.checkUnknownSubType(thisHub, nonNullOtherHub, counters)) { + if (checkUnknownSubType(thisHub, nonNullOtherHub, counters)) { return trueValue; } } @@ -266,9 +266,15 @@ public Templates(OptionValues options, SnippetCounter.Group.Factory factory, Hot this.instanceofWithProfile = snippet(providers, InstanceOfSnippets.class, "instanceofWithProfile"); this.instanceofExact = snippet(providers, InstanceOfSnippets.class, "instanceofExact"); this.instanceofPrimary = snippet(providers, InstanceOfSnippets.class, "instanceofPrimary"); - this.instanceofSecondary = snippet(providers, InstanceOfSnippets.class, "instanceofSecondary", SECONDARY_SUPER_CACHE_LOCATION); - this.instanceofDynamic = snippet(providers, InstanceOfSnippets.class, "instanceofDynamic", SECONDARY_SUPER_CACHE_LOCATION); - this.isAssignableFrom = snippet(providers, InstanceOfSnippets.class, "isAssignableFrom", SECONDARY_SUPER_CACHE_LOCATION); + if (JavaVersionUtil.JAVA_SPEC == 21) { + this.instanceofSecondary = snippet(providers, InstanceOfSnippets.class, "instanceofSecondary", SECONDARY_SUPER_CACHE_LOCATION); + this.instanceofDynamic = snippet(providers, InstanceOfSnippets.class, "instanceofDynamic", SECONDARY_SUPER_CACHE_LOCATION); + this.isAssignableFrom = snippet(providers, InstanceOfSnippets.class, "isAssignableFrom", SECONDARY_SUPER_CACHE_LOCATION); + } else { + this.instanceofSecondary = snippet(providers, InstanceOfSnippets.class, "instanceofSecondary"); + this.instanceofDynamic = snippet(providers, InstanceOfSnippets.class, "instanceofDynamic"); + this.isAssignableFrom = snippet(providers, InstanceOfSnippets.class, "isAssignableFrom"); + } this.counters = new Counters(factory); } diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/replacements/TypeCheckSnippetUtils.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/replacements/TypeCheckSnippetUtils.java index dfa79f0c2b44..8cf5adfc126d 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/replacements/TypeCheckSnippetUtils.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/replacements/TypeCheckSnippetUtils.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013, 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2013, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -25,19 +25,26 @@ package jdk.graal.compiler.hotspot.replacements; import static jdk.graal.compiler.hotspot.GraalHotSpotVMConfig.INJECTED_VMCONFIG; +import static jdk.graal.compiler.hotspot.stubs.LookUpSecondarySupersTableStub.SECONDARY_SUPERS_TABLE_MASK; +import static jdk.graal.compiler.hotspot.stubs.LookUpSecondarySupersTableStub.loadSecondarySupersElement; +import static jdk.graal.compiler.nodes.extended.BranchProbabilityNode.FREQUENT_PROBABILITY; import static jdk.graal.compiler.nodes.extended.BranchProbabilityNode.NOT_LIKELY_PROBABILITY; import static jdk.graal.compiler.nodes.extended.BranchProbabilityNode.probability; import java.util.Arrays; +import jdk.graal.compiler.api.replacements.Fold; import jdk.graal.compiler.debug.Assertions; import jdk.graal.compiler.hotspot.nodes.type.KlassPointerStamp; +import jdk.graal.compiler.hotspot.stubs.LookUpSecondarySupersTableStub; import jdk.graal.compiler.hotspot.word.KlassPointer; +import jdk.graal.compiler.lir.SyncPort; import jdk.graal.compiler.nodes.ConstantNode; import jdk.graal.compiler.nodes.StructuredGraph; import jdk.graal.compiler.nodes.TypeCheckHints; import jdk.graal.compiler.replacements.SnippetCounter; import jdk.graal.compiler.replacements.SnippetCounter.Group; +import jdk.graal.compiler.serviceprovider.JavaVersionUtil; import jdk.graal.compiler.word.Word; import jdk.vm.ci.hotspot.HotSpotResolvedObjectType; import jdk.vm.ci.meta.MetaAccessProvider; @@ -49,9 +56,14 @@ */ public class TypeCheckSnippetUtils { + @Fold + static boolean isJDK21() { + return JavaVersionUtil.JAVA_SPEC == 21; + } + static boolean checkSecondarySubType(KlassPointer t, KlassPointer sNonNull, Counters counters) { // if (S.cache == T) return true - if (sNonNull.readKlassPointer(HotSpotReplacementsUtil.secondarySuperCacheOffset(INJECTED_VMCONFIG), HotSpotReplacementsUtil.SECONDARY_SUPER_CACHE_LOCATION).equal(t)) { + if (isJDK21() && sNonNull.readKlassPointer(HotSpotReplacementsUtil.secondarySuperCacheOffset(INJECTED_VMCONFIG), HotSpotReplacementsUtil.SECONDARY_SUPER_CACHE_LOCATION).equal(t)) { counters.cacheHit.inc(); return true; } @@ -83,25 +95,91 @@ static boolean checkUnknownSubType(KlassPointer t, KlassPointer sNonNull, Counte return checkSelfAndSupers(t, sNonNull, counters); } - private static boolean checkSelfAndSupers(KlassPointer t, KlassPointer s, Counters counters) { + // @formatter:off + @SyncPort(from = "https://github.com/openjdk/jdk/blob/8032d640c0d34fe507392a1d4faa4ff2005c771d/src/hotspot/cpu/x86/macroAssembler_x86.cpp#L4786-L4881", + sha1 = "c0e2fdd973dc975757d58080ba94efe628d6a380") + // @formatter:on + static boolean checkSelfAndSupers(KlassPointer t, KlassPointer s, Counters counters) { // if (T == S) return true if (s.equal(t)) { counters.equalsSecondary.inc(); return true; } - // if (S.scan_s_s_array(T)) { S.cache = T; return true; } Word secondarySupers = s.readWord(HotSpotReplacementsUtil.secondarySupersOffset(INJECTED_VMCONFIG), HotSpotReplacementsUtil.SECONDARY_SUPERS_LOCATION); - int length = secondarySupers.readInt(HotSpotReplacementsUtil.metaspaceArrayLengthOffset(INJECTED_VMCONFIG), HotSpotReplacementsUtil.METASPACE_ARRAY_LENGTH_LOCATION); - for (int i = 0; i < length; i++) { - if (probability(NOT_LIKELY_PROBABILITY, t.equal(loadSecondarySupersElement(secondarySupers, i)))) { - s.writeKlassPointer(HotSpotReplacementsUtil.secondarySuperCacheOffset(INJECTED_VMCONFIG), t, HotSpotReplacementsUtil.SECONDARY_SUPER_CACHE_LOCATION); - counters.secondariesHit.inc(); - return true; + + if (isJDK21()) { + int length = secondarySupers.readInt(HotSpotReplacementsUtil.metaspaceArrayLengthOffset(INJECTED_VMCONFIG), HotSpotReplacementsUtil.METASPACE_ARRAY_LENGTH_LOCATION); + for (int i = 0; i < length; i++) { + if (probability(NOT_LIKELY_PROBABILITY, t.equal(loadSecondarySupersElement(secondarySupers, i)))) { + s.writeKlassPointer(HotSpotReplacementsUtil.secondarySuperCacheOffset(INJECTED_VMCONFIG), t, HotSpotReplacementsUtil.SECONDARY_SUPER_CACHE_LOCATION); + counters.secondariesHit.inc(); + return true; + } } + counters.secondariesMiss.inc(); + return false; + } + + // Since JDK-8180450, HotSpot uses a hashed lookup of secondary supers. It maintains a 6-bit + // hash and a 64-bit bitmap for each class. A negative lookup is simply testing if T.hash-th + // bit is not present in the S.bitmap. A positive lookup contains a fast path that tests the + // corresponding element in secondary supers table of S is T, and a slow path that calls + // into StubRoutines::lookup_secondary_supers_table_slow_path_stub to handle collision + // cases. + // + // When populating the secondary supers table, HotSpot first uses the hash as index to a + // temporary array. In the case of collision, HotSpot stores the secondary super in the next + // empty slot, with the possibility of swapping out existing secondary super to balance the + // search distance. E.g.: + // + // @formatter:off + // table [_, _, _, A, _, _, _, B, C, _, _, _, _, _, _, _, ...] + // bitmap 0 0 0 1 0 0 0 1 1 0 0 0 0 0 0 0 ... + // LSB + // where A.hash == 3, B.hash == 7, C.hash == 7 + // The temporary secondary supers table is then compressed: + // + // table [A, B, C, _, _, _, _, _, _, _, _, _, _, _, _, _, ...] + // bitmap 0 0 0 1 0 0 0 1 1 0 0 0 0 0 0 0 ... + // @formatter:on + // + // The index to the compressed secondary supers table is calculated via + // Long.bitCount(S.bitmap << (63 - T.hash)) - 1 + + // bit will be folded when T is constant. + int bit = t.readByte(HotSpotReplacementsUtil.klassHashSlotOffset(INJECTED_VMCONFIG), HotSpotReplacementsUtil.KLASS_HASH_SLOT_LOCATION); + long bitmap = s.readLong(HotSpotReplacementsUtil.klassBitmapOffset(INJECTED_VMCONFIG), HotSpotReplacementsUtil.KLASS_BITMAP_LOCATION); + + long bitmapShifted = bitmap << (SECONDARY_SUPERS_TABLE_MASK - bit); + if (probability(NOT_LIKELY_PROBABILITY, bitmapShifted >= 0)) { + // This is equivalent to bitmap & (1L << bit) == 0 + counters.bitmapMiss.inc(); + return false; } - counters.secondariesMiss.inc(); - return false; + + // Note that Array::_data[0] is offsetted by 8 in an Array. HotSpot assumes + // Array::base_offset_in_bytes() == wordSize, and uses Long.bitCount(bitmapShifted) + // as index to the hashed secondary supers table without adjusting the array base. + // We instead rely on the compiler for constant folding. + // Use long to avoid unnecessary zero extension. + long index = Long.bitCount(bitmapShifted) - 1L; + KlassPointer hashed = loadSecondarySupersElement(secondarySupers, index); + + if (probability(FREQUENT_PROBABILITY, t.equal(hashed))) { + counters.hashHit.inc(); + return true; + } + + if (probability(NOT_LIKELY_PROBABILITY, (bitmap & (1L << ((bit + 1) & SECONDARY_SUPERS_TABLE_MASK))) == 0)) { + // Next slot is empty -- there is no collision. + counters.bitmapMiss.inc(); + return false; + } + + bitmap = Long.rotateRight(bitmap, bit); + counters.hashMissStub.inc(); + return LookUpSecondarySupersTableStub.lookupSecondarySupersTableStub(t, secondarySupers, bitmap, index + 1); } static class Counters { @@ -113,6 +191,9 @@ static class Counters { final SnippetCounter cacheHit; final SnippetCounter secondariesHit; final SnippetCounter secondariesMiss; + final SnippetCounter bitmapMiss; + final SnippetCounter hashHit; + final SnippetCounter hashMissStub; final SnippetCounter displayHit; final SnippetCounter displayMiss; final SnippetCounter equalsSecondary; @@ -127,6 +208,9 @@ static class Counters { cacheHit = new SnippetCounter(group, "cacheHit", "secondary type cache hit"); secondariesHit = new SnippetCounter(group, "secondariesHit", "secondaries scan succeeded"); secondariesMiss = new SnippetCounter(group, "secondariesMiss", "secondaries scan failed"); + bitmapMiss = new SnippetCounter(group, "bitmapMiss", "secondary type not present in bitmap"); + hashHit = new SnippetCounter(group, "hashHit", "secondary type found in hashed table"); + hashMissStub = new SnippetCounter(group, "hashMissStub", "secondary type not found in first attempt, call into runtime"); displayHit = new SnippetCounter(group, "displayHit", "primary type test succeeded"); displayMiss = new SnippetCounter(group, "displayMiss", "primary type test failed"); equalsSecondary = new SnippetCounter(group, "T_equals_S", "object type was equal to secondary type"); @@ -174,8 +258,4 @@ static Hints createHints(TypeCheckHints hints, MetaAccessProvider metaAccess, bo return new Hints(hubs, isPositive); } - static KlassPointer loadSecondarySupersElement(Word metaspaceArray, int index) { - return KlassPointer.fromWord(metaspaceArray.readWord(HotSpotReplacementsUtil.metaspaceArrayBaseOffset(INJECTED_VMCONFIG) + index * HotSpotReplacementsUtil.wordSize(), - HotSpotReplacementsUtil.SECONDARY_SUPERS_ELEMENT_LOCATION)); - } } diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/stubs/LookUpSecondarySupersTableStub.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/stubs/LookUpSecondarySupersTableStub.java new file mode 100644 index 000000000000..74a30e3b815f --- /dev/null +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/stubs/LookUpSecondarySupersTableStub.java @@ -0,0 +1,128 @@ +/* + * Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package jdk.graal.compiler.hotspot.stubs; + +import static jdk.graal.compiler.core.common.spi.ForeignCallDescriptor.CallSideEffect.NO_SIDE_EFFECT; +import static jdk.graal.compiler.hotspot.GraalHotSpotVMConfig.INJECTED_VMCONFIG; +import static jdk.graal.compiler.hotspot.meta.HotSpotForeignCallDescriptor.Transition.LEAF_NO_VZERO; +import static jdk.graal.compiler.hotspot.meta.HotSpotForeignCallsProviderImpl.NO_LOCATIONS; +import static jdk.graal.compiler.nodes.extended.BranchProbabilityNode.NOT_FREQUENT_PROBABILITY; +import static jdk.graal.compiler.nodes.extended.BranchProbabilityNode.NOT_LIKELY_PROBABILITY; +import static jdk.graal.compiler.nodes.extended.BranchProbabilityNode.probability; + +import org.graalvm.word.WordFactory; + +import jdk.graal.compiler.api.replacements.Snippet; +import jdk.graal.compiler.core.common.spi.ForeignCallDescriptor; +import jdk.graal.compiler.graph.Node.ConstantNodeParameter; +import jdk.graal.compiler.graph.Node.NodeIntrinsic; +import jdk.graal.compiler.hotspot.HotSpotForeignCallLinkage; +import jdk.graal.compiler.hotspot.meta.HotSpotForeignCallDescriptor; +import jdk.graal.compiler.hotspot.meta.HotSpotProviders; +import jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil; +import jdk.graal.compiler.hotspot.word.KlassPointer; +import jdk.graal.compiler.lir.SyncPort; +import jdk.graal.compiler.nodes.extended.ForeignCallNode; +import jdk.graal.compiler.options.OptionValues; +import jdk.graal.compiler.word.Word; + +/** + * Stub for secondary supers table lookup in the case of collision. + */ +public class LookUpSecondarySupersTableStub extends SnippetStub { + + public static final int SECONDARY_SUPERS_TABLE_SIZE = 64; + public static final int SECONDARY_SUPERS_TABLE_MASK = SECONDARY_SUPERS_TABLE_SIZE - 1; + public static final long SECONDARY_SUPERS_BITMAP_FULL = ~0L; + + public static final HotSpotForeignCallDescriptor LOOKUP_SECONDARY_SUPERS_TABLE_SLOW_PATH = new HotSpotForeignCallDescriptor(LEAF_NO_VZERO, NO_SIDE_EFFECT, NO_LOCATIONS, + "lookupSecondarySupersTableSlowPath", boolean.class, KlassPointer.class, Word.class, long.class, long.class); + + public LookUpSecondarySupersTableStub(OptionValues options, HotSpotProviders providers, HotSpotForeignCallLinkage linkage) { + super("lookupSecondarySupersTableSlowPath", options, providers, linkage); + } + + // @formatter:off + @SyncPort(from = "https://github.com/openjdk/jdk/blob/8032d640c0d34fe507392a1d4faa4ff2005c771d/src/hotspot/cpu/x86/macroAssembler_x86.cpp#L4883-L4992", + sha1 = "555f4e42531f3f1fc32bac28b2f4e3337b42374f") + // @formatter:on + @Snippet + private static boolean lookupSecondarySupersTableSlowPath(KlassPointer t, Word secondarySupers, long bitmap, long index) { + int length = secondarySupers.readInt(HotSpotReplacementsUtil.metaspaceArrayLengthOffset(INJECTED_VMCONFIG), HotSpotReplacementsUtil.METASPACE_ARRAY_LENGTH_LOCATION); + + if (probability(NOT_FREQUENT_PROBABILITY, bitmap == SECONDARY_SUPERS_BITMAP_FULL)) { + // Degenerate case: more than 64 secondary supers. + for (int i = 0; i < length; i++) { + if (probability(NOT_LIKELY_PROBABILITY, t.equal(loadSecondarySupersElement(secondarySupers, i)))) { + return true; + } + } + return false; + } + + long i = index; + long currentBitmap = bitmap; + do { + // Check for array wraparound. + if (probability(NOT_LIKELY_PROBABILITY, i >= length)) { + i = 0; + } + + if (probability(NOT_LIKELY_PROBABILITY, t.equal(loadSecondarySupersElement(secondarySupers, i)))) { + return true; + } + + // This is slightly different from HotSpot stub - we first rotate the bitmap and then + // test the next bit, to avoid while(true) loop. + currentBitmap = Long.rotateRight(currentBitmap, 1); + i++; + } while (probability(NOT_LIKELY_PROBABILITY, (currentBitmap & 0b10) != 0)); + + return false; + } + + public static KlassPointer loadSecondarySupersElement(Word metaspaceArray, long index) { + return KlassPointer.fromWord(metaspaceArray.readWord(WordFactory.signed(HotSpotReplacementsUtil.metaspaceArrayBaseOffset(INJECTED_VMCONFIG) + index * HotSpotReplacementsUtil.wordSize()), + HotSpotReplacementsUtil.SECONDARY_SUPERS_ELEMENT_LOCATION)); + } + + @NodeIntrinsic(ForeignCallNode.class) + private static native boolean lookupSecondarySupersTableStub(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer t, Word secondarySupers, long bitmap, long index); + + /** + * Slow path called when there is a collision in the hashed lookup in the secondary supers + * array. The caller must test the corresponding bit in bitmap for the {@code index}-th element. + * + * @param t type being checked + * @param secondarySupers metaspace array from the receiver class for secondary supers + * @param bitmap bitmap for the hashed secondary supers, rotated such that the second least + * significant bit points to the {@code index}-th element + * @param index index pointing to the next element in the secondary supers array in the case of + * collision + */ + public static boolean lookupSecondarySupersTableStub(KlassPointer t, Word secondarySupers, long bitmap, long index) { + return lookupSecondarySupersTableStub(LOOKUP_SECONDARY_SUPERS_TABLE_SLOW_PATH, t, secondarySupers, bitmap, index); + } +} diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/lir/amd64/AMD64BitCountOp.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/lir/amd64/AMD64BitCountOp.java new file mode 100644 index 000000000000..391ea936f6e9 --- /dev/null +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/lir/amd64/AMD64BitCountOp.java @@ -0,0 +1,129 @@ +/* + * Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package jdk.graal.compiler.lir.amd64; + +import static jdk.graal.compiler.lir.LIRInstruction.OperandFlag.ILLEGAL; +import static jdk.graal.compiler.lir.LIRInstruction.OperandFlag.REG; +import static jdk.vm.ci.code.ValueUtil.asRegister; + +import jdk.graal.compiler.asm.amd64.AMD64MacroAssembler; +import jdk.graal.compiler.lir.LIRInstructionClass; +import jdk.graal.compiler.lir.asm.CompilationResultBuilder; +import jdk.graal.compiler.lir.gen.LIRGeneratorTool; +import jdk.vm.ci.amd64.AMD64Kind; +import jdk.vm.ci.code.Register; +import jdk.vm.ci.meta.Value; + +/** + * Population count of an integer or long value. + */ +public final class AMD64BitCountOp extends AMD64LIRInstruction { + + public static final LIRInstructionClass TYPE = LIRInstructionClass.create(AMD64BitCountOp.class); + + @Def protected Value dstValue; + @Alive protected Value srcValue; + + @Temp protected Value rtmpValue; + @Temp({REG, ILLEGAL}) protected Value rtmp1Value; + + public AMD64BitCountOp(LIRGeneratorTool tool, Value dstValue, Value srcValue) { + super(TYPE); + + this.dstValue = dstValue; + this.srcValue = srcValue; + + this.rtmpValue = tool.newVariable(srcValue.getValueKind()); + + if (srcValue.getPlatformKind() == AMD64Kind.DWORD) { + this.rtmp1Value = Value.ILLEGAL; + } else { + this.rtmp1Value = tool.newVariable(srcValue.getValueKind()); + } + } + + @Override + public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { + Register dst = asRegister(dstValue); + Register src = asRegister(srcValue); + Register rtmp = asRegister(rtmpValue); + + switch ((AMD64Kind) srcValue.getPlatformKind()) { + case DWORD: + masm.movl(dst, src); + masm.movl(rtmp, src); + masm.shrl(rtmp, 1); + masm.andl(rtmp, 0x55555555); + masm.subl(dst, rtmp); + masm.movl(rtmp, dst); + masm.andl(rtmp, 0x33333333); + masm.shrl(dst, 2); + masm.andl(dst, 0x33333333); + masm.addl(rtmp, dst); + masm.movl(dst, rtmp); + masm.shrl(dst, 4); + masm.addl(dst, rtmp); + masm.andl(dst, 0xf0f0f0f); + masm.movl(rtmp, dst); + masm.shrl(rtmp, 8); + masm.addl(rtmp, dst); + masm.movl(dst, rtmp); + masm.shrl(dst, 16); + masm.addl(dst, rtmp); + masm.andl(dst, 0x3f); + break; + default: + Register longImm = asRegister(rtmp1Value); + + masm.movq(dst, src); + masm.movq(rtmp, src); + masm.shrq(dst, 1); + masm.movq(longImm, 0x5555555555555555L); + masm.andq(dst, longImm); + masm.subq(rtmp, dst); + masm.movq(dst, rtmp); + masm.movq(longImm, 0x3333333333333333L); + masm.andq(dst, longImm); + masm.shrq(rtmp, 2); + masm.andq(rtmp, longImm); + masm.addq(dst, rtmp); + masm.movq(rtmp, dst); + masm.shrq(rtmp, 4); + masm.addq(rtmp, dst); + masm.movq(longImm, 0xf0f0f0f0f0f0f0fL); + masm.andq(rtmp, longImm); + masm.movq(dst, rtmp); + masm.shrq(dst, 8); + masm.addq(dst, rtmp); + masm.movq(rtmp, dst); + masm.shrq(rtmp, 16); + masm.addq(rtmp, dst); + masm.movq(dst, rtmp); + masm.shrq(dst, 32); + masm.addq(dst, rtmp); + masm.andl(dst, 0x7f); + } + } +} diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/lir/amd64/AMD64ControlFlow.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/lir/amd64/AMD64ControlFlow.java index 161e295fa228..880b81b51b55 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/lir/amd64/AMD64ControlFlow.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/lir/amd64/AMD64ControlFlow.java @@ -324,6 +324,25 @@ public boolean makeNullCheckFor(Value value, LIRFrameState nullCheckState, int i } } + public static class BitTestAndBranchOp extends BranchOp { + public static final LIRInstructionClass TYPE = LIRInstructionClass.create(BitTestAndBranchOp.class); + + @Use protected AllocatableValue value; + private final int index; + + public BitTestAndBranchOp(LabelRef trueDestination, LabelRef falseDestination, AllocatableValue value, double trueDestinationProbability, int index) { + super(TYPE, ConditionFlag.CarryClear, trueDestination, falseDestination, trueDestinationProbability); + this.value = value; + this.index = index; + } + + @Override + public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { + masm.btq(asRegister(value), index); + super.emitCode(crb, masm); + } + } + public static class CmpBranchOp extends BranchOp implements StandardOp.ImplicitNullCheck { public static final LIRInstructionClass TYPE = LIRInstructionClass.create(CmpBranchOp.class); diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/replacements/StandardGraphBuilderPlugins.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/replacements/StandardGraphBuilderPlugins.java index 3b58a1f43d47..91a39569b109 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/replacements/StandardGraphBuilderPlugins.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/replacements/StandardGraphBuilderPlugins.java @@ -190,6 +190,7 @@ import jdk.graal.compiler.replacements.nodes.ArrayEqualsNode; import jdk.graal.compiler.replacements.nodes.BigIntegerMulAddNode; import jdk.graal.compiler.replacements.nodes.BigIntegerSquareToLenNode; +import jdk.graal.compiler.replacements.nodes.BitCountNode; import jdk.graal.compiler.replacements.nodes.CipherBlockChainingAESNode; import jdk.graal.compiler.replacements.nodes.CountPositivesNode; import jdk.graal.compiler.replacements.nodes.CounterModeAESNode; @@ -850,6 +851,13 @@ public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Rec return true; } }); + r.register(new InvocationPlugin("bitCount", type) { + @Override + public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode value) { + b.push(JavaKind.Int, b.append(new BitCountNode(value).canonical(null))); + return true; + } + }); } private static void registerCharacterPlugins(InvocationPlugins plugins) { diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/replacements/aarch64/AArch64BitCountNode.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/replacements/aarch64/AArch64BitCountNode.java deleted file mode 100644 index 04222232db88..000000000000 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/replacements/aarch64/AArch64BitCountNode.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright (c) 2019, 2020, Oracle and/or its affiliates. All rights reserved. - * Copyright (c) 2019, Arm Limited. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Oracle designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - */ -package jdk.graal.compiler.replacements.aarch64; - -import static jdk.graal.compiler.nodeinfo.NodeCycles.CYCLES_4; -import static jdk.graal.compiler.nodeinfo.NodeSize.SIZE_4; - -import jdk.graal.compiler.graph.NodeClass; -import jdk.graal.compiler.nodeinfo.NodeInfo; -import jdk.graal.compiler.nodes.ValueNode; -import jdk.graal.compiler.replacements.nodes.BitCountNode; - -@NodeInfo(cycles = CYCLES_4, size = SIZE_4) -public final class AArch64BitCountNode extends BitCountNode { - - public static final NodeClass TYPE = NodeClass.create(AArch64BitCountNode.class); - - public AArch64BitCountNode(ValueNode value) { - super(TYPE, value); - } -} diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/replacements/aarch64/AArch64GraphBuilderPlugins.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/replacements/aarch64/AArch64GraphBuilderPlugins.java index 801171ca79f2..0487a4e303fa 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/replacements/aarch64/AArch64GraphBuilderPlugins.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/replacements/aarch64/AArch64GraphBuilderPlugins.java @@ -121,13 +121,6 @@ public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Rec return true; } }); - r.register(new InvocationPlugin("bitCount", type) { - @Override - public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode value) { - b.push(JavaKind.Int, b.append(new AArch64BitCountNode(value).canonical(null))); - return true; - } - }); } private static void registerFloatPlugins(InvocationPlugins plugins, Replacements replacements) { diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/replacements/amd64/AMD64GraphBuilderPlugins.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/replacements/amd64/AMD64GraphBuilderPlugins.java index 7a02e219aa5d..278f329c7137 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/replacements/amd64/AMD64GraphBuilderPlugins.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/replacements/amd64/AMD64GraphBuilderPlugins.java @@ -76,7 +76,6 @@ import jdk.graal.compiler.replacements.nodes.ArrayCompareToNode; import jdk.graal.compiler.replacements.nodes.ArrayIndexOfNode; import jdk.graal.compiler.replacements.nodes.BinaryMathIntrinsicNode; -import jdk.graal.compiler.replacements.nodes.BitCountNode; import jdk.graal.compiler.replacements.nodes.CountLeadingZerosNode; import jdk.graal.compiler.replacements.nodes.CountTrailingZerosNode; import jdk.graal.compiler.replacements.nodes.FloatToHalfFloatNode; @@ -136,15 +135,6 @@ public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Rec return true; } }); - - r.registerConditional(arch.getFeatures().contains(CPUFeature.POPCNT), new InvocationPlugin("bitCount", type) { - @Override - public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode value) { - b.push(JavaKind.Int, b.append(new BitCountNode(value).canonical(null))); - return true; - } - }); - r.registerConditional(arch.getFeatures().contains(CPUFeature.BMI2), new InvocationPlugin("compress", type, type) { @Override public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode value, ValueNode mask) { diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/replacements/nodes/BitCountNode.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/replacements/nodes/BitCountNode.java index 765f4a5772b5..651e8be99525 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/replacements/nodes/BitCountNode.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/replacements/nodes/BitCountNode.java @@ -24,8 +24,8 @@ */ package jdk.graal.compiler.replacements.nodes; -import static jdk.graal.compiler.nodeinfo.NodeCycles.CYCLES_2; -import static jdk.graal.compiler.nodeinfo.NodeSize.SIZE_1; +import static jdk.graal.compiler.nodeinfo.NodeCycles.CYCLES_4; +import static jdk.graal.compiler.nodeinfo.NodeSize.SIZE_4; import jdk.graal.compiler.core.common.type.IntegerStamp; import jdk.graal.compiler.core.common.type.Stamp; @@ -45,7 +45,7 @@ import jdk.vm.ci.meta.JavaConstant; import jdk.vm.ci.meta.JavaKind; -@NodeInfo(cycles = CYCLES_2, size = SIZE_1) +@NodeInfo(cycles = CYCLES_4, size = SIZE_4) public class BitCountNode extends UnaryNode implements ArithmeticLIRLowerable { public static final NodeClass TYPE = NodeClass.create(BitCountNode.class);