Skip to content

Commit

Permalink
Merge branch 'master' into sha3-avx512-intrinsic
Browse files Browse the repository at this point in the history
  • Loading branch information
@ferakocz
ferakocz committed Oct 5, 2024
2 parents e48dd67 + 1c3e56c commit 1b5b71f
Show file tree
Hide file tree
Showing 515 changed files with 28,459 additions and 6,256 deletions.
7 changes: 7 additions & 0 deletions make/hotspot/gensrc/GensrcAdlc.gmk
View file
Original file line number Diff line number Diff line change
Expand Up @@ -200,6 +200,13 @@ ifeq ($(call check-jvm-feature, compiler2), true)
)))
endif

ifeq ($(call check-jvm-feature, g1gc), true)
AD_SRC_FILES += $(call uniq, $(wildcard $(foreach d, $(AD_SRC_ROOTS), \
$d/cpu/$(HOTSPOT_TARGET_CPU_ARCH)/gc/g1/g1_$(HOTSPOT_TARGET_CPU).ad \
$d/cpu/$(HOTSPOT_TARGET_CPU_ARCH)/gc/g1/g1_$(HOTSPOT_TARGET_CPU_ARCH).ad \
)))
endif

SINGLE_AD_SRCFILE := $(ADLC_SUPPORT_DIR)/all-ad-src.ad

INSERT_FILENAME_AWK_SCRIPT := \
Expand Down
102 changes: 16 additions & 86 deletions src/hotspot/cpu/aarch64/aarch64.ad
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2620,7 +2620,8 @@ static bool is_vector_bitwise_not_pattern(Node* n, Node* m) {
bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
if (is_vshift_con_pattern(n, m) ||
is_vector_bitwise_not_pattern(n, m) ||
is_valid_sve_arith_imm_pattern(n, m)) {
is_valid_sve_arith_imm_pattern(n, m) ||
is_encode_and_store_pattern(n, m)) {
mstack.push(m, Visit);
return true;
}
Expand Down Expand Up @@ -4931,60 +4932,6 @@ operand vRegD_V7()
interface(REG_INTER);
%}

operand vRegD_V12()
%{
constraint(ALLOC_IN_RC(v12_reg));
match(RegD);
op_cost(0);
format %{ %}
interface(REG_INTER);
%}

operand vRegD_V13()
%{
constraint(ALLOC_IN_RC(v13_reg));
match(RegD);
op_cost(0);
format %{ %}
interface(REG_INTER);
%}

operand vRegD_V14()
%{
constraint(ALLOC_IN_RC(v14_reg));
match(RegD);
op_cost(0);
format %{ %}
interface(REG_INTER);
%}

operand vRegD_V15()
%{
constraint(ALLOC_IN_RC(v15_reg));
match(RegD);
op_cost(0);
format %{ %}
interface(REG_INTER);
%}

operand vRegD_V16()
%{
constraint(ALLOC_IN_RC(v16_reg));
match(RegD);
op_cost(0);
format %{ %}
interface(REG_INTER);
%}

operand vRegD_V17()
%{
constraint(ALLOC_IN_RC(v17_reg));
match(RegD);
op_cost(0);
format %{ %}
interface(REG_INTER);
%}

operand pReg()
%{
constraint(ALLOC_IN_RC(pr_reg));
Expand Down Expand Up @@ -6464,7 +6411,7 @@ instruct loadP(iRegPNoSp dst, memory mem)
instruct loadN(iRegNNoSp dst, memory mem)
%{
match(Set dst (LoadN mem));
predicate(!needs_acquiring_load(n));
predicate(!needs_acquiring_load(n) && n->as_Load()->barrier_data() == 0);

ins_cost(4 * INSN_COST);
format %{ "ldrw $dst, $mem\t# compressed ptr" %}
Expand Down Expand Up @@ -6893,7 +6840,7 @@ instruct storeimmP0(immP0 zero, memory mem)
instruct storeN(iRegN src, memory mem)
%{
match(Set mem (StoreN mem src));
predicate(!needs_releasing_store(n));
predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);

ins_cost(INSN_COST);
format %{ "strw $src, $mem\t# compressed ptr" %}
Expand All @@ -6906,7 +6853,7 @@ instruct storeN(iRegN src, memory mem)
instruct storeImmN0(immN0 zero, memory mem)
%{
match(Set mem (StoreN mem zero));
predicate(!needs_releasing_store(n));
predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);

ins_cost(INSN_COST);
format %{ "strw zr, $mem\t# compressed ptr" %}
Expand Down Expand Up @@ -7140,6 +7087,7 @@ instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
%{
match(Set dst (LoadN mem));
predicate(n->as_Load()->barrier_data() == 0);

ins_cost(VOLATILE_REF_COST);
format %{ "ldarw $dst, $mem\t# compressed ptr" %}
Expand Down Expand Up @@ -7307,6 +7255,7 @@ instruct storeimmP0_volatile(immP0 zero, /* sync_memory*/indirect mem)
instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
%{
match(Set mem (StoreN mem src));
predicate(n->as_Store()->barrier_data() == 0);

ins_cost(VOLATILE_REF_COST);
format %{ "stlrw $src, $mem\t# compressed ptr" %}
Expand All @@ -7319,6 +7268,7 @@ instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
instruct storeimmN0_volatile(immN0 zero, /* sync_memory*/indirect mem)
%{
match(Set mem (StoreN mem zero));
predicate(n->as_Store()->barrier_data() == 0);

ins_cost(VOLATILE_REF_COST);
format %{ "stlrw zr, $mem\t# compressed ptr" %}
Expand Down Expand Up @@ -8115,6 +8065,7 @@ instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval
instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{

match(Set res (CompareAndSwapN mem (Binary oldval newval)));
predicate(n->as_LoadStore()->barrier_data() == 0);
ins_cost(2 * VOLATILE_REF_COST);

effect(KILL cr);
Expand Down Expand Up @@ -8229,7 +8180,7 @@ instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP new

instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{

predicate(needs_acquiring_load_exclusive(n));
predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
match(Set res (CompareAndSwapN mem (Binary oldval newval)));
ins_cost(VOLATILE_REF_COST);

Expand Down Expand Up @@ -8334,6 +8285,7 @@ instruct compareAndExchangeL(iRegLNoSp res, indirect mem, iRegL oldval, iRegL ne
// This pattern is generated automatically from cas.m4.
// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
predicate(n->as_LoadStore()->barrier_data() == 0);
match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
ins_cost(2 * VOLATILE_REF_COST);
effect(TEMP_DEF res, KILL cr);
Expand Down Expand Up @@ -8443,7 +8395,7 @@ instruct compareAndExchangeLAcq(iRegLNoSp res, indirect mem, iRegL oldval, iRegL
// This pattern is generated automatically from cas.m4.
// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
instruct compareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
predicate(needs_acquiring_load_exclusive(n));
predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
ins_cost(VOLATILE_REF_COST);
effect(TEMP_DEF res, KILL cr);
Expand Down Expand Up @@ -8555,6 +8507,7 @@ instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL ne
// This pattern is generated automatically from cas.m4.
// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
predicate(n->as_LoadStore()->barrier_data() == 0);
match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
ins_cost(2 * VOLATILE_REF_COST);
effect(KILL cr);
Expand Down Expand Up @@ -8674,7 +8627,7 @@ instruct weakCompareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL
// This pattern is generated automatically from cas.m4.
// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
instruct weakCompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
predicate(needs_acquiring_load_exclusive(n));
predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
ins_cost(VOLATILE_REF_COST);
effect(KILL cr);
Expand Down Expand Up @@ -8735,6 +8688,7 @@ instruct get_and_setL(indirect mem, iRegL newv, iRegLNoSp prev) %{
%}

instruct get_and_setN(indirect mem, iRegN newv, iRegINoSp prev) %{
predicate(n->as_LoadStore()->barrier_data() == 0);
match(Set prev (GetAndSetN mem newv));
ins_cost(2 * VOLATILE_REF_COST);
format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
Expand Down Expand Up @@ -8778,7 +8732,7 @@ instruct get_and_setLAcq(indirect mem, iRegL newv, iRegLNoSp prev) %{
%}

instruct get_and_setNAcq(indirect mem, iRegN newv, iRegINoSp prev) %{
predicate(needs_acquiring_load_exclusive(n));
predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
match(Set prev (GetAndSetN mem newv));
ins_cost(VOLATILE_REF_COST);
format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
Expand Down Expand Up @@ -16605,30 +16559,6 @@ instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
ins_pipe(pipe_class_memory);
%}

instruct arrays_hashcode(iRegP_R1 ary, iRegI_R2 cnt, iRegI_R0 result, immI basic_type,
vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
vRegD_V12 vtmp8, vRegD_V13 vtmp9, vRegD_V14 vtmp10,
vRegD_V15 vtmp11, vRegD_V16 vtmp12, vRegD_V17 vtmp13,
rFlagsReg cr)
%{
match(Set result (VectorizedHashCode (Binary ary cnt) (Binary result basic_type)));
effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5, TEMP vtmp6,
TEMP vtmp7, TEMP vtmp8, TEMP vtmp9, TEMP vtmp10, TEMP vtmp11, TEMP vtmp12, TEMP vtmp13,
USE_KILL ary, USE_KILL cnt, USE basic_type, KILL cr);

format %{ "Array HashCode array[] $ary,$cnt,$result,$basic_type -> $result // KILL all" %}
ins_encode %{
address tpc = __ arrays_hashcode($ary$$Register, $cnt$$Register, $result$$Register,
(BasicType)$basic_type$$constant);
if (tpc == nullptr) {
ciEnv::current()->record_failure("CodeCache is full");
return;
}
%}
ins_pipe(pipe_class_memory);
%}

instruct count_positives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
%{
match(Set result (CountPositives ary1 len));
Expand Down
68 changes: 1 addition & 67 deletions src/hotspot/cpu/aarch64/assembler_aarch64.hpp
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2024, Red Hat Inc. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
Expand Down Expand Up @@ -287,11 +287,6 @@ class Instruction_aarch64 {
f(r->raw_encoding(), lsb + 4, lsb);
}

//<0-15>reg: As `rf(FloatRegister)`, but only the lower 16 FloatRegisters are allowed.
void lrf(FloatRegister r, int lsb) {
f(r->raw_encoding(), lsb + 3, lsb);
}

void prf(PRegister r, int lsb) {
f(r->raw_encoding(), lsb + 3, lsb);
}
Expand Down Expand Up @@ -770,7 +765,6 @@ class Assembler : public AbstractAssembler {
#define f current_insn.f
#define sf current_insn.sf
#define rf current_insn.rf
#define lrf current_insn.lrf
#define srf current_insn.srf
#define zrf current_insn.zrf
#define prf current_insn.prf
Expand Down Expand Up @@ -1596,16 +1590,6 @@ class Assembler : public AbstractAssembler {

#undef INSN

// Load/store a register, but with a BasicType parameter. Loaded signed integer values are
// extended to 64 bits.
void load(Register Rt, const Address &adr, BasicType bt) {
int op = (is_signed_subword_type(bt) || bt == T_INT) ? 0b10 : 0b01;
ld_st2(Rt, adr, exact_log2(type2aelembytes(bt)), op);
}
void store(Register Rt, const Address &adr, BasicType bt) {
ld_st2(Rt, adr, exact_log2(type2aelembytes(bt)), 0b00);
}

/* SIMD extensions
*
* We just use FloatRegister in the following. They are exactly the same
Expand Down Expand Up @@ -2603,7 +2587,6 @@ template<typename R, typename... Rx>
INSN(addpv, 0, 0b101111, true); // accepted arrangements: T8B, T16B, T4H, T8H, T2S, T4S, T2D
INSN(smullv, 0, 0b110000, false); // accepted arrangements: T8B, T16B, T4H, T8H, T2S, T4S
INSN(umullv, 1, 0b110000, false); // accepted arrangements: T8B, T16B, T4H, T8H, T2S, T4S
INSN(smlalv, 0, 0b100000, false); // accepted arrangements: T8B, T16B, T4H, T8H, T2S, T4S
INSN(umlalv, 1, 0b100000, false); // accepted arrangements: T8B, T16B, T4H, T8H, T2S, T4S
INSN(maxv, 0, 0b011001, false); // accepted arrangements: T8B, T16B, T4H, T8H, T2S, T4S
INSN(minv, 0, 0b011011, false); // accepted arrangements: T8B, T16B, T4H, T8H, T2S, T4S
Expand Down Expand Up @@ -2877,28 +2860,6 @@ template<typename R, typename... Rx>
// FMULX - Vector - Scalar
INSN(fmulxvs, 1, 0b1001);

#undef INSN

#define INSN(NAME, op1, op2) \
void NAME(FloatRegister Vd, SIMD_Arrangement T, FloatRegister Vn, FloatRegister Vm, int index) { \
starti; \
assert(T == T4H || T == T8H || T == T2S || T == T4S, "invalid arrangement"); \
assert(index >= 0 && \
((T == T2S && index <= 1) || (T != T2S && index <= 3) || (T == T8H && index <= 7)), \
"invalid index"); \
assert((T != T4H && T != T8H) || Vm->encoding() < 16, "invalid source SIMD&FP register"); \
f(0, 31), f((int)T & 1, 30), f(op1, 29), f(0b01111, 28, 24); \
if (T == T4H || T == T8H) { \
f(0b01, 23, 22), f(index & 0b11, 21, 20), lrf(Vm, 16), f(index >> 2 & 1, 11); \
} else { \
f(0b10, 23, 22), f(index & 1, 21), rf(Vm, 16), f(index >> 1, 11); \
} \
f(op2, 15, 12), f(0, 10), rf(Vn, 5), rf(Vd, 0); \
}

// MUL - Vector - Scalar
INSN(mulvs, 0, 0b1000);

#undef INSN

// Floating-point Reciprocal Estimate
Expand Down Expand Up @@ -3062,33 +3023,6 @@ template<typename R, typename... Rx>
umov(Xd, Vn, T, index);
}

protected:
void _xaddwv(bool is_unsigned, FloatRegister Vd, FloatRegister Vn, SIMD_Arrangement Ta,
FloatRegister Vm, SIMD_Arrangement Tb) {
starti;
assert((Tb >> 1) + 1 == (Ta >> 1), "Incompatible arrangement");
f(0, 31), f((int)Tb & 1, 30), f(is_unsigned ? 1 : 0, 29), f(0b01110, 28, 24);
f((int)(Ta >> 1) - 1, 23, 22), f(1, 21), rf(Vm, 16), f(0b000100, 15, 10), rf(Vn, 5), rf(Vd, 0);
}

public:
#define INSN(NAME, assertion, is_unsigned) \
void NAME(FloatRegister Vd, FloatRegister Vn, SIMD_Arrangement Ta, FloatRegister Vm, \
SIMD_Arrangement Tb) { \
assert((assertion), "invalid arrangement"); \
_xaddwv(is_unsigned, Vd, Vn, Ta, Vm, Tb); \
}

public:

INSN(uaddwv, Tb == T8B || Tb == T4H || Tb == T2S, /*is_unsigned*/true)
INSN(uaddwv2, Tb == T16B || Tb == T8H || Tb == T4S, /*is_unsigned*/true)
INSN(saddwv, Tb == T8B || Tb == T4H || Tb == T2S, /*is_unsigned*/false)
INSN(saddwv2, Tb == T16B || Tb == T8H || Tb == T4S, /*is_unsigned*/false)

#undef INSN


private:
void _pmull(FloatRegister Vd, SIMD_Arrangement Ta, FloatRegister Vn, FloatRegister Vm, SIMD_Arrangement Tb) {
starti;
Expand Down
27 changes: 0 additions & 27 deletions src/hotspot/cpu/aarch64/c2_CodeStubs_aarch64.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,31 +64,4 @@ void C2EntryBarrierStub::emit(C2_MacroAssembler& masm) {
__ emit_int32(0); // nmethod guard value
}

int C2HandleAnonOMOwnerStub::max_size() const {
// Max size of stub has been determined by testing with 0, in which case
// C2CodeStubList::emit() will throw an assertion and report the actual size that
// is needed.
return 24;
}

void C2HandleAnonOMOwnerStub::emit(C2_MacroAssembler& masm) {
__ bind(entry());
Register mon = monitor();
Register t = tmp();
assert(t != noreg, "need tmp register");

// Fix owner to be the current thread.
__ str(rthread, Address(mon, ObjectMonitor::owner_offset()));

// Pop owner object from lock-stack.
__ ldrw(t, Address(rthread, JavaThread::lock_stack_top_offset()));
__ subw(t, t, oopSize);
#ifdef ASSERT
__ str(zr, Address(rthread, t));
#endif
__ strw(t, Address(rthread, JavaThread::lock_stack_top_offset()));

__ b(continuation());
}

#undef __
Loading

0 comments on commit 1b5b71f

Please sign in to comment.