diff --git a/src/hotspot/cpu/aarch64/aarch64.ad b/src/hotspot/cpu/aarch64/aarch64.ad index a1fe9da8f61..3780f316300 100644 --- a/src/hotspot/cpu/aarch64/aarch64.ad +++ b/src/hotspot/cpu/aarch64/aarch64.ad @@ -14216,7 +14216,7 @@ instruct clearArray_reg_reg_immL0(iRegL_R11 cnt, iRegP_R10 base, immL0 zero, Uni instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, iRegL val, Universe dummy, rFlagsReg cr) %{ - predicate(((ClearArrayNode*)n)->word_copy_only()); + predicate(((ClearArrayNode*)n)->requires_word_fill()); match(Set dummy (ClearArray (Binary cnt base) val)); effect(USE_KILL cnt, USE_KILL base, KILL cr); @@ -14234,7 +14234,7 @@ instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, immL0 zero %{ predicate((uint64_t)n->in(2)->in(1)->get_long() < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord) - && !((ClearArrayNode*)n)->word_copy_only()); + && !((ClearArrayNode*)n)->requires_word_fill()); match(Set dummy (ClearArray (Binary cnt base) zero)); effect(TEMP temp, USE_KILL base, KILL cr); diff --git a/src/hotspot/cpu/ppc/ppc.ad b/src/hotspot/cpu/ppc/ppc.ad index 15c04013a74..38ad3f7ed50 100644 --- a/src/hotspot/cpu/ppc/ppc.ad +++ b/src/hotspot/cpu/ppc/ppc.ad @@ -11186,8 +11186,8 @@ instruct inlineCallClearArray(rarg1RegL cnt, rarg2RegP base, immL_0 zero, Univer %} // Clear-array with dynamic array length and non-zero value. -instruct inlineCallClearArrayWordCopy(rarg1RegL cnt, rarg2RegP base, iRegLdst val, Universe dummy, regCTR ctr) %{ - predicate(((ClearArrayNode*)n)->word_copy_only()); +instruct inlineCallClearArrayWordFill(rarg1RegL cnt, rarg2RegP base, iRegLdst val, Universe dummy, regCTR ctr) %{ + predicate(((ClearArrayNode*)n)->requires_word_fill()); match(Set dummy (ClearArray (Binary cnt base) val)); effect(USE_KILL base, KILL ctr); ins_cost(8 * MEMORY_REF_COST); diff --git a/src/hotspot/cpu/riscv/riscv.ad b/src/hotspot/cpu/riscv/riscv.ad index e5e75036ffd..f86b33c922c 100644 --- a/src/hotspot/cpu/riscv/riscv.ad +++ b/src/hotspot/cpu/riscv/riscv.ad @@ -11277,7 +11277,7 @@ instruct clearArray_reg_reg(iRegL_R29 cnt, iRegP_R28 base, iRegL val, Universe dummy) %{ // temp registers must match the one used in StubGenerator::generate_zero_blocks() - predicate(((ClearArrayNode*)n)->word_copy_only()); + predicate(((ClearArrayNode*)n)->requires_word_fill()); match(Set dummy (ClearArray (Binary cnt base) val)); effect(USE_KILL cnt, USE_KILL base, TEMP tmp1, TEMP tmp2, KILL cr); @@ -11296,7 +11296,7 @@ instruct clearArray_imm_reg(immL cnt, iRegP_R28 base, immL0 zero, Universe dummy predicate(!UseRVV && (uint64_t)n->in(2)->in(1)->get_long() < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord) - && !((ClearArrayNode*)n)->word_copy_only()); + && !((ClearArrayNode*)n)->requires_word_fill()); match(Set dummy (ClearArray (Binary cnt base) zero)); effect(USE_KILL base, KILL cr); diff --git a/src/hotspot/cpu/x86/macroAssembler_x86.cpp b/src/hotspot/cpu/x86/macroAssembler_x86.cpp index 8046537da19..fa55d29cd15 100644 --- a/src/hotspot/cpu/x86/macroAssembler_x86.cpp +++ b/src/hotspot/cpu/x86/macroAssembler_x86.cpp @@ -6506,11 +6506,12 @@ void MacroAssembler::remove_frame(int initial_framesize, bool needs_stack_repair #ifdef COMPILER2 -// clear memory of size 'cnt' qwords, starting at 'base' using XMM/YMM/ZMM registers -void MacroAssembler::xmm_clear_mem(Register base, Register cnt, Register val, XMMRegister xtmp, KRegister mask) { +// Fill memory with 'val', for 'cnt' qwords starting at 'base', using XMM/YMM/ZMM registers. +void MacroAssembler::xmm_fill_mem(Register base, Register cnt, Register val, XMMRegister xtmp, KRegister mask) { // cnt - number of qwords (8-byte words). // base - start address, qword aligned. - Label L_zero_64_bytes, L_loop, L_sloop, L_tail, L_end; + // val - qword pattern to fill. + Label L_fill_64_bytes, L_loop, L_sloop, L_tail, L_end; bool use64byteVector = (MaxVectorSize == 64) && (CopyAVX3Threshold == 0); if (use64byteVector) { evpbroadcastq(xtmp, val, AVX_512bit); @@ -6522,7 +6523,7 @@ void MacroAssembler::xmm_clear_mem(Register base, Register cnt, Register val, XM movdq(xtmp, val); punpcklqdq(xtmp, xtmp); } - jmp(L_zero_64_bytes); + jmp(L_fill_64_bytes); BIND(L_loop); if (MaxVectorSize >= 32) { @@ -6535,11 +6536,11 @@ void MacroAssembler::xmm_clear_mem(Register base, Register cnt, Register val, XM } addptr(base, 64); - BIND(L_zero_64_bytes); + BIND(L_fill_64_bytes); subptr(cnt, 8); jccb(Assembler::greaterEqual, L_loop); - // Copy trailing 64 bytes + // Fill trailing 64 bytes. if (use64byteVector) { addptr(cnt, 8); jccb(Assembler::equal, L_end); @@ -6665,10 +6666,12 @@ void MacroAssembler::clear_mem(Register base, int cnt, Register rtmp, XMMRegiste } void MacroAssembler::clear_mem(Register base, Register cnt, Register val, XMMRegister xtmp, - bool is_large, bool word_copy_only, KRegister mask) { + bool is_large, bool requires_word_fill, KRegister mask) { // cnt - number of qwords (8-byte words). // base - start address, qword aligned. // is_large - if optimizers know cnt is larger than InitArrayShortSize + // requires_word_fill - if true, val contains the qword pattern to fill; if + // false, val is scratch and this method creates zero assert(base==rdi, "base register must be edi for rep stos"); assert(val==rax, "val register must be eax for rep stos"); assert(cnt==rcx, "cnt register must be ecx for rep stos"); @@ -6677,6 +6680,10 @@ void MacroAssembler::clear_mem(Register base, Register cnt, Register val, XMMReg Label DONE; + if (!requires_word_fill) { + xorptr(val, val); + } + if (!is_large) { Label LOOP, LONG; cmpptr(cnt, InitArrayShortSize/BytesPerLong); @@ -6695,12 +6702,13 @@ void MacroAssembler::clear_mem(Register base, Register cnt, Register val, XMMReg BIND(LONG); } - // Use longer rep-prefixed ops for non-small counts: - if (UseFastStosb && !word_copy_only) { + // Use longer rep-prefixed ops for non-small counts. rep stosb is valid only + // for zeroing; an arbitrary qword pattern must be copied in full. + if (UseFastStosb && !requires_word_fill) { shlptr(cnt, 3); // convert to number of bytes rep_stosb(); } else if (UseXMMForObjInit) { - xmm_clear_mem(base, cnt, val, xtmp, mask); + xmm_fill_mem(base, cnt, val, xtmp, mask); } else { rep_stos(); } diff --git a/src/hotspot/cpu/x86/macroAssembler_x86.hpp b/src/hotspot/cpu/x86/macroAssembler_x86.hpp index 35714d85e47..7f424966468 100644 --- a/src/hotspot/cpu/x86/macroAssembler_x86.hpp +++ b/src/hotspot/cpu/x86/macroAssembler_x86.hpp @@ -1970,15 +1970,16 @@ class MacroAssembler: public Assembler { // Inline type specific methods #include "asm/macroAssembler_common.hpp" - // clear memory of size 'cnt' qwords, starting at 'base'; - // if 'is_large' is set, do not try to produce short loop - void clear_mem(Register base, Register cnt, Register val, XMMRegister xtmp, bool is_large, bool word_copy_only, KRegister mask=knoreg); + // Clear or fill 'cnt' qwords starting at 'base'. If 'requires_word_fill' is + // set, use 'val' as the fill value; otherwise, create zero in 'val'. If + // 'is_large' is set, do not try to produce a short loop. + void clear_mem(Register base, Register cnt, Register val, XMMRegister xtmp, bool is_large, bool requires_word_fill, KRegister mask=knoreg); // clear memory initialization sequence for constant size; void clear_mem(Register base, int cnt, Register rtmp, XMMRegister xtmp, KRegister mask=knoreg); - // clear memory of size 'cnt' qwords, starting at 'base' using XMM/YMM registers - void xmm_clear_mem(Register base, Register cnt, Register rtmp, XMMRegister xtmp, KRegister mask=knoreg); + // Fill memory with 'val', for 'cnt' qwords starting at 'base', using XMM/YMM/ZMM registers. + void xmm_fill_mem(Register base, Register cnt, Register val, XMMRegister xtmp, KRegister mask=knoreg); // Fill primitive arrays void generate_fill(BasicType t, bool aligned, diff --git a/src/hotspot/cpu/x86/x86.ad b/src/hotspot/cpu/x86/x86.ad index 5e222a18bc8..a38f5e30ae9 100644 --- a/src/hotspot/cpu/x86/x86.ad +++ b/src/hotspot/cpu/x86/x86.ad @@ -14779,16 +14779,18 @@ instruct MoveL2D_reg_reg(regD dst, rRegL src) %{ %} -// Fast clearing of an array -// Small non-constant lenght ClearArray for non-AVX512 targets. -instruct rep_stos(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegL val, +// Small zero fill for non-AVX512 targets. +instruct rep_stos(rcx_RegL cnt, rdi_RegP base, regD tmp, immL0 zero, rax_RegL val, Universe dummy, rFlagsReg cr) %{ - predicate(!((ClearArrayNode*)n)->is_large() && !((ClearArrayNode*)n)->word_copy_only() && (UseAVX <= 2)); - match(Set dummy (ClearArray (Binary cnt base) val)); - effect(USE_KILL cnt, USE_KILL base, TEMP tmp, USE_KILL val, KILL cr); + predicate((UseAVX <= 2) && + !((ClearArrayNode*)n)->is_large() && + ((ClearArrayNode*)n)->is_zero_fill()); + match(Set dummy (ClearArray (Binary cnt base) zero)); + effect(USE_KILL cnt, USE_KILL base, TEMP tmp, KILL val, KILL cr); format %{ $$template + $$emit$$"xorq rax, rax\t# ClearArray:\n\t" $$emit$$"cmp InitArrayShortSize,rcx\n\t" $$emit$$"jg LARGE\n\t" $$emit$$"dec rcx\n\t" @@ -14807,24 +14809,24 @@ instruct rep_stos(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegL val, $$emit$$"vinserti128_high $tmp, $tmp\n\t" $$emit$$"jmpq L_zero_64_bytes\n\t" $$emit$$"# L_loop:\t# 64-byte LOOP\n\t" - $$emit$$"vmovdqu $tmp,(rax)\n\t" - $$emit$$"vmovdqu $tmp,0x20(rax)\n\t" - $$emit$$"add 0x40,rax\n\t" + $$emit$$"vmovdqu $tmp,($base)\n\t" + $$emit$$"vmovdqu $tmp,0x20($base)\n\t" + $$emit$$"add 0x40,$base\n\t" $$emit$$"# L_zero_64_bytes:\n\t" $$emit$$"sub 0x8,rcx\n\t" $$emit$$"jge L_loop\n\t" $$emit$$"add 0x4,rcx\n\t" $$emit$$"jl L_tail\n\t" - $$emit$$"vmovdqu $tmp,(rax)\n\t" - $$emit$$"add 0x20,rax\n\t" + $$emit$$"vmovdqu $tmp,($base)\n\t" + $$emit$$"add 0x20,$base\n\t" $$emit$$"sub 0x4,rcx\n\t" $$emit$$"# L_tail:\t# Clearing tail bytes\n\t" $$emit$$"add 0x4,rcx\n\t" $$emit$$"jle L_end\n\t" $$emit$$"dec rcx\n\t" $$emit$$"# L_sloop:\t# 8-byte short loop\n\t" - $$emit$$"vmovq xmm0,(rax)\n\t" - $$emit$$"add 0x8,rax\n\t" + $$emit$$"vmovq $tmp,($base)\n\t" + $$emit$$"add 0x8,$base\n\t" $$emit$$"dec rcx\n\t" $$emit$$"jge L_sloop\n\t" $$emit$$"# L_end:\n\t" @@ -14840,10 +14842,13 @@ instruct rep_stos(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegL val, ins_pipe(pipe_slow); %} -instruct rep_stos_word_copy(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegL val, +// Small word fill for non-AVX512 targets. +instruct rep_stos_word_fill(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegL val, Universe dummy, rFlagsReg cr) %{ - predicate(!((ClearArrayNode*)n)->is_large() && ((ClearArrayNode*)n)->word_copy_only() && (UseAVX <= 2)); + predicate((UseAVX <= 2) && + !((ClearArrayNode*)n)->is_large() && + ((ClearArrayNode*)n)->requires_word_fill()); match(Set dummy (ClearArray (Binary cnt base) val)); effect(USE_KILL cnt, USE_KILL base, TEMP tmp, USE_KILL val, KILL cr); @@ -14861,26 +14866,26 @@ instruct rep_stos_word_copy(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegL val, $$emit$$"movdq $tmp, $val\n\t" $$emit$$"punpcklqdq $tmp, $tmp\n\t" $$emit$$"vinserti128_high $tmp, $tmp\n\t" - $$emit$$"jmpq L_zero_64_bytes\n\t" + $$emit$$"jmpq L_fill_64_bytes\n\t" $$emit$$"# L_loop:\t# 64-byte LOOP\n\t" - $$emit$$"vmovdqu $tmp,(rax)\n\t" - $$emit$$"vmovdqu $tmp,0x20(rax)\n\t" - $$emit$$"add 0x40,rax\n\t" - $$emit$$"# L_zero_64_bytes:\n\t" + $$emit$$"vmovdqu $tmp,($base)\n\t" + $$emit$$"vmovdqu $tmp,0x20($base)\n\t" + $$emit$$"add 0x40,$base\n\t" + $$emit$$"# L_fill_64_bytes:\n\t" $$emit$$"sub 0x8,rcx\n\t" $$emit$$"jge L_loop\n\t" $$emit$$"add 0x4,rcx\n\t" $$emit$$"jl L_tail\n\t" - $$emit$$"vmovdqu $tmp,(rax)\n\t" - $$emit$$"add 0x20,rax\n\t" + $$emit$$"vmovdqu $tmp,($base)\n\t" + $$emit$$"add 0x20,$base\n\t" $$emit$$"sub 0x4,rcx\n\t" - $$emit$$"# L_tail:\t# Clearing tail bytes\n\t" + $$emit$$"# L_tail:\t# Filling tail bytes\n\t" $$emit$$"add 0x4,rcx\n\t" $$emit$$"jle L_end\n\t" $$emit$$"dec rcx\n\t" $$emit$$"# L_sloop:\t# 8-byte short loop\n\t" - $$emit$$"vmovq xmm0,(rax)\n\t" - $$emit$$"add 0x8,rax\n\t" + $$emit$$"vmovq $tmp,($base)\n\t" + $$emit$$"add 0x8,$base\n\t" $$emit$$"dec rcx\n\t" $$emit$$"jge L_sloop\n\t" $$emit$$"# L_end:\n\t" @@ -14896,14 +14901,16 @@ instruct rep_stos_word_copy(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegL val, ins_pipe(pipe_slow); %} -// Small non-constant length ClearArray for AVX512 targets. -instruct rep_stos_evex(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegL val, +// Small zero fill for AVX512 targets. +instruct rep_stos_evex(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, immL0 zero, rax_RegL val, Universe dummy, rFlagsReg cr) %{ - predicate(!((ClearArrayNode*)n)->is_large() && !((ClearArrayNode*)n)->word_copy_only() && (UseAVX > 2)); - match(Set dummy (ClearArray (Binary cnt base) val)); + predicate((UseAVX > 2) && + !((ClearArrayNode*)n)->is_large() && + ((ClearArrayNode*)n)->is_zero_fill()); + match(Set dummy (ClearArray (Binary cnt base) zero)); ins_cost(125); - effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, USE_KILL val, KILL cr); + effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, KILL val, KILL cr); format %{ $$template $$emit$$"xorq rax, rax\t# ClearArray:\n\t" @@ -14920,28 +14927,29 @@ instruct rep_stos_evex(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_ $$emit$$"shlq rcx,3\t# Convert doublewords to bytes\n\t" $$emit$$"rep stosb\t# Store rax to *rdi++ while rcx--\n\t" } else if (UseXMMForObjInit) { - $$emit$$"mov rdi,rax\n\t" - $$emit$$"vpxor ymm0,ymm0,ymm0\n\t" + $$emit$$"movdq $tmp, $val\n\t" + $$emit$$"punpcklqdq $tmp, $tmp\n\t" + $$emit$$"vinserti128_high $tmp, $tmp\n\t" $$emit$$"jmpq L_zero_64_bytes\n\t" $$emit$$"# L_loop:\t# 64-byte LOOP\n\t" - $$emit$$"vmovdqu ymm0,(rax)\n\t" - $$emit$$"vmovdqu ymm0,0x20(rax)\n\t" - $$emit$$"add 0x40,rax\n\t" + $$emit$$"vmovdqu $tmp,($base)\n\t" + $$emit$$"vmovdqu $tmp,0x20($base)\n\t" + $$emit$$"add 0x40,$base\n\t" $$emit$$"# L_zero_64_bytes:\n\t" $$emit$$"sub 0x8,rcx\n\t" $$emit$$"jge L_loop\n\t" $$emit$$"add 0x4,rcx\n\t" $$emit$$"jl L_tail\n\t" - $$emit$$"vmovdqu ymm0,(rax)\n\t" - $$emit$$"add 0x20,rax\n\t" + $$emit$$"vmovdqu $tmp,($base)\n\t" + $$emit$$"add 0x20,$base\n\t" $$emit$$"sub 0x4,rcx\n\t" $$emit$$"# L_tail:\t# Clearing tail bytes\n\t" $$emit$$"add 0x4,rcx\n\t" $$emit$$"jle L_end\n\t" $$emit$$"dec rcx\n\t" $$emit$$"# L_sloop:\t# 8-byte short loop\n\t" - $$emit$$"vmovq xmm0,(rax)\n\t" - $$emit$$"add 0x8,rax\n\t" + $$emit$$"vmovq $tmp,($base)\n\t" + $$emit$$"add 0x8,$base\n\t" $$emit$$"dec rcx\n\t" $$emit$$"jge L_sloop\n\t" $$emit$$"# L_end:\n\t" @@ -14957,16 +14965,18 @@ instruct rep_stos_evex(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_ ins_pipe(pipe_slow); %} -instruct rep_stos_evex_word_copy(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegL val, +// Small word fill for AVX512 targets. +instruct rep_stos_evex_word_fill(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegL val, Universe dummy, rFlagsReg cr) %{ - predicate(!((ClearArrayNode*)n)->is_large() && ((ClearArrayNode*)n)->word_copy_only() && (UseAVX > 2)); + predicate((UseAVX > 2) && + !((ClearArrayNode*)n)->is_large() && + ((ClearArrayNode*)n)->requires_word_fill()); match(Set dummy (ClearArray (Binary cnt base) val)); ins_cost(125); effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, USE_KILL val, KILL cr); format %{ $$template - $$emit$$"xorq rax, rax\t# ClearArray:\n\t" $$emit$$"cmp InitArrayShortSize,rcx\n\t" $$emit$$"jg LARGE\n\t" $$emit$$"dec rcx\n\t" @@ -14976,32 +14986,30 @@ instruct rep_stos_evex_word_copy(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg $$emit$$"jge LOOP\n\t" $$emit$$"jmp DONE\n\t" $$emit$$"# LARGE:\n\t" - if (UseFastStosb) { - $$emit$$"shlq rcx,3\t# Convert doublewords to bytes\n\t" - $$emit$$"rep stosb\t# Store rax to *rdi++ while rcx--\n\t" - } else if (UseXMMForObjInit) { - $$emit$$"mov rdi,rax\n\t" - $$emit$$"vpxor ymm0,ymm0,ymm0\n\t" - $$emit$$"jmpq L_zero_64_bytes\n\t" + if (UseXMMForObjInit) { + $$emit$$"movdq $tmp, $val\n\t" + $$emit$$"punpcklqdq $tmp, $tmp\n\t" + $$emit$$"vinserti128_high $tmp, $tmp\n\t" + $$emit$$"jmpq L_fill_64_bytes\n\t" $$emit$$"# L_loop:\t# 64-byte LOOP\n\t" - $$emit$$"vmovdqu ymm0,(rax)\n\t" - $$emit$$"vmovdqu ymm0,0x20(rax)\n\t" - $$emit$$"add 0x40,rax\n\t" - $$emit$$"# L_zero_64_bytes:\n\t" + $$emit$$"vmovdqu $tmp,($base)\n\t" + $$emit$$"vmovdqu $tmp,0x20($base)\n\t" + $$emit$$"add 0x40,$base\n\t" + $$emit$$"# L_fill_64_bytes:\n\t" $$emit$$"sub 0x8,rcx\n\t" $$emit$$"jge L_loop\n\t" $$emit$$"add 0x4,rcx\n\t" $$emit$$"jl L_tail\n\t" - $$emit$$"vmovdqu ymm0,(rax)\n\t" - $$emit$$"add 0x20,rax\n\t" + $$emit$$"vmovdqu $tmp,($base)\n\t" + $$emit$$"add 0x20,$base\n\t" $$emit$$"sub 0x4,rcx\n\t" - $$emit$$"# L_tail:\t# Clearing tail bytes\n\t" + $$emit$$"# L_tail:\t# Filling tail bytes\n\t" $$emit$$"add 0x4,rcx\n\t" $$emit$$"jle L_end\n\t" $$emit$$"dec rcx\n\t" $$emit$$"# L_sloop:\t# 8-byte short loop\n\t" - $$emit$$"vmovq xmm0,(rax)\n\t" - $$emit$$"add 0x8,rax\n\t" + $$emit$$"vmovq $tmp,($base)\n\t" + $$emit$$"add 0x8,$base\n\t" $$emit$$"dec rcx\n\t" $$emit$$"jge L_sloop\n\t" $$emit$$"# L_end:\n\t" @@ -15017,15 +15025,18 @@ instruct rep_stos_evex_word_copy(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ins_pipe(pipe_slow); %} -// Large non-constant length ClearArray for non-AVX512 targets. -instruct rep_stos_large(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegL val, +// Large zero fill for non-AVX512 targets. +instruct rep_stos_large(rcx_RegL cnt, rdi_RegP base, regD tmp, immL0 zero, rax_RegL val, Universe dummy, rFlagsReg cr) %{ - predicate(((ClearArrayNode*)n)->is_large() && !((ClearArrayNode*)n)->word_copy_only() && (UseAVX <= 2)); - match(Set dummy (ClearArray (Binary cnt base) val)); - effect(USE_KILL cnt, USE_KILL base, TEMP tmp, USE_KILL val, KILL cr); + predicate((UseAVX <= 2) && + ((ClearArrayNode*)n)->is_large() && + ((ClearArrayNode*)n)->is_zero_fill()); + match(Set dummy (ClearArray (Binary cnt base) zero)); + effect(USE_KILL cnt, USE_KILL base, TEMP tmp, KILL val, KILL cr); format %{ $$template + $$emit$$"xorq rax, rax\t# ClearArray:\n\t" if (UseFastStosb) { $$emit$$"shlq rcx,3\t# Convert doublewords to bytes\n\t" $$emit$$"rep stosb\t# Store rax to *rdi++ while rcx--" @@ -15035,24 +15046,24 @@ instruct rep_stos_large(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegL val, $$emit$$"vinserti128_high $tmp, $tmp\n\t" $$emit$$"jmpq L_zero_64_bytes\n\t" $$emit$$"# L_loop:\t# 64-byte LOOP\n\t" - $$emit$$"vmovdqu $tmp,(rax)\n\t" - $$emit$$"vmovdqu $tmp,0x20(rax)\n\t" - $$emit$$"add 0x40,rax\n\t" + $$emit$$"vmovdqu $tmp,($base)\n\t" + $$emit$$"vmovdqu $tmp,0x20($base)\n\t" + $$emit$$"add 0x40,$base\n\t" $$emit$$"# L_zero_64_bytes:\n\t" $$emit$$"sub 0x8,rcx\n\t" $$emit$$"jge L_loop\n\t" $$emit$$"add 0x4,rcx\n\t" $$emit$$"jl L_tail\n\t" - $$emit$$"vmovdqu $tmp,(rax)\n\t" - $$emit$$"add 0x20,rax\n\t" + $$emit$$"vmovdqu $tmp,($base)\n\t" + $$emit$$"add 0x20,$base\n\t" $$emit$$"sub 0x4,rcx\n\t" $$emit$$"# L_tail:\t# Clearing tail bytes\n\t" $$emit$$"add 0x4,rcx\n\t" $$emit$$"jle L_end\n\t" $$emit$$"dec rcx\n\t" $$emit$$"# L_sloop:\t# 8-byte short loop\n\t" - $$emit$$"vmovq xmm0,(rax)\n\t" - $$emit$$"add 0x8,rax\n\t" + $$emit$$"vmovq $tmp,($base)\n\t" + $$emit$$"add 0x8,$base\n\t" $$emit$$"dec rcx\n\t" $$emit$$"jge L_sloop\n\t" $$emit$$"# L_end:\n\t" @@ -15067,10 +15078,13 @@ instruct rep_stos_large(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegL val, ins_pipe(pipe_slow); %} -instruct rep_stos_large_word_copy(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegL val, +// Large word fill for non-AVX512 targets. +instruct rep_stos_large_word_fill(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegL val, Universe dummy, rFlagsReg cr) %{ - predicate(((ClearArrayNode*)n)->is_large() && ((ClearArrayNode*)n)->word_copy_only() && (UseAVX <= 2)); + predicate((UseAVX <= 2) && + ((ClearArrayNode*)n)->is_large() && + ((ClearArrayNode*)n)->requires_word_fill()); match(Set dummy (ClearArray (Binary cnt base) val)); effect(USE_KILL cnt, USE_KILL base, TEMP tmp, USE_KILL val, KILL cr); @@ -15079,26 +15093,26 @@ instruct rep_stos_large_word_copy(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_Reg $$emit$$"movdq $tmp, $val\n\t" $$emit$$"punpcklqdq $tmp, $tmp\n\t" $$emit$$"vinserti128_high $tmp, $tmp\n\t" - $$emit$$"jmpq L_zero_64_bytes\n\t" + $$emit$$"jmpq L_fill_64_bytes\n\t" $$emit$$"# L_loop:\t# 64-byte LOOP\n\t" - $$emit$$"vmovdqu $tmp,(rax)\n\t" - $$emit$$"vmovdqu $tmp,0x20(rax)\n\t" - $$emit$$"add 0x40,rax\n\t" - $$emit$$"# L_zero_64_bytes:\n\t" + $$emit$$"vmovdqu $tmp,($base)\n\t" + $$emit$$"vmovdqu $tmp,0x20($base)\n\t" + $$emit$$"add 0x40,$base\n\t" + $$emit$$"# L_fill_64_bytes:\n\t" $$emit$$"sub 0x8,rcx\n\t" $$emit$$"jge L_loop\n\t" $$emit$$"add 0x4,rcx\n\t" $$emit$$"jl L_tail\n\t" - $$emit$$"vmovdqu $tmp,(rax)\n\t" - $$emit$$"add 0x20,rax\n\t" + $$emit$$"vmovdqu $tmp,($base)\n\t" + $$emit$$"add 0x20,$base\n\t" $$emit$$"sub 0x4,rcx\n\t" - $$emit$$"# L_tail:\t# Clearing tail bytes\n\t" + $$emit$$"# L_tail:\t# Filling tail bytes\n\t" $$emit$$"add 0x4,rcx\n\t" $$emit$$"jle L_end\n\t" $$emit$$"dec rcx\n\t" $$emit$$"# L_sloop:\t# 8-byte short loop\n\t" - $$emit$$"vmovq xmm0,(rax)\n\t" - $$emit$$"add 0x8,rax\n\t" + $$emit$$"vmovq $tmp,($base)\n\t" + $$emit$$"add 0x8,$base\n\t" $$emit$$"dec rcx\n\t" $$emit$$"jge L_sloop\n\t" $$emit$$"# L_end:\n\t" @@ -15113,47 +15127,49 @@ instruct rep_stos_large_word_copy(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_Reg ins_pipe(pipe_slow); %} -// Large non-constant length ClearArray for AVX512 targets. -instruct rep_stos_large_evex(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegL val, +// Large zero fill for AVX512 targets. +instruct rep_stos_large_evex(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, immL0 zero, rax_RegL val, Universe dummy, rFlagsReg cr) %{ - predicate(((ClearArrayNode*)n)->is_large() && !((ClearArrayNode*)n)->word_copy_only() && (UseAVX > 2)); - match(Set dummy (ClearArray (Binary cnt base) val)); - effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, USE_KILL val, KILL cr); + predicate((UseAVX > 2) && + ((ClearArrayNode*)n)->is_large() && + ((ClearArrayNode*)n)->is_zero_fill()); + match(Set dummy (ClearArray (Binary cnt base) zero)); + effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, KILL val, KILL cr); format %{ $$template + $$emit$$"xorq $val, $val\t# ClearArray:\n\t" if (UseFastStosb) { - $$emit$$"xorq rax, rax\t# ClearArray:\n\t" $$emit$$"shlq rcx,3\t# Convert doublewords to bytes\n\t" $$emit$$"rep stosb\t# Store rax to *rdi++ while rcx--" } else if (UseXMMForObjInit) { - $$emit$$"mov rdi,rax\t# ClearArray:\n\t" - $$emit$$"vpxor ymm0,ymm0,ymm0\n\t" + $$emit$$"movdq $tmp, $val\n\t" + $$emit$$"punpcklqdq $tmp, $tmp\n\t" + $$emit$$"vinserti128_high $tmp, $tmp\n\t" $$emit$$"jmpq L_zero_64_bytes\n\t" $$emit$$"# L_loop:\t# 64-byte LOOP\n\t" - $$emit$$"vmovdqu ymm0,(rax)\n\t" - $$emit$$"vmovdqu ymm0,0x20(rax)\n\t" - $$emit$$"add 0x40,rax\n\t" + $$emit$$"vmovdqu $tmp,($base)\n\t" + $$emit$$"vmovdqu $tmp,0x20($base)\n\t" + $$emit$$"add 0x40,$base\n\t" $$emit$$"# L_zero_64_bytes:\n\t" $$emit$$"sub 0x8,rcx\n\t" $$emit$$"jge L_loop\n\t" $$emit$$"add 0x4,rcx\n\t" $$emit$$"jl L_tail\n\t" - $$emit$$"vmovdqu ymm0,(rax)\n\t" - $$emit$$"add 0x20,rax\n\t" + $$emit$$"vmovdqu $tmp,($base)\n\t" + $$emit$$"add 0x20,$base\n\t" $$emit$$"sub 0x4,rcx\n\t" $$emit$$"# L_tail:\t# Clearing tail bytes\n\t" $$emit$$"add 0x4,rcx\n\t" $$emit$$"jle L_end\n\t" $$emit$$"dec rcx\n\t" $$emit$$"# L_sloop:\t# 8-byte short loop\n\t" - $$emit$$"vmovq xmm0,(rax)\n\t" - $$emit$$"add 0x8,rax\n\t" + $$emit$$"vmovq $tmp,($base)\n\t" + $$emit$$"add 0x8,$base\n\t" $$emit$$"dec rcx\n\t" $$emit$$"jge L_sloop\n\t" $$emit$$"# L_end:\n\t" } else { - $$emit$$"xorq rax, rax\t# ClearArray:\n\t" $$emit$$"rep stosq\t# Store rax to *rdi++ while rcx--" } %} @@ -15164,46 +15180,45 @@ instruct rep_stos_large_evex(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp ins_pipe(pipe_slow); %} -instruct rep_stos_large_evex_word_copy(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegL val, +// Large word fill for AVX512 targets. +instruct rep_stos_large_evex_word_fill(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegL val, Universe dummy, rFlagsReg cr) %{ - predicate(((ClearArrayNode*)n)->is_large() && ((ClearArrayNode*)n)->word_copy_only() && (UseAVX > 2)); + predicate((UseAVX > 2) && + ((ClearArrayNode*)n)->is_large() && + ((ClearArrayNode*)n)->requires_word_fill()); match(Set dummy (ClearArray (Binary cnt base) val)); effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, USE_KILL val, KILL cr); format %{ $$template - if (UseFastStosb) { - $$emit$$"xorq rax, rax\t# ClearArray:\n\t" - $$emit$$"shlq rcx,3\t# Convert doublewords to bytes\n\t" - $$emit$$"rep stosb\t# Store rax to *rdi++ while rcx--" - } else if (UseXMMForObjInit) { - $$emit$$"mov rdi,rax\t# ClearArray:\n\t" - $$emit$$"vpxor ymm0,ymm0,ymm0\n\t" - $$emit$$"jmpq L_zero_64_bytes\n\t" + if (UseXMMForObjInit) { + $$emit$$"movdq $tmp, $val\t# ClearArray:\n\t" + $$emit$$"punpcklqdq $tmp, $tmp\n\t" + $$emit$$"vinserti128_high $tmp, $tmp\n\t" + $$emit$$"jmpq L_fill_64_bytes\n\t" $$emit$$"# L_loop:\t# 64-byte LOOP\n\t" - $$emit$$"vmovdqu ymm0,(rax)\n\t" - $$emit$$"vmovdqu ymm0,0x20(rax)\n\t" - $$emit$$"add 0x40,rax\n\t" - $$emit$$"# L_zero_64_bytes:\n\t" + $$emit$$"vmovdqu $tmp,($base)\n\t" + $$emit$$"vmovdqu $tmp,0x20($base)\n\t" + $$emit$$"add 0x40,$base\n\t" + $$emit$$"# L_fill_64_bytes:\n\t" $$emit$$"sub 0x8,rcx\n\t" $$emit$$"jge L_loop\n\t" $$emit$$"add 0x4,rcx\n\t" $$emit$$"jl L_tail\n\t" - $$emit$$"vmovdqu ymm0,(rax)\n\t" - $$emit$$"add 0x20,rax\n\t" + $$emit$$"vmovdqu $tmp,($base)\n\t" + $$emit$$"add 0x20,$base\n\t" $$emit$$"sub 0x4,rcx\n\t" - $$emit$$"# L_tail:\t# Clearing tail bytes\n\t" + $$emit$$"# L_tail:\t# Filling tail bytes\n\t" $$emit$$"add 0x4,rcx\n\t" $$emit$$"jle L_end\n\t" $$emit$$"dec rcx\n\t" $$emit$$"# L_sloop:\t# 8-byte short loop\n\t" - $$emit$$"vmovq xmm0,(rax)\n\t" - $$emit$$"add 0x8,rax\n\t" + $$emit$$"vmovq $tmp,($base)\n\t" + $$emit$$"add 0x8,$base\n\t" $$emit$$"dec rcx\n\t" $$emit$$"jge L_sloop\n\t" $$emit$$"# L_end:\n\t" } else { - $$emit$$"xorq rax, rax\t# ClearArray:\n\t" $$emit$$"rep stosq\t# Store rax to *rdi++ while rcx--" } %} @@ -15214,14 +15229,16 @@ instruct rep_stos_large_evex_word_copy(rcx_RegL cnt, rdi_RegP base, legRegD tmp, ins_pipe(pipe_slow); %} -// Small constant length ClearArray for AVX512 targets. -instruct rep_stos_im(immL cnt, rRegP base, regD tmp, rax_RegL val, kReg ktmp, Universe dummy, rFlagsReg cr) +// Small constant-count zero fill for AVX512 targets. +instruct rep_stos_im(immL cnt, rRegP base, regD tmp, immL0 zero, rRegI val, kReg ktmp, Universe dummy, rFlagsReg cr) %{ - predicate(!((ClearArrayNode*)n)->is_large() && !((ClearArrayNode*)n)->word_copy_only() && - ((MaxVectorSize >= 32) && VM_Version::supports_avx512vl())); - match(Set dummy (ClearArray (Binary cnt base) val)); + predicate((MaxVectorSize >= 32) && + VM_Version::supports_avx512vl() && + !((ClearArrayNode*)n)->is_large() && + ((ClearArrayNode*)n)->is_zero_fill()); + match(Set dummy (ClearArray (Binary cnt base) zero)); ins_cost(100); - effect(TEMP tmp, USE_KILL val, TEMP ktmp, KILL cr); + effect(TEMP tmp, TEMP val, TEMP ktmp, KILL cr); format %{ "clear_mem_imm $base , $cnt \n\t" %} ins_encode %{ __ clear_mem($base$$Register, $cnt$$constant, $val$$Register, $tmp$$XMMRegister, $ktmp$$KRegister); diff --git a/src/hotspot/share/opto/memnode.hpp b/src/hotspot/share/opto/memnode.hpp index ee60e7fefaa..55a42bf3d1c 100644 --- a/src/hotspot/share/opto/memnode.hpp +++ b/src/hotspot/share/opto/memnode.hpp @@ -1151,13 +1151,16 @@ class GetAndSetNNode : public LoadStoreNode { //------------------------------ClearArray------------------------------------- class ClearArrayNode: public Node { private: + // True if cnt is larger than InitArrayShortSize bool _is_large; - bool _word_copy_only; + // True if the fill value is a non-constant or non-zero 64-bit value. Such a + // value must be copied as a complete word and cannot use byte-wise zeroing. + bool _requires_word_fill; static Node* make_address(Node* dest, Node* offset, bool raw_base, PhaseGVN* phase); public: ClearArrayNode( Node *ctrl, Node *arymem, Node *word_cnt, Node *base, Node* val, bool is_large) : Node(ctrl, arymem, word_cnt, base, val), _is_large(is_large), - _word_copy_only(val->bottom_type()->isa_long() && (!val->bottom_type()->is_long()->is_con() || val->bottom_type()->is_long()->get_con() != 0)) { + _requires_word_fill(val->bottom_type()->isa_long() && (!val->bottom_type()->is_long()->is_con() || val->bottom_type()->is_long()->get_con() != 0)) { init_class_id(Class_ClearArray); } virtual int Opcode() const; @@ -1169,7 +1172,8 @@ class ClearArrayNode: public Node { virtual Node *Ideal(PhaseGVN *phase, bool can_reshape); virtual uint match_edge(uint idx) const; bool is_large() const { return _is_large; } - bool word_copy_only() const { return _word_copy_only; } + bool is_zero_fill() const { return !_requires_word_fill; } + bool requires_word_fill() const { return _requires_word_fill; } virtual uint size_of() const { return sizeof(ClearArrayNode); } virtual uint hash() const { return Node::hash() + _is_large; } virtual bool cmp(const Node& n) const {