Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Avoid copy on Leopard GF8 #227

Draft
wants to merge 4 commits into
base: master
Choose a base branch
from
Draft

Avoid copy on Leopard GF8 #227

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
8b1c2b8
Avoid copy on Leopard GF8
klauspost Nov 21, 2022
a97e582
Pool work shards.
klauspost Nov 22, 2022
87f23e8
Use zero array instead of pool.
klauspost Nov 22, 2022
52b7f16
Use first for size.
klauspost Nov 22, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
107 changes: 93 additions & 14 deletions _gen/gf8.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -119,9 +119,14 @@ func genGF8() {

x := [8]int{}
for skipMask := range x[:] {
{
for _, withDst := range []bool{false, true} {
var suffix = "avx2_" + fmt.Sprint(skipMask)
TEXT("ifftDIT48_"+suffix, attr.NOSPLIT, fmt.Sprintf("func(work [][]byte, dist int, t01, t23, t02 *[2*16]uint8)"))
dstString := ""
if withDst {
dstString = "dst, "
suffix = "dst_" + suffix
}
TEXT("ifftDIT48_"+suffix, attr.NOSPLIT, fmt.Sprintf("func(%swork [][]byte, dist int, t01, t23, t02 *[2*16]uint8)", dstString))
Pragma("noescape")
var t01, t23, t02 table256
// Load and expand tables
Expand Down Expand Up @@ -153,6 +158,11 @@ func genGF8() {

var work [4]reg.GPVirtual
workTable := Load(Param("work").Base(), GP64()) // &work[0]
var dst [4]reg.GPVirtual
dstTable := GP64()
if withDst {
Load(Param("dst").Base(), dstTable) // &dst[0]
}
bytes := GP64()
MOVQ(Mem{Base: workTable, Disp: 8}, bytes)

Expand All @@ -162,6 +172,10 @@ func genGF8() {
work[i] = GP64()
// work[i] = &workTable[dist*i]
MOVQ(Mem{Base: workTable, Index: offset, Scale: 1}, work[i])
if withDst {
dst[i] = GP64()
MOVQ(Mem{Base: dstTable, Index: offset, Scale: 1}, dst[i])
}
if i < len(work)-1 {
ADDQ(dist, offset)
}
Expand Down Expand Up @@ -231,8 +245,14 @@ func genGF8() {

// Store + Next loop:
for i := range work {
VMOVDQU(workReg[i], Mem{Base: work[i], Disp: 0})
VMOVDQU(workReg2[i], Mem{Base: work[i], Disp: 32})
if withDst {
VMOVDQU(workReg[i], Mem{Base: dst[i], Disp: 0})
VMOVDQU(workReg2[i], Mem{Base: dst[i], Disp: 32})
ADDQ(U8(64), dst[i])
} else {
VMOVDQU(workReg[i], Mem{Base: work[i], Disp: 0})
VMOVDQU(workReg2[i], Mem{Base: work[i], Disp: 32})
}
ADDQ(U8(64), work[i])
}

Expand All @@ -242,9 +262,14 @@ func genGF8() {
VZEROUPPER()
RET()
}
{
for _, withDst := range []bool{false, true} {
var suffix = "avx2_" + fmt.Sprint(skipMask)
TEXT("fftDIT48_"+suffix, attr.NOSPLIT, fmt.Sprintf("func(work [][]byte, dist int, t01, t23, t02 *[2*16]uint8)"))
dstString := ""
if withDst {
dstString = "dst, "
suffix = "dst_" + suffix
}
TEXT("fftDIT48_"+suffix, attr.NOSPLIT, fmt.Sprintf("func(%swork [][]byte, dist int, t01, t23, t02 *[2*16]uint8)", dstString))
Pragma("noescape")
var t01, t23, t02 table256
// Load and expand tables
Expand Down Expand Up @@ -280,16 +305,26 @@ func genGF8() {
dist := Load(Param("dist"), GP64())

var work [4]reg.GPVirtual
var dst [4]reg.GPVirtual

workTable := Load(Param("work").Base(), GP64()) // &work[0]
bytes := GP64()
MOVQ(Mem{Base: workTable, Disp: 8}, bytes)
dstTable := GP64()
if withDst {
Load(Param("dst").Base(), dstTable) // &dst[0]
}

offset := GP64()
XORQ(offset, offset)
for i := range work {
work[i] = GP64()
// work[i] = &workTable[dist*i]
MOVQ(Mem{Base: workTable, Index: offset, Scale: 1}, work[i])
if withDst {
dst[i] = GP64()
MOVQ(Mem{Base: dstTable, Index: offset, Scale: 1}, dst[i])
}
if i < len(work)-1 {
ADDQ(dist, offset)
}
Expand Down Expand Up @@ -356,8 +391,14 @@ func genGF8() {

// Store + Next loop:
for i := range work {
VMOVDQU(workReg[i], Mem{Base: work[i], Disp: 0})
VMOVDQU(workReg2[i], Mem{Base: work[i], Disp: 32})
if withDst {
VMOVDQU(workReg[i], Mem{Base: dst[i], Disp: 0})
VMOVDQU(workReg2[i], Mem{Base: dst[i], Disp: 32})
ADDQ(U8(64), dst[i])
} else {
VMOVDQU(workReg[i], Mem{Base: work[i], Disp: 0})
VMOVDQU(workReg2[i], Mem{Base: work[i], Disp: 32})
}
ADDQ(U8(64), work[i])
}

Expand All @@ -371,9 +412,14 @@ func genGF8() {

// GFNI
for skipMask := range x[:] {
{
for _, withDst := range []bool{false, true} {
var suffix = "gfni_" + fmt.Sprint(skipMask)
TEXT("ifftDIT48_"+suffix, attr.NOSPLIT, fmt.Sprintf("func(work [][]byte, dist int, t01, t23, t02 uint64)"))
dstString := ""
if withDst {
dstString = "dst, "
suffix = "dst_" + suffix
}
TEXT("ifftDIT48_"+suffix, attr.NOSPLIT, fmt.Sprintf("func(%swork [][]byte, dist int, t01, t23, t02 uint64)", dstString))
Pragma("noescape")
var t01, t23, t02 table512 = ZMM(), ZMM(), ZMM()
// Load and expand tables
Expand All @@ -394,6 +440,11 @@ func genGF8() {

var work [4]reg.GPVirtual
workTable := Load(Param("work").Base(), GP64()) // &work[0]
var dst [4]reg.GPVirtual
dstTable := GP64()
if withDst {
Load(Param("dst").Base(), dstTable) // &dst[0]
}
bytes := GP64()
MOVQ(Mem{Base: workTable, Disp: 8}, bytes)

Expand All @@ -403,6 +454,10 @@ func genGF8() {
work[i] = GP64()
// work[i] = &workTable[dist*i]
MOVQ(Mem{Base: workTable, Index: offset, Scale: 1}, work[i])
if withDst {
dst[i] = GP64()
MOVQ(Mem{Base: dstTable, Index: offset, Scale: 1}, dst[i])
}
if i < len(work)-1 {
ADDQ(dist, offset)
}
Expand Down Expand Up @@ -442,7 +497,12 @@ func genGF8() {

// Store + Next loop:
for i := range work {
VMOVDQU64(workReg[i], Mem{Base: work[i], Disp: 0})
if withDst {
VMOVDQU64(workReg[i], Mem{Base: dst[i], Disp: 0})
ADDQ(U8(64), dst[i])
} else {
VMOVDQU64(workReg[i], Mem{Base: work[i], Disp: 0})
}
ADDQ(U8(64), work[i])
}

Expand All @@ -452,9 +512,14 @@ func genGF8() {
VZEROUPPER()
RET()
}
{
for _, withDst := range []bool{false, true} {
var suffix = "gfni_" + fmt.Sprint(skipMask)
TEXT("fftDIT48_"+suffix, attr.NOSPLIT, fmt.Sprintf("func(work [][]byte, dist int, t01, t23, t02 uint64)"))
dstString := ""
if withDst {
dstString = "dst, "
suffix = "dst_" + suffix
}
TEXT("fftDIT48_"+suffix, attr.NOSPLIT, fmt.Sprintf("func(%swork [][]byte, dist int, t01, t23, t02 uint64)", dstString))
Pragma("noescape")
var t01, t23, t02 table512 = ZMM(), ZMM(), ZMM()
// Load and expand tables
Expand All @@ -475,6 +540,11 @@ func genGF8() {

var work [4]reg.GPVirtual
workTable := Load(Param("work").Base(), GP64()) // &work[0]
var dst [4]reg.GPVirtual
dstTable := GP64()
if withDst {
Load(Param("dst").Base(), dstTable) // &dst[0]
}
bytes := GP64()
MOVQ(Mem{Base: workTable, Disp: 8}, bytes)

Expand All @@ -484,6 +554,10 @@ func genGF8() {
work[i] = GP64()
// work[i] = &workTable[dist*i]
MOVQ(Mem{Base: workTable, Index: offset, Scale: 1}, work[i])
if withDst {
dst[i] = GP64()
MOVQ(Mem{Base: dstTable, Index: offset, Scale: 1}, dst[i])
}
if i < len(work)-1 {
ADDQ(dist, offset)
}
Expand Down Expand Up @@ -522,7 +596,12 @@ func genGF8() {

// Store + Next loop:
for i := range work {
VMOVDQU64(workReg[i], Mem{Base: work[i], Disp: 0})
if withDst {
VMOVDQU64(workReg[i], Mem{Base: dst[i], Disp: 0})
ADDQ(U8(64), dst[i])
} else {
VMOVDQU64(workReg[i], Mem{Base: work[i], Disp: 0})
}
ADDQ(U8(64), work[i])
}

Expand Down
81 changes: 81 additions & 0 deletions galois_amd64.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -308,6 +308,87 @@ func ifftDIT48(work [][]byte, dist int, log_m01, log_m23, log_m02 ffe8, o *optio
ifftDIT4Ref8(work, dist, log_m01, log_m23, log_m02, o)
}

// 4-way butterfly
func ifftDIT48Dst(dst, work [][]byte, dist int, log_m01, log_m23, log_m02 ffe8, o *options) {
if len(work[0]) == 0 {
return
}

if o.useGFNI {
// Note that these currently require that length is multiple of 64.
t01 := gf2p811dMulMatrices[log_m01]
t23 := gf2p811dMulMatrices[log_m23]
t02 := gf2p811dMulMatrices[log_m02]
if log_m01 == modulus8 {
if log_m23 == modulus8 {
if log_m02 == modulus8 {
ifftDIT48_dst_gfni_7(dst, work, dist*24, t01, t23, t02)
} else {
ifftDIT48_dst_gfni_3(dst, work, dist*24, t01, t23, t02)
}
} else {
if log_m02 == modulus8 {
ifftDIT48_dst_gfni_5(dst, work, dist*24, t01, t23, t02)
} else {
ifftDIT48_dst_gfni_1(dst, work, dist*24, t01, t23, t02)
}
}
} else {
if log_m23 == modulus8 {
if log_m02 == modulus8 {
ifftDIT48_dst_gfni_6(dst, work, dist*24, t01, t23, t02)
} else {
ifftDIT48_dst_gfni_2(dst, work, dist*24, t01, t23, t02)
}
} else {
if log_m02 == modulus8 {
ifftDIT48_dst_gfni_4(dst, work, dist*24, t01, t23, t02)
} else {
ifftDIT48_dst_gfni_0(dst, work, dist*24, t01, t23, t02)
}
}
}
return
}
if o.useAVX2 {
// Note that these currently require that length is multiple of 64.
t01 := &multiply256LUT8[log_m01]
t23 := &multiply256LUT8[log_m23]
t02 := &multiply256LUT8[log_m02]
if log_m01 == modulus8 {
if log_m23 == modulus8 {
if log_m02 == modulus8 {
ifftDIT48_dst_avx2_7(dst, work, dist*24, t01, t23, t02)
} else {
ifftDIT48_dst_avx2_3(dst, work, dist*24, t01, t23, t02)
}
} else {
if log_m02 == modulus8 {
ifftDIT48_dst_avx2_5(dst, work, dist*24, t01, t23, t02)
} else {
ifftDIT48_dst_avx2_1(dst, work, dist*24, t01, t23, t02)
}
}
} else {
if log_m23 == modulus8 {
if log_m02 == modulus8 {
ifftDIT48_dst_avx2_6(dst, work, dist*24, t01, t23, t02)
} else {
ifftDIT48_dst_avx2_2(dst, work, dist*24, t01, t23, t02)
}
} else {
if log_m02 == modulus8 {
ifftDIT48_dst_avx2_4(dst, work, dist*24, t01, t23, t02)
} else {
ifftDIT48_dst_avx2_0(dst, work, dist*24, t01, t23, t02)
}
}
}
return
}
ifftDIT4DstRef8(dst, work, dist, log_m01, log_m23, log_m02, o)
}

func fftDIT4(work [][]byte, dist int, log_m01, log_m23, log_m02 ffe, o *options) {
if len(work[0]) == 0 {
return
Expand Down
5 changes: 5 additions & 0 deletions galois_arm64.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,11 @@ func ifftDIT48(work [][]byte, dist int, log_m01, log_m23, log_m02 ffe8, o *optio
ifftDIT4Ref8(work, dist, log_m01, log_m23, log_m02, o)
}

// 4-way butterfly with separate destination
func ifftDIT48Dst(dst, work [][]byte, dist int, log_m01, log_m23, log_m02 ffe8, o *options) {
ifftDIT4DstRef8(dst, work, dist, log_m01, log_m23, log_m02, o)
}

// 4-way butterfly
func fftDIT4(work [][]byte, dist int, log_m01, log_m23, log_m02 ffe, o *options) {
fftDIT4Ref(work, dist, log_m01, log_m23, log_m02, o)
Expand Down
Loading