From 0a62b7c40d93d67ae03867059b09aa6ad6f7cdf5 Mon Sep 17 00:00:00 2001
From: Greg Tucker <greg.b.tucker@intel.com>
Date: Fri, 24 Mar 2017 11:34:53 -0700
Subject: [PATCH] raid: Add avx512 version of xor
Change-Id: I8f8e79f3442ef76268f60a8e61d2c36aedb1ccc1
Signed-off-by: Greg Tucker <greg.b.tucker@intel.com>
---
raid/Makefile.am | 1 +
raid/raid_multibinary.asm | 14 ++-
raid/xor_gen_avx512.asm | 217 ++++++++++++++++++++++++++++++++++++++
raid/xor_gen_perf.c | 4 +-
4 files changed, 231 insertions(+), 5 deletions(-)
create mode 100644 raid/xor_gen_avx512.asm
diff --git a/raid/Makefile.am b/raid/Makefile.am
index 5420549..73e87ca 100644
--- a/raid/Makefile.am
+++ b/raid/Makefile.am
@@ -39,6 +39,7 @@ lsrc_x86_64 += \
raid/pq_gen_avx.asm \
raid/xor_gen_avx.asm \
raid/pq_gen_avx2.asm \
+ raid/xor_gen_avx512.asm \
raid/raid_multibinary.asm
lsrc_x86_32 += \
diff --git a/raid/raid_multibinary.asm b/raid/raid_multibinary.asm
index f079656..97e2f1d 100644
--- a/raid/raid_multibinary.asm
+++ b/raid/raid_multibinary.asm
@@ -54,12 +54,20 @@ extern pq_check_sse
extern xor_check_base
extern xor_check_sse
+%ifdef HAVE_AS_KNOWS_AVX512
+ extern xor_gen_avx512
+%endif
+
mbin_interface xor_gen
mbin_interface pq_gen
-mbin_dispatch_init5 xor_gen, xor_gen_base, xor_gen_sse, xor_gen_avx, xor_gen_avx
-mbin_dispatch_init5 pq_gen, pq_gen_base, pq_gen_sse, pq_gen_avx, pq_gen_avx2
-
+%ifdef HAVE_AS_KNOWS_AVX512
+ mbin_dispatch_init6 xor_gen, xor_gen_base, xor_gen_sse, xor_gen_avx, xor_gen_avx, xor_gen_avx512
+ mbin_dispatch_init5 pq_gen, pq_gen_base, pq_gen_sse, pq_gen_avx, pq_gen_avx2
+%else
+ mbin_dispatch_init5 xor_gen, xor_gen_base, xor_gen_sse, xor_gen_avx, xor_gen_avx
+ mbin_dispatch_init5 pq_gen, pq_gen_base, pq_gen_sse, pq_gen_avx, pq_gen_avx2
+%endif
section .data
diff --git a/raid/xor_gen_avx512.asm b/raid/xor_gen_avx512.asm
new file mode 100644
index 0000000..6892f85
--- /dev/null
+++ b/raid/xor_gen_avx512.asm
@@ -0,0 +1,217 @@
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+; Copyright(c) 2011-2017 Intel Corporation All rights reserved.
+;
+; Redistribution and use in source and binary forms, with or without
+; modification, are permitted provided that the following conditions
+; are met:
+; * Redistributions of source code must retain the above copyright
+; notice, this list of conditions and the following disclaimer.
+; * Redistributions in binary form must reproduce the above copyright
+; notice, this list of conditions and the following disclaimer in
+; the documentation and/or other materials provided with the
+; distribution.
+; * Neither the name of Intel Corporation nor the names of its
+; contributors may be used to endorse or promote products derived
+; from this software without specific prior written permission.
+;
+; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+; "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+; LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+; A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+; OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+; SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+; LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+; DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+; THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;;; Optimized xor of N source vectors using AVX512
+;;; int xor_gen_avx512(int vects, int len, void **array)
+
+;;; Generates xor parity vector from N (vects-1) sources in array of pointers
+;;; (**array). Last pointer is the dest.
+;;; Vectors must be aligned to 32 bytes. Length can be any value.
+
+%include "reg_sizes.asm"
+
+%ifdef HAVE_AS_KNOWS_AVX512
+
+%ifidn __OUTPUT_FORMAT__, elf64
+ %define arg0 rdi
+ %define arg1 rsi
+ %define arg2 rdx
+ %define arg3 rcx
+ %define arg4 r8
+ %define arg5 r9
+ %define tmp r11
+ %define tmp3 arg4
+ %define func(x) x:
+ %define return rax
+ %define FUNC_SAVE
+ %define FUNC_RESTORE
+
+%elifidn __OUTPUT_FORMAT__, win64
+ %define arg0 rcx
+ %define arg1 rdx
+ %define arg2 r8
+ %define arg3 r9
+ %define tmp r11
+ %define tmp3 r10
+ %define func(x) proc_frame x
+ %define return rax
+ %define stack_size 2*16 + 8 ;must be an odd multiple of 8
+
+ %macro FUNC_SAVE 0
+ alloc_stack stack_size
+ vmovdqu [rsp + 0*16], xmm6
+ vmovdqu [rsp + 1*16], xmm7
+ end_prolog
+ %endmacro
+ %macro FUNC_RESTORE 0
+ vmovdqu xmm6, [rsp + 0*16]
+ vmovdqu xmm7, [rsp + 1*316]
+ add rsp, stack_size
+ %endmacro
+
+%endif ;output formats
+
+
+%define vec arg0
+%define len arg1
+%define ptr arg3
+%define tmp2 rax
+%define tmp2.b al
+%define pos tmp3
+%define PS 8
+
+%define NO_NT_LDST
+;;; Use Non-temporal load/stor
+%ifdef NO_NT_LDST
+ %define XLDR vmovdqu8
+ %define XSTR vmovdqu8
+%else
+ %define XLDR vmovntdqa
+ %define XSTR vmovntdq
+%endif
+
+
+default rel
+[bits 64]
+
+section .text
+
+align 16
+global xor_gen_avx512:function
+func(xor_gen_avx512)
+ FUNC_SAVE
+ sub vec, 2 ;Keep as offset to last source
+ jng return_fail ;Must have at least 2 sources
+ cmp len, 0
+ je return_pass
+ test len, (128-1) ;Check alignment of length
+ jnz len_not_aligned
+
+len_aligned_128bytes:
+ sub len, 128
+ mov pos, 0
+
+loop128:
+ mov tmp, vec ;Back to last vector
+ mov tmp2, [arg2+vec*PS] ;Fetch last pointer in array
+ sub tmp, 1 ;Next vect
+ XLDR zmm0, [tmp2+pos] ;Start with end of array in last vector
+ XLDR zmm1, [tmp2+pos+64] ;Keep xor parity in xmm0-7
+
+next_vect:
+ mov ptr, [arg2+tmp*PS]
+ sub tmp, 1
+ XLDR zmm4, [ptr+pos] ;Get next vector (source)
+ XLDR zmm5, [ptr+pos+64]
+ vpxorq zmm0, zmm0, zmm4 ;Add to xor parity
+ vpxorq zmm1, zmm1, zmm5
+ jge next_vect ;Loop for each source
+
+ mov ptr, [arg2+PS+vec*PS] ;Address of parity vector
+ XSTR [ptr+pos], zmm0 ;Write parity xor vector
+ XSTR [ptr+pos+64], zmm1
+ add pos, 128
+ cmp pos, len
+ jle loop128
+
+return_pass:
+ FUNC_RESTORE
+ mov return, 0
+ ret
+
+
+;;; Do one byte at a time for no alignment case
+loop_1byte:
+ mov tmp, vec ;Back to last vector
+ mov ptr, [arg2+vec*PS] ;Fetch last pointer in array
+ mov tmp2.b, [ptr+len-1] ;Get array n
+ sub tmp, 1
+nextvect_1byte:
+ mov ptr, [arg2+tmp*PS]
+ xor tmp2.b, [ptr+len-1]
+ sub tmp, 1
+ jge nextvect_1byte
+
+ mov tmp, vec
+ add tmp, 1 ;Add back to point to last vec
+ mov ptr, [arg2+tmp*PS]
+ mov [ptr+len-1], tmp2.b ;Write parity
+ sub len, 1
+ test len, (PS-1)
+ jnz loop_1byte
+
+ cmp len, 0
+ je return_pass
+ test len, (128-1) ;If not 0 and 128bit aligned
+ jz len_aligned_128bytes ; then do aligned case. len = y * 128
+
+ ;; else we are 8-byte aligned so fall through to recheck
+
+
+ ;; Unaligned length cases
+len_not_aligned:
+ test len, (PS-1)
+ jne loop_1byte
+ mov tmp3, len
+ and tmp3, (128-1) ;Do the unaligned bytes 8 at a time
+
+ ;; Run backwards 8 bytes at a time for (tmp3) bytes
+loop8_bytes:
+ mov tmp, vec ;Back to last vector
+ mov ptr, [arg2+vec*PS] ;Fetch last pointer in array
+ mov tmp2, [ptr+len-PS] ;Get array n
+ sub tmp, 1
+nextvect_8bytes:
+ mov ptr, [arg2+tmp*PS] ;Get pointer to next vector
+ xor tmp2, [ptr+len-PS]
+ sub tmp, 1
+ jge nextvect_8bytes ;Loop for each source
+
+ mov tmp, vec
+ add tmp, 1 ;Add back to point to last vec
+ mov ptr, [arg2+tmp*PS]
+ mov [ptr+len-PS], tmp2 ;Write parity
+ sub len, PS
+ sub tmp3, PS
+ jg loop8_bytes
+
+ cmp len, 128 ;Now len is aligned to 128B
+ jge len_aligned_128bytes ;We can do the rest aligned
+
+ cmp len, 0
+ je return_pass
+
+return_fail:
+ FUNC_RESTORE
+ mov return, 1
+ ret
+
+endproc_frame
+
+%endif ; ifdef HAVE_AS_KNOWS_AVX512
diff --git a/raid/xor_gen_perf.c b/raid/xor_gen_perf.c
index 53a963d..25b33cb 100644
--- a/raid/xor_gen_perf.c
+++ b/raid/xor_gen_perf.c
@@ -40,7 +40,7 @@
// Loop many times over same
# define TEST_SOURCES 10
# define TEST_LEN 8*1024
-# define TEST_LOOPS 400000
+# define TEST_LOOPS 2000000
# define TEST_TYPE_STR "_warm"
#else
// Uncached test. Pull from large mem base.
@@ -72,7 +72,7 @@ int main(int argc, char *argv[])
// Allocate the arrays
for (i = 0; i < TEST_SOURCES + 1; i++) {
void *buf;
- ret = posix_memalign(&buf, 32, TEST_LEN);
+ ret = posix_memalign(&buf, 64, TEST_LEN);
if (ret) {
printf("alloc error: Fail");
return 1;