openstack/deb-libisal
Code Issues Proposed changes
869712d389
deb-libisal/igzip/igzip_buffer_utils_01.asm
Roy Oursler f97de75fc6 igzip: Port improvements to stateless compress to stateful compress 
Signed-off-by: Roy Oursler <roy.j.oursler@intel.com>
Reviewed-by: Greg Tucker <greg.b.tucker@intel.com>
2016-12-02 18:40:27 -07:00

544 lines
13 KiB
NASM
Raw Blame History

;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
; Copyright(c) 2011-2016 Intel Corporation All rights reserved.
;
; Redistribution and use in source and binary forms, with or without
; modification, are permitted provided that the following conditions
; are met:
; * Redistributions of source code must retain the above copyright
; notice, this list of conditions and the following disclaimer.
; * Redistributions in binary form must reproduce the above copyright
; notice, this list of conditions and the following disclaimer in
; the documentation and/or other materials provided with the
; distribution.
; * Neither the name of Intel Corporation nor the names of its
; contributors may be used to endorse or promote products derived
; from this software without specific prior written permission.
;
; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
; "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
; LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
; A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
; OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
; SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
; LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
; DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
; THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
%ifndef BUFFER_UTILS
%define BUFFER_UTILS
%include "options.asm"
extern pshufb_shf_table
extern mask3
%ifdef FIX_CACHE_READ
%define movntdqa movdqa
%else
%macro prefetchnta 1
%endm
%endif
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
; code for doing the CRC calculation as part of copy-in, using pclmulqdq
; "shift" 4 input registers down 4 places
; macro FOLD4 xmm0, xmm1, xmm2, xmm3, const, tmp0, tmp1
%macro FOLD4 7
%define %%xmm0 %1 ; xmm reg, in/out
%define %%xmm1 %2 ; xmm reg, in/out
%define %%xmm2 %3 ; xmm reg, in/out
%define %%xmm3 %4 ; xmm reg, in/out
%define %%const %5 ; xmm reg, in
%define %%tmp0 %6 ; xmm reg, tmp
%define %%tmp1 %7 ; xmm reg, tmp
movaps %%tmp0, %%xmm0
movaps %%tmp1, %%xmm1
pclmulqdq %%xmm0, %%const, 0x01
pclmulqdq %%xmm1, %%const, 0x01
pclmulqdq %%tmp0, %%const, 0x10
pclmulqdq %%tmp1, %%const, 0x10
xorps %%xmm0, %%tmp0
xorps %%xmm1, %%tmp1
movaps %%tmp0, %%xmm2
movaps %%tmp1, %%xmm3
pclmulqdq %%xmm2, %%const, 0x01
pclmulqdq %%xmm3, %%const, 0x01
pclmulqdq %%tmp0, %%const, 0x10
pclmulqdq %%tmp1, %%const, 0x10
xorps %%xmm2, %%tmp0
xorps %%xmm3, %%tmp1
%endm
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
; "shift" 3 input registers down 4 places
; macro FOLD3 x0, x1, x2, x3, const, tmp0
; x0 x1 x2 x3
; In A B C D
; Out D A' B' C'
%macro FOLD3 6
%define %%x0 %1 ; xmm reg, in/out
%define %%x1 %2 ; xmm reg, in/out
%define %%x2 %3 ; xmm reg, in/out
%define %%x3 %4 ; xmm reg, in/out
%define %%const %5 ; xmm reg, in
%define %%tmp0 %6 ; xmm reg, tmp
movdqa %%tmp0, %%x3
movaps %%x3, %%x2
pclmulqdq %%x2, %%const, 0x01
pclmulqdq %%x3, %%const, 0x10
xorps %%x3, %%x2
movaps %%x2, %%x1
pclmulqdq %%x1, %%const, 0x01
pclmulqdq %%x2, %%const, 0x10
xorps %%x2, %%x1
movaps %%x1, %%x0
pclmulqdq %%x0, %%const, 0x01
pclmulqdq %%x1, %%const, 0x10
xorps %%x1, %%x0
movdqa %%x0, %%tmp0
%endm
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
; "shift" 2 input registers down 4 places
; macro FOLD2 x0, x1, x2, x3, const, tmp0
; x0 x1 x2 x3
; In A B C D
; Out C D A' B'
%macro FOLD2 6
%define %%x0 %1 ; xmm reg, in/out
%define %%x1 %2 ; xmm reg, in/out
%define %%x2 %3 ; xmm reg, in/out
%define %%x3 %4 ; xmm reg, in/out
%define %%const %5 ; xmm reg, in
%define %%tmp0 %6 ; xmm reg, tmp
movdqa %%tmp0, %%x3
movaps %%x3, %%x1
pclmulqdq %%x1, %%const, 0x01
pclmulqdq %%x3, %%const, 0x10
xorps %%x3, %%x1
movdqa %%x1, %%tmp0
movdqa %%tmp0, %%x2
movaps %%x2, %%x0
pclmulqdq %%x0, %%const, 0x01
pclmulqdq %%x2, %%const, 0x10
xorps %%x2, %%x0
movdqa %%x0, %%tmp0
%endm
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
; "shift" 1 input registers down 4 places
; macro FOLD1 x0, x1, x2, x3, const, tmp0
; x0 x1 x2 x3
; In A B C D
; Out B C D A'
%macro FOLD1 6
%define %%x0 %1 ; xmm reg, in/out
%define %%x1 %2 ; xmm reg, in/out
%define %%x2 %3 ; xmm reg, in/out
%define %%x3 %4 ; xmm reg, in/out
%define %%const %5 ; xmm reg, in
%define %%tmp0 %6 ; xmm reg, tmp
movdqa %%tmp0, %%x3
movaps %%x3, %%x0
pclmulqdq %%x0, %%const, 0x01
pclmulqdq %%x3, %%const, 0x10
xorps %%x3, %%x0
movdqa %%x0, %%x1
movdqa %%x1, %%x2
movdqa %%x2, %%tmp0
%endm
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
; macro PARTIAL_FOLD x0, x1, x2, x3, xp, size, xfold, xt0, xt1, xt2, xt3
; XP X3 X2 X1 X0 tmp2
; Initial state xI HG FE DC BA
; after shift IH GF ED CB A0
; after fold ff GF ED CB ff = merge(IH, A0)
;
%macro PARTIAL_FOLD 12
%define %%x0 %1 ; xmm reg, in/out
%define %%x1 %2 ; xmm reg, in/out
%define %%x2 %3 ; xmm reg, in/out
%define %%x3 %4 ; xmm reg, in/out
%define %%xp %5 ; xmm partial reg, in/clobbered
%define %%size %6 ; GPR, in/clobbered (1...15)
%define %%const %7 ; xmm reg, in
%define %%shl %8 ; xmm reg, tmp
%define %%shr %9 ; xmm reg, tmp
%define %%tmp2 %10 ; xmm reg, tmp
%define %%tmp3 %11 ; xmm reg, tmp
%define %%gtmp %12 ; GPR, tmp
; {XP X3 X2 X1 X0} = {xI HG FE DC BA}
shl %%size, 4 ; size *= 16
lea %%gtmp, [pshufb_shf_table - 16 WRT_OPT]
movdqa %%shl, [%%gtmp + %%size] ; shl constant
movdqa %%shr, %%shl
pxor %%shr, [mask3 WRT_OPT] ; shr constant
movdqa %%tmp2, %%x0 ; tmp2 = BA
pshufb %%tmp2, %%shl ; tmp2 = A0
pshufb %%x0, %%shr ; x0 = 0B
movdqa %%tmp3, %%x1 ; tmp3 = DC
pshufb %%tmp3, %%shl ; tmp3 = C0
por %%x0, %%tmp3 ; x0 = CB
pshufb %%x1, %%shr ; x1 = 0D
movdqa %%tmp3, %%x2 ; tmp3 = FE
pshufb %%tmp3, %%shl ; tmp3 = E0
por %%x1, %%tmp3 ; x1 = ED
pshufb %%x2, %%shr ; x2 = 0F
movdqa %%tmp3, %%x3 ; tmp3 = HG
pshufb %%tmp3, %%shl ; tmp3 = G0
por %%x2, %%tmp3 ; x2 = GF
pshufb %%x3, %%shr ; x3 = 0H
pshufb %%xp, %%shl ; xp = I0
por %%x3, %%xp ; x3 = IH
; fold tmp2 into X3
movaps %%tmp3, %%tmp2
pclmulqdq %%tmp2, %%const, 0x01
pclmulqdq %%tmp3, %%const, 0x10
xorps %%x3, %%tmp2
xorps %%x3, %%tmp3
%endm
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
; LOAD_FRACTIONAL_XMM: Packs xmm register with data when data input is less than 16 bytes.
; Returns 0 if data has length 0.
; Input: The input data (src), that data's length (size).
; Output: The packed xmm register (xmm_out).
; size is clobbered.
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
%macro LOAD_FRACTIONAL_XMM 3
%define %%xmm_out %1 ; %%xmm_out is an xmm register
%define %%src %2
%define %%size %3
pxor %%xmm_out, %%xmm_out
cmp %%size, 0
je %%_done
add %%src, %%size
cmp %%size, 8
jl %%_byte_loop
sub %%src, 8
pinsrq %%xmm_out, [%%src], 0 ;Read in 8 bytes if they exists
sub %%size, 8
je %%_done
%%_byte_loop: ;Read in data 1 byte at a time while data is left
pslldq %%xmm_out, 1
dec %%src
pinsrb %%xmm_out, BYTE [%%src], 0
dec %%size
jg %%_byte_loop
%%_done:
%endmacro ; LOAD_FRACTIONAL_XMM
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
; copy x bytes (rounded up to 16 bytes) from src to dst
; src & dst are unaligned
; macro COPY_IN_CRC dst, src, size_in_bytes, tmp, x0, x1, x2, x3, xfold,
; xt0, xt1, xt2, xt3, xt4
%macro COPY_IN_CRC 14
%define %%dst %1 ; reg, in/clobbered
%define %%src %2 ; reg, in/clobbered
%define %%size %3 ; reg, in/clobbered
%define %%tmp %4 ; reg, tmp
%define %%x0 %5 ; xmm, in/out: crc state
%define %%x1 %6 ; xmm, in/out: crc state
%define %%x2 %7 ; xmm, in/out: crc state
%define %%x3 %8 ; xmm, in/out: crc state
%define %%xfold %9 ; xmm, in: (loaded from fold4)
%define %%xtmp0 %10 ; xmm, tmp
%define %%xtmp1 %11 ; xmm, tmp
%define %%xtmp2 %12 ; xmm, tmp
%define %%xtmp3 %13 ; xmm, tmp
%define %%xtmp4 %14 ; xmm, tmp
cmp %%size, 16
jl %%lt_16
; align source
xor %%tmp, %%tmp
sub %%tmp, %%src
and %%tmp, 15
jz %%already_aligned
; need to align, tmp contains number of bytes to transfer
movdqu %%xtmp0, [%%src]
movdqu [%%dst], %%xtmp0
add %%dst, %%tmp
add %%src, %%tmp
sub %%size, %%tmp
%ifndef DEFLATE
push %%dst
PARTIAL_FOLD %%x0, %%x1, %%x2, %%x3, %%xtmp0, %%tmp, %%xfold, \
%%xtmp1, %%xtmp2, %%xtmp3, %%xtmp4, %%dst
pop %%dst
%endif
%%already_aligned:
sub %%size, 64
jl %%end_loop
jmp %%loop
align 16
%%loop:
movntdqa %%xtmp0, [%%src+0*16]
movntdqa %%xtmp1, [%%src+1*16]
movntdqa %%xtmp2, [%%src+2*16]
%ifndef DEFLATE
FOLD4 %%x0, %%x1, %%x2, %%x3, %%xfold, %%xtmp3, %%xtmp4
%endif
movntdqa %%xtmp3, [%%src+3*16]
movdqu [%%dst+0*16], %%xtmp0
movdqu [%%dst+1*16], %%xtmp1
movdqu [%%dst+2*16], %%xtmp2
movdqu [%%dst+3*16], %%xtmp3
%ifndef DEFLATE
pxor %%x0, %%xtmp0
pxor %%x1, %%xtmp1
pxor %%x2, %%xtmp2
pxor %%x3, %%xtmp3
%endif
add %%src, 4*16
add %%dst, 4*16
sub %%size, 4*16
jge %%loop
%%end_loop:
; %%size contains (num bytes left - 64)
add %%size, 16
jge %%three_full_regs
add %%size, 16
jge %%two_full_regs
add %%size, 16
jge %%one_full_reg
add %%size, 16
%%no_full_regs: ; 0 <= %%size < 16, no full regs
jz %%done ; if no bytes left, we're done
jmp %%partial
;; Handle case where input is <16 bytes
%%lt_16:
test %%size, %%size
jz %%done ; if no bytes left, we're done
jmp %%partial
%%one_full_reg:
movntdqa %%xtmp0, [%%src+0*16]
%ifndef DEFLATE
FOLD1 %%x0, %%x1, %%x2, %%x3, %%xfold, %%xtmp3
%endif
movdqu [%%dst+0*16], %%xtmp0
%ifndef DEFLATE
pxor %%x3, %%xtmp0
%endif
test %%size, %%size
jz %%done ; if no bytes left, we're done
add %%dst, 1*16
add %%src, 1*16
jmp %%partial
%%two_full_regs:
movntdqa %%xtmp0, [%%src+0*16]
movntdqa %%xtmp1, [%%src+1*16]
%ifndef DEFLATE
FOLD2 %%x0, %%x1, %%x2, %%x3, %%xfold, %%xtmp3
%endif
movdqu [%%dst+0*16], %%xtmp0
movdqu [%%dst+1*16], %%xtmp1
%ifndef DEFLATE
pxor %%x2, %%xtmp0
pxor %%x3, %%xtmp1
%endif
test %%size, %%size
jz %%done ; if no bytes left, we're done
add %%dst, 2*16
add %%src, 2*16
jmp %%partial
%%three_full_regs:
movntdqa %%xtmp0, [%%src+0*16]
movntdqa %%xtmp1, [%%src+1*16]
movntdqa %%xtmp2, [%%src+2*16]
%ifndef DEFLATE
FOLD3 %%x0, %%x1, %%x2, %%x3, %%xfold, %%xtmp3
%endif
movdqu [%%dst+0*16], %%xtmp0
movdqu [%%dst+1*16], %%xtmp1
movdqu [%%dst+2*16], %%xtmp2
%ifndef DEFLATE
pxor %%x1, %%xtmp0
pxor %%x2, %%xtmp1
pxor %%x3, %%xtmp2
%endif
test %%size, %%size
jz %%done ; if no bytes left, we're done
add %%dst, 3*16
add %%src, 3*16
; fall through to %%partial
%%partial: ; 0 <= %%size < 16
%ifndef DEFLATE
mov %%tmp, %%size
%endif
LOAD_FRACTIONAL_XMM %%xtmp0, %%src, %%size
movdqu [%%dst], %%xtmp0
%ifndef DEFLATE
PARTIAL_FOLD %%x0, %%x1, %%x2, %%x3, %%xtmp0, %%tmp, %%xfold, \
%%xtmp1, %%xtmp2, %%xtmp3, %%xtmp4, %%dst
%endif
%%done:
%endm
;%assign K 1024;
;%assign D 8 * K; ; Amount of history
;%assign LA 18 * 16; ; Max look-ahead, rounded up to 32 byte boundary
; copy D + LA bytes from src to dst
; dst is aligned
;void copy_D_LA(uint8_t *dst, uint8_t *src);
; arg 1: rcx : dst
; arg 2: rdx : src
; copy_D_LA dst, src, tmp, xtmp0, xtmp1, xtmp2, xtmp3
%macro copy_D_LA 7
%define %%dst %1 ; reg, clobbered
%define %%src %2 ; reg, clobbered
%define %%tmp %3
%define %%xtmp0 %4
%define %%xtmp1 %5
%define %%xtmp2 %6
%define %%xtmp3 %7
%assign %%SIZE (D + LA) / 16 ; number of DQ words to be copied
%assign %%SIZE4 %%SIZE/4
lea %%tmp, [%%dst + 4 * 16 * %%SIZE4]
jmp %%copy_D_LA_1
align 16
%%copy_D_LA_1:
movdqu %%xtmp0, [%%src]
movdqu %%xtmp1, [%%src+16]
movdqu %%xtmp2, [%%src+32]
movdqu %%xtmp3, [%%src+48]
movdqa [%%dst], %%xtmp0
movdqa [%%dst+16], %%xtmp1
movdqa [%%dst+32], %%xtmp2
movdqa [%%dst+48], %%xtmp3
add %%src, 4*16
add %%dst, 4*16
cmp %%dst, %%tmp
jne %%copy_D_LA_1
%assign %%i 0
%rep (%%SIZE - 4 * %%SIZE4)
%if (%%i == 0)
movdqu %%xtmp0, [%%src + %%i*16]
%elif (%%i == 1)
movdqu %%xtmp1, [%%src + %%i*16]
%elif (%%i == 2)
movdqu %%xtmp2, [%%src + %%i*16]
%elif (%%i == 3)
movdqu %%xtmp3, [%%src + %%i*16]
%else
%error too many i
% error
%endif
%assign %%i %%i+1
%endrep
%assign %%i 0
%rep (%%SIZE - 4 * %%SIZE4)
%if (%%i == 0)
movdqa [%%dst + %%i*16], %%xtmp0
%elif (%%i == 1)
movdqa [%%dst + %%i*16], %%xtmp1
%elif (%%i == 2)
movdqa [%%dst + %%i*16], %%xtmp2
%elif (%%i == 3)
movdqa [%%dst + %%i*16], %%xtmp3
%else
%error too many i
% error
%endif
%assign %%i %%i+1
%endrep
%endm
%endif
View Git Blame Copy Permalink