yaole
599 lines
20 KiB
ArmAsm
599 lines
20 KiB
ArmAsm
#include "textflag.h"
|
||
|
||
// SM3 block routine.
|
||
//
|
||
// No "function stitching" now.
|
||
//
|
||
// The algorithm is detailed in GB/T 32905-2016
|
||
|
||
// FFt(x,y,z) = GGt(x,y,z) = Parity(x,y,z) for 0 <= t <= 15
|
||
// FFt(x,y,z) = Maj(x,y,z) for 15 <= t <= 63
|
||
// GGt(x,y,z) = Ch(x,y,z) for 15 <= t <= 63
|
||
//
|
||
// Wt = Mt; for 0 <= t <= 15
|
||
// Wt = P1(Wt-16 xor Wt-9 xor ROTL(Wt-3,15))
|
||
// xor ROTL(Wi-13, 7) xor Wt-6 for 16 <= t <= 67
|
||
// W't = Wt xor Wt+4 for 0 <= t <= 63.
|
||
//
|
||
// a = V0
|
||
// b = V1
|
||
// c = V2
|
||
// d = V3
|
||
// e = V4
|
||
// f = V5
|
||
// g = V6
|
||
// h = V7
|
||
//
|
||
// for t = 0 to 63 {
|
||
// SS1 = ROTL(ROTL(a,12) + E + ROTL(Tt, t mod 32), 7)
|
||
// SS2 = SS1 xor ROTL(a,12)
|
||
// TT1 = FFt(a,b,c) + D + SS2 +W't
|
||
// TT2 = GGt(e,f,g) + h + SS1 +Wt
|
||
// d = c
|
||
// c = ROTL(b,9)
|
||
// b = a
|
||
// a = TT1
|
||
// h = g
|
||
// g = ROTL(f,19)
|
||
// f = e
|
||
// e = P0(TT2)
|
||
// }
|
||
//
|
||
// V0 = a xor V0
|
||
// V1 = b xor V1
|
||
// V2 = c xor V2
|
||
// V3 = d xor V3
|
||
// V4 = e xor V4
|
||
// V5 = f xor V5
|
||
// V6 = g xor V6
|
||
// V7 = h xor V7
|
||
|
||
// Definitions for AVX2 version
|
||
|
||
// xorm (mem), reg
|
||
// Xor reg to mem using reg-mem xor and store
|
||
#define xorm(P1, P2) \
|
||
XORL P2, P1; \
|
||
MOVL P1, P2
|
||
|
||
#define XDWORD0 Y4
|
||
#define XDWORD1 Y5
|
||
#define XDWORD2 Y6
|
||
#define XDWORD3 Y7
|
||
|
||
#define XWORD0 X4
|
||
#define XWORD1 X5
|
||
#define XWORD2 X6
|
||
#define XWORD3 X7
|
||
|
||
#define XTMP0 Y0
|
||
#define XTMP1 Y1
|
||
#define XTMP2 Y2
|
||
#define XTMP3 Y3
|
||
#define XTMP4 Y8
|
||
#define XTMP5 Y11
|
||
|
||
#define XFER Y9
|
||
|
||
#define BYTE_FLIP_MASK Y13 // mask to convert LE -> BE
|
||
#define X_BYTE_FLIP_MASK X13
|
||
|
||
#define NUM_BYTES DX
|
||
#define INP DI
|
||
|
||
#define CTX SI // Beginning of digest in memory (a, b, c, ... , h)
|
||
|
||
#define a AX
|
||
#define b BX
|
||
#define c CX
|
||
#define d R8
|
||
#define e DX
|
||
#define f R9
|
||
#define g R10
|
||
#define h R11
|
||
|
||
#define old_h R11
|
||
|
||
#define TBL BP
|
||
|
||
#define SRND SI // SRND is same register as CTX
|
||
|
||
#define T1 R12
|
||
|
||
#define y0 R13
|
||
#define y1 R14
|
||
#define y2 R15
|
||
#define y3 DI
|
||
|
||
// Offsets
|
||
#define XFER_SIZE 2*2*68*4
|
||
#define INP_END_SIZE 8
|
||
#define INP_SIZE 8
|
||
|
||
#define _XFER 0
|
||
#define _INP_END _XFER + XFER_SIZE
|
||
#define _INP _INP_END + INP_END_SIZE
|
||
#define STACK_SIZE _INP + INP_SIZE
|
||
|
||
|
||
// update XDWORD0
|
||
#define SM3_SCHED(XDWORD0, XDWORD1, XDWORD2, XDWORD3)\
|
||
; \ // ################################### Message Schedule ###########################
|
||
VPALIGNR $8, XDWORD2, XDWORD3, XTMP0; \ // XTMP0 = W[-6]
|
||
VPALIGNR $12, XDWORD0, XDWORD1, XTMP1; \ // XTMP1 = W[-13]
|
||
VPSLLD $7, XTMP1, XTMP2; \
|
||
VPSRLD $(32-7), XTMP1, XTMP1; \
|
||
VPXOR XTMP2, XTMP1, XTMP1; \ // XTMP1 = (w[-13] <<< 7)
|
||
VPXOR XTMP1, XTMP0, XTMP0; \ // XTMP0 = (w[-13] <<< 7) ^ w[-6]
|
||
; \
|
||
VPALIGNR $12, XDWORD1, XDWORD2, XTMP1; \ // XTMP1 = W[-9]
|
||
VPXOR XDWORD0, XTMP1, XTMP1; \ // XTMP1 = W[-9]^W[-16]
|
||
; \
|
||
VPSHUFD $0xA5, XDWORD3, XTMP2; \ // XTMP2 = W[-3] {BBAA}
|
||
VPSRLQ $17, XTMP2, XTMP4; \ // XTMP4 = W[-3] <<< 15 {xBxA}
|
||
VPSHUFB shuff_00BA<>(SB), XTMP4, XTMP4; \ // XTMP4 = W[-3] <<< 15 {00BA}
|
||
VPXOR XTMP1, XTMP4, XTMP4; \ // XTMP4 = W[-9]^W[-16] ^ (W[-3] <<< 15) {xxBA}
|
||
VPSHUFD $0x50, XTMP4, XTMP4; \ // XTMP4 = W[-9]^W[-16] ^ (W[-3] <<< 15) {BBAA}
|
||
VPSRLQ $17, XTMP4,XTMP2; \ // {xBxA}
|
||
VPSRLQ $9, XTMP4,XTMP3; \ // {xBxA}
|
||
VPXOR XTMP2, XTMP4, XTMP4; \ //
|
||
VPXOR XTMP3, XTMP4, XTMP4; \ // XTMP4 = p1 {xBxA}
|
||
VPSHUFB shuff_00BA<>(SB), XTMP4, XTMP4; \ // XTMP4 = p1 {00BA}
|
||
VPXOR XTMP4, XTMP0, XTMP5; \ // XTMP5 = {..., ..., W[1], W[0]}
|
||
; \
|
||
VPALIGNR $4, XDWORD3, XTMP5, XTMP2; \ // XTMP2 = {W[0], W[-1], W[-2], W[-3]}
|
||
VPSHUFD $0xFA, XTMP2, XTMP2; \ // XTMP2 = {W[0], W[0], W[-1], W[-1]} {DDCC}
|
||
VPSRLQ $17, XTMP2, XTMP4; \ // XTMP4 = W[-3] <<< 15 {xDxC}
|
||
VPSHUFB shuff_DC00<>(SB), XTMP4, XTMP4; \ // XTMP4 = W[-3] <<< 15 {DC00}
|
||
VPXOR XTMP1, XTMP4, XTMP4; \ // XTMP4 = W[-9]^W[-16] ^ (W[-3] <<< 15) {DCxx}
|
||
VPSHUFD $0xFA, XTMP4, XTMP4; \ // XTMP4 = W[-9]^W[-16] ^ (W[-3] <<< 15) {DDCC}
|
||
VPSRLQ $17, XTMP4,XTMP2; \
|
||
VPSRLQ $9, XTMP4,XTMP3; \
|
||
VPXOR XTMP2, XTMP4, XTMP4; \
|
||
VPXOR XTMP3, XTMP4, XTMP4; \ // XTMP4 = p1 {xDxC}
|
||
VPSHUFB shuff_DC00<>(SB), XTMP4, XTMP4; \ // XTMP4 = p1 {DC00}
|
||
VPXOR XTMP4, XTMP5,XDWORD0
|
||
|
||
|
||
// 前16轮计算-0
|
||
// SRND = # of {4 round}
|
||
// T is saved in (dist_T)(TBL)(SRND*1)
|
||
// W is saved in (dist_W)(SP)(SRND*2)
|
||
// W' is saved in (dist_W + 68*4*2)(SP)(SRND*1)
|
||
#define DO_ROUND16(dist_T, dist_W, a, b, c, d, e, f, g, h) \
|
||
; \ // ################################### RND 0 - 15 ###########################
|
||
RORXL $(32-12), a, y2; \ // y2 = a <<< 12
|
||
MOVL y2, y3; \
|
||
ADDL e, y3; \ // y3 = (a <<< 12) + e
|
||
ADDL (dist_T)(TBL)(SRND*1), y3; \ // y3 = (a <<< 12) + e + T
|
||
RORXL $(32-7), y3, y3; \ // y3 = ss1
|
||
ADDL y3, h; \ // h = h+ss1
|
||
XORL y2, y3; \ // y3 = ss2
|
||
ADDL y3, d; \ // d = d+ss2
|
||
; \
|
||
ADDL (dist_W)(SP)(SRND*2), h; \ // h = h + ss1 + w
|
||
ADDL (dist_W+68*4*2)(SP)(SRND*2), d; \ // d = d + ss2 + w'
|
||
; \
|
||
MOVL a, y1; \ // y1 = a //FF //PARITY
|
||
XORL b, y1; \ // y1 = a^b //FF //PARITY
|
||
XORL c, y1; \ // y1 = a^b^c //FF //PARITY
|
||
ADDL y1, d; \ // d = TT1
|
||
; \
|
||
MOVL e, y2; \ // y2 = e // GG //PARITY
|
||
XORL f, y2; \ // y2 = e^f // GG //PARITY
|
||
XORL g, y2; \ // y2 = e^f^g // GG //PARITY
|
||
ADDL y2, h; \ // h = TT2
|
||
; \
|
||
RORXL $(32-8), h, y0; \ // y0 = TT2<<<8
|
||
XORL h,y0; \ // y0 = (TT2<<<8)^TT2
|
||
RORXL $(32-9), y0, y0; \ // y0 = ((TT2<<<8)^TT2)<<<9
|
||
XORL y0, h; \ // h = p0(TT2)
|
||
; \
|
||
RORXL $(32-9), b, b; \
|
||
RORXL $(32-19), f, f
|
||
|
||
// 后48轮计算
|
||
#define DO_ROUND48(dist_T, dist_W, a, b, c, d, e, f, g, h) \
|
||
; \ // ################################### RND 16 - 63 ###########################
|
||
RORXL $(32-12), a, y2; \ // y2 = a <<< 12
|
||
MOVL y2, y3; \
|
||
ADDL e, y3; \ // y3 = (a <<< 12) + e
|
||
ADDL (dist_T)(TBL)(SRND*1), y3; \ // y3 = (a <<< 12) + e + T
|
||
RORXL $(32-7), y3, y3; \ // y3 = ss1
|
||
ADDL y3, h; \ // h = h+ss1
|
||
XORL y2, y3; \ // y3 = ss2
|
||
ADDL y3, d; \ // d = d+ss2
|
||
; \
|
||
ADDL (dist_W)(SP)(SRND*2), h; \ // h = h + ss1 + w
|
||
ADDL (dist_W+68*4*2)(SP)(SRND*2), d; \ // d = d + ss2 + w'
|
||
; \
|
||
MOVL a, y3; \ // y3 = a //FF MAJA
|
||
ORL c, y3; \ // y3 = a|c // MAJA
|
||
ANDL b, y3; \ // y3 = (a|c)&b // MAJA
|
||
MOVL a, T1; \ // T1 = a // MAJB
|
||
ANDL c, T1; \ // T1 = a&c // MAJB
|
||
ORL T1, y3; \ // y3 = MAJ = ((a|c)&b)|(a&c) // MAJ
|
||
ADDL y3, d; \ // d = TT1
|
||
; \
|
||
MOVL f, y2; \ // y2 = f //GG CH
|
||
XORL g, y2; \ // y2 = f^g // CH
|
||
ANDL e, y2; \ // y2 = (f^g)&e // CH
|
||
XORL g, y2; \ // y2 = CH = ((f^g)&e)^g // CH
|
||
ADDL y2, h; \ // h = TT2
|
||
; \
|
||
RORXL $(32-8), h, y0; \ // y0 = TT2<<<8
|
||
XORL h,y0; \ // y0 = (TT2<<<8)^TT2
|
||
RORXL $(32-9), y0, y0; \ // y0 = ((TT2<<<8)^TT2)<<<9
|
||
XORL y0, h; \ // h = p0(TT2)
|
||
; \
|
||
RORXL $(32-9), b, b; \
|
||
RORXL $(32-19), f, f
|
||
|
||
|
||
// stack:
|
||
// block0 block1
|
||
// 0*8: W[0:4] V[0:4]
|
||
// 1*8: W[4:8] V[4:8]
|
||
// ...
|
||
// 67*8: W[64:68] V[64:68]
|
||
// 68*8: W'[0:4] V'[0:4]
|
||
// 69*8: W'[4:8] V'[4:8]
|
||
// ...
|
||
// 135*8: W'[64:68] V'[64:68]
|
||
// 136*8: _INP_END(SP) - Pointer to the last block
|
||
// 137*8: _INP(SP) - Save INP in round computation.
|
||
//
|
||
// STACK_SIZE = 1088+8+8 = 1104
|
||
// func blockAsmAVX2(dig *digest, p []byte)
|
||
TEXT ·blockAsmAVX2(SB), 0, $1104-32
|
||
MOVQ dig+0(FP), CTX
|
||
MOVQ p_base+8(FP), INP
|
||
MOVQ p_len+16(FP), NUM_BYTES
|
||
|
||
LEAQ -64(INP)(NUM_BYTES*1), NUM_BYTES // Pointer to the last block
|
||
MOVQ NUM_BYTES, _INP_END(SP) // save to stack
|
||
|
||
CMPQ NUM_BYTES, INP
|
||
JE avx2_only_one_block
|
||
|
||
// Load initial digest
|
||
MOVL 0(CTX), a // a = H0
|
||
MOVL 4(CTX), b // b = H1
|
||
MOVL 8(CTX), c // c = H2
|
||
MOVL 12(CTX), d // d = H3
|
||
MOVL 16(CTX), e // e = H4
|
||
MOVL 20(CTX), f // f = H5
|
||
MOVL 24(CTX), g // g = H6
|
||
MOVL 28(CTX), h // h = H7
|
||
|
||
avx2_loop0: // at each iteration works with one block (512 bit)
|
||
|
||
// load two blocks，64*2 bytes
|
||
VMOVDQU (0*32)(INP), XTMP0 // p[0:4]
|
||
VMOVDQU (1*32)(INP), XTMP1
|
||
VMOVDQU (2*32)(INP), XTMP2
|
||
VMOVDQU (3*32)(INP), XTMP3
|
||
|
||
VMOVDQU flip_mask<>(SB), BYTE_FLIP_MASK
|
||
|
||
// Apply Byte Flip Mask: LE -> BE
|
||
VPSHUFB BYTE_FLIP_MASK, XTMP0, XTMP0
|
||
VPSHUFB BYTE_FLIP_MASK, XTMP1, XTMP1
|
||
VPSHUFB BYTE_FLIP_MASK, XTMP2, XTMP2
|
||
VPSHUFB BYTE_FLIP_MASK, XTMP3, XTMP3
|
||
|
||
// XTMP0:XTMP1 - first block
|
||
// XTMP0: w7, w6, w5, w4, w3, w2, w1,w0
|
||
// XTMP1: w15,w14,w13,w12,w11,w10,w9,w8
|
||
// XTMP3:XTMP2 - second block
|
||
// XTMP2: u7, u6, u5, u4, u3, u2, u1,u0
|
||
// XTMP3: u15,u14,u13,u12,u11,u10,u9,u8
|
||
|
||
// XDWORD0: u3, u2, u1, u0, w3, w2, w1,w0
|
||
// XDWORD1: u7, u6, u5, u4, w7, w6, w5, w4
|
||
// XDWORD2: u11,u10,u9, u8, w11,w10,w9,w8
|
||
// XDWORD3: u15,u14,u13,u12,w15,w14,w13,w12
|
||
|
||
// Transpose data into high/low parts
|
||
VPERM2I128 $0x20, XTMP2, XTMP0, XDWORD0 // w3, w2, w1, w0
|
||
VPERM2I128 $0x31, XTMP2, XTMP0, XDWORD1 // w7, w6, w5, w4
|
||
VPERM2I128 $0x20, XTMP3, XTMP1, XDWORD2 // w11, w10, w9, w8
|
||
VPERM2I128 $0x31, XTMP3, XTMP1, XDWORD3 // w15, w14, w13, w12
|
||
MOVQ $T256<>(SB), TBL // Loading address of table with round-specific constants
|
||
|
||
avx2_last_block_enter:
|
||
ADDQ $64, INP
|
||
MOVQ INP, _INP(SP)
|
||
XORQ SRND, SRND // SRND increace 16 of each 4 rounds (dist of T)
|
||
|
||
// for w0 - w15
|
||
// Do 4 rounds and scheduling
|
||
VMOVDQU XDWORD0, (_XFER + 0*32)(SP)(SRND*2)
|
||
VPXOR XDWORD0, XDWORD1, XFER
|
||
VMOVDQU XFER, (_XFER + 0*32+68*4*2)(SP)(SRND*2)
|
||
SM3_SCHED(XDWORD0, XDWORD1, XDWORD2, XDWORD3)
|
||
DO_ROUND16(0*4, 0*32+0, a, b, c, d, e, f, g, h)
|
||
DO_ROUND16(1*4, 0*32+4, d, a, b, c, h, e, f, g)
|
||
DO_ROUND16(2*4, 0*32+8, c, d, a, b, g, h, e, f)
|
||
DO_ROUND16(3*4, 0*32+12, b, c, d, a, f, g, h, e)
|
||
|
||
VMOVDQU XDWORD1, (_XFER + 1*32)(SP)(SRND*2)
|
||
VPXOR XDWORD1, XDWORD2, XFER
|
||
VMOVDQU XFER, (_XFER + 1*32+68*4*2)(SP)(SRND*2)
|
||
SM3_SCHED(XDWORD1, XDWORD2, XDWORD3, XDWORD0)
|
||
DO_ROUND16(4*4, 1*32+0, a, b, c, d, e, f, g, h)
|
||
DO_ROUND16(5*4, 1*32+4, d, a, b, c, h, e, f, g)
|
||
DO_ROUND16(6*4, 1*32+8, c, d, a, b, g, h, e, f)
|
||
DO_ROUND16(7*4, 1*32+12, b, c, d, a, f, g, h, e)
|
||
|
||
VMOVDQU XDWORD2, (_XFER + 2*32)(SP)(SRND*2)
|
||
VPXOR XDWORD2, XDWORD3, XFER
|
||
VMOVDQU XFER, (_XFER + 2*32+68*4*2)(SP)(SRND*2)
|
||
SM3_SCHED(XDWORD2, XDWORD3, XDWORD0, XDWORD1)
|
||
DO_ROUND16(8*4, 2*32+0, a, b, c, d, e, f, g, h)
|
||
DO_ROUND16(9*4, 2*32+4, d, a, b, c, h, e, f, g)
|
||
DO_ROUND16(10*4,2*32+8, c, d, a, b, g, h, e, f)
|
||
DO_ROUND16(11*4,2*32+12, b, c, d, a, f, g, h, e)
|
||
|
||
VMOVDQU XDWORD3, (_XFER + 3*32)(SP)(SRND*2)
|
||
VPXOR XDWORD3, XDWORD0, XFER
|
||
VMOVDQU XFER, (_XFER + 3*32+68*4*2)(SP)(SRND*2)
|
||
SM3_SCHED(XDWORD3, XDWORD0, XDWORD1, XDWORD2)
|
||
DO_ROUND16(12*4, 3*32+0, a, b, c, d, e, f, g, h)
|
||
DO_ROUND16(13*4, 3*32+4, d, a, b, c, h, e, f, g)
|
||
DO_ROUND16(14*4, 3*32+8, c, d, a, b, g, h, e, f)
|
||
DO_ROUND16(15*4, 3*32+12, b, c, d, a, f, g, h, e)
|
||
|
||
ADDQ $4*16, SRND
|
||
|
||
avx2_loop1:
|
||
// for w16 - w47 with scheduling (32 rounds)
|
||
VMOVDQU XDWORD0, (_XFER + 0*32)(SP)(SRND*2)
|
||
VPXOR XDWORD0, XDWORD1, XFER
|
||
VMOVDQU XFER, (_XFER + 0*32+68*4*2)(SP)(SRND*2)
|
||
SM3_SCHED(XDWORD0, XDWORD1, XDWORD2, XDWORD3)
|
||
DO_ROUND48(0*4, 0*32+0, a, b, c, d, e, f, g, h)
|
||
DO_ROUND48(1*4, 0*32+4, d, a, b, c, h, e, f, g)
|
||
DO_ROUND48(2*4, 0*32+8, c, d, a, b, g, h, e, f)
|
||
DO_ROUND48(3*4, 0*32+12, b, c, d, a, f, g, h, e)
|
||
|
||
VMOVDQU XDWORD1, (_XFER + 1*32)(SP)(SRND*2)
|
||
VPXOR XDWORD1, XDWORD2, XFER
|
||
VMOVDQU XFER, (_XFER + 1*32+68*4*2)(SP)(SRND*2)
|
||
SM3_SCHED(XDWORD1, XDWORD2, XDWORD3, XDWORD0)
|
||
DO_ROUND48(4*4, 1*32+0, a, b, c, d, e, f, g, h)
|
||
DO_ROUND48(5*4, 1*32+4, d, a, b, c, h, e, f, g)
|
||
DO_ROUND48(6*4, 1*32+8, c, d, a, b, g, h, e, f)
|
||
DO_ROUND48(7*4, 1*32+12, b, c, d, a, f, g, h, e)
|
||
|
||
VMOVDQU XDWORD2, (_XFER + 2*32)(SP)(SRND*2)
|
||
VPXOR XDWORD2, XDWORD3, XFER
|
||
VMOVDQU XFER, (_XFER + 2*32+68*4*2)(SP)(SRND*2)
|
||
SM3_SCHED(XDWORD2, XDWORD3, XDWORD0, XDWORD1)
|
||
DO_ROUND48(8*4, 2*32+0, a, b, c, d, e, f, g, h)
|
||
DO_ROUND48(9*4, 2*32+4, d, a, b, c, h, e, f, g)
|
||
DO_ROUND48(10*4,2*32+8, c, d, a, b, g, h, e, f)
|
||
DO_ROUND48(11*4,2*32+12, b, c, d, a, f, g, h, e)
|
||
|
||
VMOVDQU XDWORD3, (_XFER + 3*32)(SP)(SRND*2)
|
||
VPXOR XDWORD3, XDWORD0, XFER
|
||
VMOVDQU XFER, (_XFER + 3*32+68*4*2)(SP)(SRND*2)
|
||
SM3_SCHED(XDWORD3, XDWORD0, XDWORD1, XDWORD2)
|
||
DO_ROUND48(12*4,3*32+0, a, b, c, d, e, f, g, h)
|
||
DO_ROUND48(13*4,3*32+4, d, a, b, c, h, e, f, g)
|
||
DO_ROUND48(14*4,3*32+8, c, d, a, b, g, h, e, f)
|
||
DO_ROUND48(15*4,3*32+12, b, c, d, a, f, g, h, e)
|
||
|
||
ADDQ $4*16, SRND
|
||
CMPQ SRND, $12*16
|
||
JB avx2_loop1
|
||
|
||
// w48 - w63 processed with one scheduling (last 16 rounds)
|
||
VMOVDQU XDWORD0, (_XFER + 0*32)(SP)(SRND*2)
|
||
VPXOR XDWORD0, XDWORD1, XFER
|
||
VMOVDQU XFER, (_XFER + 0*32+68*4*2)(SP)(SRND*2)
|
||
SM3_SCHED(XDWORD0, XDWORD1, XDWORD2, XDWORD3) // scheduling XDWORD0 for W64-W67
|
||
DO_ROUND48(0*4, 0*32+0, a, b, c, d, e, f, g, h)
|
||
DO_ROUND48(1*4, 0*32+4, d, a, b, c, h, e, f, g)
|
||
DO_ROUND48(2*4, 0*32+8, c, d, a, b, g, h, e, f)
|
||
DO_ROUND48(3*4, 0*32+12, b, c, d, a, f, g, h, e)
|
||
|
||
VMOVDQU XDWORD1, (_XFER + 1*32)(SP)(SRND*2)
|
||
VPXOR XDWORD1, XDWORD2, XFER
|
||
VMOVDQU XFER, (_XFER + 1*32+68*4*2)(SP)(SRND*2)
|
||
DO_ROUND48(4*4, 1*32+0, a, b, c, d, e, f, g, h)
|
||
DO_ROUND48(5*4, 1*32+4, d, a, b, c, h, e, f, g)
|
||
DO_ROUND48(6*4, 1*32+8, c, d, a, b, g, h, e, f)
|
||
DO_ROUND48(7*4, 1*32+12, b, c, d, a, f, g, h, e)
|
||
|
||
VMOVDQU XDWORD2, (_XFER + 2*32)(SP)(SRND*2)
|
||
VPXOR XDWORD2, XDWORD3, XFER
|
||
VMOVDQU XFER, (_XFER + 2*32+68*4*2)(SP)(SRND*2)
|
||
DO_ROUND48(8*4, 2*32+0, a, b, c, d, e, f, g, h)
|
||
DO_ROUND48(9*4, 2*32+4, d, a, b, c, h, e, f, g)
|
||
DO_ROUND48(10*4,2*32+8, c, d, a, b, g, h, e, f)
|
||
DO_ROUND48(11*4,2*32+12, b, c, d, a, f, g, h, e)
|
||
|
||
VMOVDQU XDWORD3, (_XFER + 3*32)(SP)(SRND*2)
|
||
VPXOR XDWORD3, XDWORD0, XFER
|
||
VMOVDQU XFER, (_XFER + 3*32+68*4*2)(SP)(SRND*2)
|
||
DO_ROUND48(12*4, 3*32+0, a, b, c, d, e, f, g, h)
|
||
DO_ROUND48(13*4, 3*32+4, d, a, b, c, h, e, f, g)
|
||
DO_ROUND48(14*4, 3*32+8, c, d, a, b, g, h, e, f)
|
||
DO_ROUND48(15*4, 3*32+12, b, c, d, a, f, g, h, e)
|
||
|
||
MOVQ dig+0(FP), CTX // d.h[8]
|
||
MOVQ _INP(SP), INP
|
||
|
||
xorm( 0(CTX), a)
|
||
xorm( 4(CTX), b)
|
||
xorm( 8(CTX), c)
|
||
xorm( 12(CTX), d)
|
||
xorm( 16(CTX), e)
|
||
xorm( 20(CTX), f)
|
||
xorm( 24(CTX), g)
|
||
xorm( 28(CTX), h)
|
||
|
||
CMPQ _INP_END(SP), INP
|
||
JB done_hash
|
||
|
||
XORQ SRND, SRND
|
||
|
||
avx2_loop3: // Do second block using previously scheduled results
|
||
DO_ROUND16(0, 16+0, a, b, c, d, e, f, g, h)
|
||
DO_ROUND16(4, 16+4, d, a, b, c, h, e, f, g)
|
||
DO_ROUND16(8, 16+8, c, d, a, b, g, h, e, f)
|
||
DO_ROUND16(12, 16+12, b, c, d, a, f, g, h, e)
|
||
|
||
ADDQ $16, SRND
|
||
CMPQ SRND, $4*16
|
||
JB avx2_loop3
|
||
avx2_loop4:
|
||
DO_ROUND48(0, 16+0,a, b, c, d, e, f, g, h)
|
||
DO_ROUND48(4, 16+4,d, a, b, c, h, e, f, g)
|
||
DO_ROUND48(8, 16+8,c, d, a, b, g, h, e, f)
|
||
DO_ROUND48(12, 16+12, b, c, d, a, f, g, h, e)
|
||
|
||
ADDQ $16, SRND
|
||
CMPQ SRND, $16*16
|
||
JB avx2_loop4
|
||
|
||
MOVQ dig+0(FP), CTX // d.h[8]
|
||
MOVQ _INP(SP), INP
|
||
ADDQ $64, INP
|
||
|
||
xorm( 0(CTX), a)
|
||
xorm( 4(CTX), b)
|
||
xorm( 8(CTX), c)
|
||
xorm( 12(CTX), d)
|
||
xorm( 16(CTX), e)
|
||
xorm( 20(CTX), f)
|
||
xorm( 24(CTX), g)
|
||
xorm( 28(CTX), h)
|
||
|
||
CMPQ _INP_END(SP), INP
|
||
JA avx2_loop0
|
||
JB done_hash
|
||
|
||
avx2_do_last_block:
|
||
|
||
VMOVDQU 0(INP), XWORD0
|
||
VMOVDQU 16(INP), XWORD1
|
||
VMOVDQU 32(INP), XWORD2
|
||
VMOVDQU 48(INP), XWORD3
|
||
|
||
VMOVDQU flip_mask<>(SB), BYTE_FLIP_MASK
|
||
|
||
VPSHUFB X_BYTE_FLIP_MASK, XWORD0, XWORD0
|
||
VPSHUFB X_BYTE_FLIP_MASK, XWORD1, XWORD1
|
||
VPSHUFB X_BYTE_FLIP_MASK, XWORD2, XWORD2
|
||
VPSHUFB X_BYTE_FLIP_MASK, XWORD3, XWORD3
|
||
MOVQ $T256<>(SB), TBL
|
||
|
||
JMP avx2_last_block_enter
|
||
|
||
avx2_only_one_block:
|
||
// Load initial digest
|
||
MOVL 0(CTX), a // a = H0
|
||
MOVL 4(CTX), b // b = H1
|
||
MOVL 8(CTX), c // c = H2
|
||
MOVL 12(CTX), d // d = H3
|
||
MOVL 16(CTX), e // e = H4
|
||
MOVL 20(CTX), f // f = H5
|
||
MOVL 24(CTX), g // g = H6
|
||
MOVL 28(CTX), h // h = H7
|
||
|
||
JMP avx2_do_last_block
|
||
|
||
done_hash:
|
||
VZEROUPPER
|
||
RET
|
||
|
||
// shuffle byte order from LE to BE
|
||
DATA flip_mask<>+0x00(SB)/8, $0x0405060700010203
|
||
DATA flip_mask<>+0x08(SB)/8, $0x0c0d0e0f08090a0b
|
||
DATA flip_mask<>+0x10(SB)/8, $0x0405060700010203
|
||
DATA flip_mask<>+0x18(SB)/8, $0x0c0d0e0f08090a0b
|
||
GLOBL flip_mask<>(SB), 8, $32
|
||
|
||
// shuffle xBxA -> 00BA
|
||
DATA shuff_00BA<>+0x00(SB)/8, $0x0b0a090803020100
|
||
DATA shuff_00BA<>+0x08(SB)/8, $0xFFFFFFFFFFFFFFFF
|
||
DATA shuff_00BA<>+0x10(SB)/8, $0x0b0a090803020100
|
||
DATA shuff_00BA<>+0x18(SB)/8, $0xFFFFFFFFFFFFFFFF
|
||
GLOBL shuff_00BA<>(SB), 8, $32
|
||
|
||
// shuffle xDxC -> DC00
|
||
DATA shuff_DC00<>+0x00(SB)/8, $0xFFFFFFFFFFFFFFFF
|
||
DATA shuff_DC00<>+0x08(SB)/8, $0x0b0a090803020100
|
||
DATA shuff_DC00<>+0x10(SB)/8, $0xFFFFFFFFFFFFFFFF
|
||
DATA shuff_DC00<>+0x18(SB)/8, $0x0b0a090803020100
|
||
GLOBL shuff_DC00<>(SB), 8, $32
|
||
|
||
// rotate of Tj: rotT[i] = Tj << (j mod 32)
|
||
DATA T256<>+0x0(SB)/4, $0x79cc4519
|
||
DATA T256<>+0x4(SB)/4, $0xf3988a32
|
||
DATA T256<>+0x8(SB)/4, $0xe7311465
|
||
DATA T256<>+0xc(SB)/4, $0xce6228cb
|
||
DATA T256<>+0x10(SB)/4, $0x9cc45197
|
||
DATA T256<>+0x14(SB)/4, $0x3988a32f
|
||
DATA T256<>+0x18(SB)/4, $0x7311465e
|
||
DATA T256<>+0x1c(SB)/4, $0xe6228cbc
|
||
DATA T256<>+0x20(SB)/4, $0xcc451979
|
||
DATA T256<>+0x24(SB)/4, $0x988a32f3
|
||
DATA T256<>+0x28(SB)/4, $0x311465e7
|
||
DATA T256<>+0x2c(SB)/4, $0x6228cbce
|
||
DATA T256<>+0x30(SB)/4, $0xc451979c
|
||
DATA T256<>+0x34(SB)/4, $0x88a32f39
|
||
DATA T256<>+0x38(SB)/4, $0x11465e73
|
||
DATA T256<>+0x3c(SB)/4, $0x228cbce6
|
||
DATA T256<>+0x40(SB)/4, $0x9d8a7a87
|
||
DATA T256<>+0x44(SB)/4, $0x3b14f50f
|
||
DATA T256<>+0x48(SB)/4, $0x7629ea1e
|
||
DATA T256<>+0x4c(SB)/4, $0xec53d43c
|
||
DATA T256<>+0x50(SB)/4, $0xd8a7a879
|
||
DATA T256<>+0x54(SB)/4, $0xb14f50f3
|
||
DATA T256<>+0x58(SB)/4, $0x629ea1e7
|
||
DATA T256<>+0x5c(SB)/4, $0xc53d43ce
|
||
DATA T256<>+0x60(SB)/4, $0x8a7a879d
|
||
DATA T256<>+0x64(SB)/4, $0x14f50f3b
|
||
DATA T256<>+0x68(SB)/4, $0x29ea1e76
|
||
DATA T256<>+0x6c(SB)/4, $0x53d43cec
|
||
DATA T256<>+0x70(SB)/4, $0xa7a879d8
|
||
DATA T256<>+0x74(SB)/4, $0x4f50f3b1
|
||
DATA T256<>+0x78(SB)/4, $0x9ea1e762
|
||
DATA T256<>+0x7c(SB)/4, $0x3d43cec5
|
||
DATA T256<>+0x80(SB)/4, $0x7a879d8a
|
||
DATA T256<>+0x84(SB)/4, $0xf50f3b14
|
||
DATA T256<>+0x88(SB)/4, $0xea1e7629
|
||
DATA T256<>+0x8c(SB)/4, $0xd43cec53
|
||
DATA T256<>+0x90(SB)/4, $0xa879d8a7
|
||
DATA T256<>+0x94(SB)/4, $0x50f3b14f
|
||
DATA T256<>+0x98(SB)/4, $0xa1e7629e
|
||
DATA T256<>+0x9c(SB)/4, $0x43cec53d
|
||
DATA T256<>+0xa0(SB)/4, $0x879d8a7a
|
||
DATA T256<>+0xa4(SB)/4, $0x0f3b14f5
|
||
DATA T256<>+0xa8(SB)/4, $0x1e7629ea
|
||
DATA T256<>+0xac(SB)/4, $0x3cec53d4
|
||
DATA T256<>+0xb0(SB)/4, $0x79d8a7a8
|
||
DATA T256<>+0xb4(SB)/4, $0xf3b14f50
|
||
DATA T256<>+0xb8(SB)/4, $0xe7629ea1
|
||
DATA T256<>+0xbc(SB)/4, $0xcec53d43
|
||
DATA T256<>+0xc0(SB)/4, $0x9d8a7a87
|
||
DATA T256<>+0xc4(SB)/4, $0x3b14f50f
|
||
DATA T256<>+0xc8(SB)/4, $0x7629ea1e
|
||
DATA T256<>+0xcc(SB)/4, $0xec53d43c
|
||
DATA T256<>+0xd0(SB)/4, $0xd8a7a879
|
||
DATA T256<>+0xd4(SB)/4, $0xb14f50f3
|
||
DATA T256<>+0xd8(SB)/4, $0x629ea1e7
|
||
DATA T256<>+0xdc(SB)/4, $0xc53d43ce
|
||
DATA T256<>+0xe0(SB)/4, $0x8a7a879d
|
||
DATA T256<>+0xe4(SB)/4, $0x14f50f3b
|
||
DATA T256<>+0xe8(SB)/4, $0x29ea1e76
|
||
DATA T256<>+0xec(SB)/4, $0x53d43cec
|
||
DATA T256<>+0xf0(SB)/4, $0xa7a879d8
|
||
DATA T256<>+0xf4(SB)/4, $0x4f50f3b1
|
||
DATA T256<>+0xf8(SB)/4, $0x9ea1e762
|
||
DATA T256<>+0xfc(SB)/4, $0x3d43cec5
|
||
GLOBL T256<>(SB), (NOPTR + RODATA), $256
|