#include "textflag.h"

// SM3 block routine.
//
// No "function stitching" now.
// 
// The algorithm is detailed in GB/T 32905-2016

// FFt(x,y,z) = GGt(x,y,z) = Parity(x,y,z) for 0 <= t <= 15
// FFt(x,y,z) = Maj(x,y,z) for 15 <= t <= 63
// GGt(x,y,z) = Ch(x,y,z)  for 15 <= t <= 63
// 
// Wt = Mt; for 0 <= t <= 15
// Wt = P1(Wt-16 xor Wt-9 xor ROTL(Wt-3,15))
//          xor ROTL(Wi-13, 7) xor Wt-6 for 16 <= t <= 67
// W't = Wt xor Wt+4  for 0 <= t <= 63.
//
// a = V0
// b = V1
// c = V2
// d = V3
// e = V4
// f = V5
// g = V6
// h = V7
//
// for t = 0 to 63 {
//    SS1 = ROTL(ROTL(a,12) + E + ROTL(Tt, t mod 32), 7)
//    SS2 = SS1 xor ROTL(a,12) 
//    TT1 = FFt(a,b,c) + D + SS2 +W't
//    TT2 = GGt(e,f,g) + h + SS1 +Wt
//    d = c
//    c = ROTL(b,9)
//    b = a
//    a = TT1
//    h = g
//    g = ROTL(f,19)
//    f = e
//    e = P0(TT2)
// }
//
// V0 = a xor V0
// V1 = b xor V1
// V2 = c xor V2
// V3 = d xor V3
// V4 = e xor V4
// V5 = f xor V5
// V6 = g xor V6
// V7 = h xor V7

// Definitions for AVX2 version

// xorm (mem), reg
// Xor reg to mem using reg-mem xor and store
#define xorm(P1, P2) \
	XORL P2, P1; \
	MOVL P1, P2

#define XDWORD0 Y4
#define XDWORD1 Y5
#define XDWORD2 Y6
#define XDWORD3 Y7

#define XWORD0 X4
#define XWORD1 X5
#define XWORD2 X6
#define XWORD3 X7

#define XTMP0 Y0
#define XTMP1 Y1
#define XTMP2 Y2
#define XTMP3 Y3
#define XTMP4 Y8
#define XTMP5 Y11

#define XFER  Y9

#define BYTE_FLIP_MASK 	Y13 // mask to convert LE -> BE
#define X_BYTE_FLIP_MASK X13

#define NUM_BYTES DX
#define INP	DI

#define CTX SI // Beginning of digest in memory (a, b, c, ... , h)

#define a AX
#define b BX
#define c CX
#define d R8
#define e DX
#define f R9
#define g R10
#define h R11

#define old_h R11

#define TBL BP

#define SRND SI // SRND is same register as CTX

#define T1 R12

#define y0 R13
#define y1 R14
#define y2 R15
#define y3 DI

// Offsets
#define XFER_SIZE 2*2*68*4
#define INP_END_SIZE 8
#define INP_SIZE 8

#define _XFER 0
#define _INP_END _XFER + XFER_SIZE
#define _INP _INP_END + INP_END_SIZE
#define STACK_SIZE _INP + INP_SIZE


// update XDWORD0
#define SM3_SCHED(XDWORD0, XDWORD1, XDWORD2, XDWORD3)\
	;                                       \ // ################################### Message Schedule ###########################
	VPALIGNR $8, XDWORD2, XDWORD3, XTMP0;   \ // XTMP0 = W[-6]
	VPALIGNR $12, XDWORD0, XDWORD1, XTMP1;  \ // XTMP1 = W[-13]
    VPSLLD   $7, XTMP1, XTMP2;              \
    VPSRLD   $(32-7), XTMP1, XTMP1;         \
    VPXOR    XTMP2, XTMP1, XTMP1;           \ // XTMP1 = (w[-13] <<< 7)
    VPXOR    XTMP1, XTMP0, XTMP0;           \ // XTMP0 = (w[-13] <<< 7) ^ w[-6]
    ;                                       \
	VPALIGNR $12, XDWORD1, XDWORD2, XTMP1;  \ // XTMP1 = W[-9]
    VPXOR   XDWORD0, XTMP1, XTMP1;          \ // XTMP1 = W[-9]^W[-16]
    ;                                       \
    VPSHUFD $0xA5, XDWORD3, XTMP2;          \ // XTMP2 = W[-3] {BBAA}
	VPSRLQ  $17, XTMP2, XTMP4;              \ // XTMP4 = W[-3] <<< 15 {xBxA}
    VPSHUFB shuff_00BA<>(SB), XTMP4, XTMP4; \ // XTMP4 = W[-3] <<< 15 {00BA}
    VPXOR   XTMP1, XTMP4, XTMP4;            \ // XTMP4 = W[-9]^W[-16] ^ (W[-3] <<< 15) {xxBA}
    VPSHUFD $0x50, XTMP4, XTMP4;            \ // XTMP4 = W[-9]^W[-16] ^ (W[-3] <<< 15) {BBAA}
    VPSRLQ  $17, XTMP4,XTMP2;               \ // {xBxA}
    VPSRLQ  $9, XTMP4,XTMP3;                \ // {xBxA}
    VPXOR   XTMP2, XTMP4, XTMP4;            \ //
    VPXOR   XTMP3, XTMP4, XTMP4;            \ // XTMP4 = p1 {xBxA}
    VPSHUFB shuff_00BA<>(SB), XTMP4, XTMP4; \ // XTMP4 = p1 {00BA}
    VPXOR   XTMP4, XTMP0, XTMP5;            \ // XTMP5 = {..., ..., W[1], W[0]}
    ;                                       \
    VPALIGNR $4, XDWORD3, XTMP5, XTMP2;     \ // XTMP2 = {W[0], W[-1], W[-2], W[-3]}
	VPSHUFD $0xFA, XTMP2, XTMP2;            \ // XTMP2 = {W[0], W[0], W[-1], W[-1]} {DDCC}
    VPSRLQ  $17, XTMP2, XTMP4;              \ // XTMP4 = W[-3] <<< 15 {xDxC}
    VPSHUFB shuff_DC00<>(SB), XTMP4, XTMP4; \ // XTMP4 = W[-3] <<< 15 {DC00}
    VPXOR   XTMP1, XTMP4, XTMP4;            \ // XTMP4 = W[-9]^W[-16] ^ (W[-3] <<< 15) {DCxx}
    VPSHUFD $0xFA, XTMP4, XTMP4;            \ // XTMP4 = W[-9]^W[-16] ^ (W[-3] <<< 15) {DDCC}
    VPSRLQ  $17, XTMP4,XTMP2;               \
    VPSRLQ  $9, XTMP4,XTMP3;                \
    VPXOR   XTMP2, XTMP4, XTMP4;            \
    VPXOR   XTMP3, XTMP4, XTMP4;            \ // XTMP4 = p1 {xDxC}
    VPSHUFB shuff_DC00<>(SB), XTMP4, XTMP4; \ // XTMP4 = p1 {DC00}
    VPXOR   XTMP4, XTMP5,XDWORD0 


// 前16轮计算-0
// SRND = # of {4 round}
// T is saved in (dist_T)(TBL)(SRND*1)
// W is saved in (dist_W)(SP)(SRND*2)
// W' is saved in (dist_W + 68*4*2)(SP)(SRND*1)
#define DO_ROUND16(dist_T, dist_W, a, b, c, d, e, f, g, h) \
	;                                       \ // ################################### RND 0 - 15 ###########################
	RORXL   $(32-12), a, y2;                \ // y2 = a <<< 12
    MOVL    y2, y3;                         \
	ADDL    e, y3;                          \ // y3 = (a <<< 12) + e
    ADDL    (dist_T)(TBL)(SRND*1), y3;      \ // y3 = (a <<< 12) + e + T
    RORXL   $(32-7), y3, y3;                \ // y3 = ss1
    ADDL    y3, h;                          \ // h = h+ss1
    XORL    y2, y3;                         \ // y3 = ss2
    ADDL    y3, d;                          \ // d = d+ss2
    ;                                       \
    ADDL    (dist_W)(SP)(SRND*2), h;        \ // h = h + ss1 + w
    ADDL    (dist_W+68*4*2)(SP)(SRND*2), d; \ // d = d + ss2 + w'
    ;                                       \
	MOVL    a, y1;                          \ // y1 = a     //FF //PARITY
    XORL    b, y1;                          \ // y1 = a^b   //FF //PARITY
    XORL    c, y1;                          \ // y1 = a^b^c //FF //PARITY
    ADDL    y1, d;                          \ // d = TT1
    ;                                       \
    MOVL    e, y2;                          \ // y2 = e		// GG  //PARITY
    XORL    f, y2;                          \ // y2 = e^f   // GG  //PARITY
    XORL    g, y2;                          \ // y2 = e^f^g // GG  //PARITY
    ADDL    y2, h;                          \ // h = TT2
    ;                                       \
	RORXL   $(32-8), h, y0;                 \ // y0 = TT2<<<8
    XORL    h,y0;                           \ // y0 = (TT2<<<8)^TT2
    RORXL   $(32-9), y0, y0;                \ // y0 = ((TT2<<<8)^TT2)<<<9
    XORL    y0, h;                          \ // h = p0(TT2)
    ;                                       \
    RORXL   $(32-9), b, b;                  \
    RORXL   $(32-19), f, f

// 后48轮计算
#define DO_ROUND48(dist_T, dist_W, a, b, c, d, e, f, g, h) \
	;                                       \ // ################################### RND 16 - 63 ###########################
	RORXL   $(32-12), a, y2;                \ // y2 = a <<< 12
    MOVL    y2, y3;                         \
	ADDL    e, y3;                          \ // y3 = (a <<< 12) + e
    ADDL    (dist_T)(TBL)(SRND*1), y3;      \ // y3 = (a <<< 12) + e + T
    RORXL   $(32-7), y3, y3;                \ // y3 = ss1
    ADDL    y3, h;                          \ // h = h+ss1
    XORL    y2, y3;                         \ // y3 = ss2
    ADDL    y3, d;                          \ // d = d+ss2
    ;                                       \
    ADDL    (dist_W)(SP)(SRND*2), h;        \ // h = h + ss1 + w
    ADDL    (dist_W+68*4*2)(SP)(SRND*2), d; \ // d = d + ss2 + w'
    ;                                       \
    MOVL    a, y3;                          \ // y3 = a         //FF MAJA
	ORL     c, y3;                          \ // y3 = a|c       // MAJA
	ANDL    b, y3;                          \ // y3 = (a|c)&b   // MAJA
	MOVL    a, T1;                          \ // T1 = a         // MAJB
	ANDL    c, T1;                          \ // T1 = a&c       // MAJB
	ORL     T1, y3;                         \ // y3 = MAJ = ((a|c)&b)|(a&c)  // MAJ
    ADDL    y3, d;                          \ // d = TT1
    ;                                       \
    MOVL    f, y2;                          \ // y2 = f                 //GG CH
	XORL    g, y2;                          \ // y2 = f^g               // CH
	ANDL    e, y2;                          \ // y2 = (f^g)&e           // CH
	XORL    g, y2;                          \ // y2 = CH = ((f^g)&e)^g  // CH
    ADDL    y2, h;                          \ // h = TT2
    ;                                       \
	RORXL   $(32-8), h, y0;                 \ // y0 = TT2<<<8
    XORL    h,y0;                           \ // y0 = (TT2<<<8)^TT2
    RORXL   $(32-9), y0, y0;                \ // y0 = ((TT2<<<8)^TT2)<<<9
    XORL    y0, h;                          \ // h = p0(TT2)
    ;                                       \
    RORXL   $(32-9), b, b;                  \
    RORXL   $(32-19), f, f


// stack:
//         block0    block1    
//  0*8:   W[0:4]    V[0:4]    
//  1*8:   W[4:8]    V[4:8]  
//          ...
// 67*8:   W[64:68]  V[64:68]
// 68*8:   W'[0:4]   V'[0:4]
// 69*8:   W'[4:8]   V'[4:8]
//          ...
// 135*8:  W'[64:68]  V'[64:68]
// 136*8:  _INP_END(SP) - Pointer to the last block
// 137*8:  _INP(SP)     - Save INP in round computation.
// 
// STACK_SIZE = 1088+8+8 = 1104
// func blockAsmAVX2(dig *digest, p []byte)
TEXT ·blockAsmAVX2(SB), 0, $1104-32
	MOVQ dig+0(FP), CTX
	MOVQ p_base+8(FP), INP
	MOVQ p_len+16(FP), NUM_BYTES

	LEAQ -64(INP)(NUM_BYTES*1), NUM_BYTES // Pointer to the last block
	MOVQ NUM_BYTES, _INP_END(SP) // save to stack

	CMPQ NUM_BYTES, INP
	JE   avx2_only_one_block

	// Load initial digest
	MOVL 0(CTX), a  // a = H0
	MOVL 4(CTX), b  // b = H1
	MOVL 8(CTX), c  // c = H2
	MOVL 12(CTX), d // d = H3
	MOVL 16(CTX), e // e = H4
	MOVL 20(CTX), f // f = H5
	MOVL 24(CTX), g // g = H6
	MOVL 28(CTX), h // h = H7

avx2_loop0: // at each iteration works with one block (512 bit)

    // load two blocks，64*2 bytes
	VMOVDQU (0*32)(INP), XTMP0 // p[0:4]
	VMOVDQU (1*32)(INP), XTMP1
	VMOVDQU (2*32)(INP), XTMP2
	VMOVDQU (3*32)(INP), XTMP3

	VMOVDQU flip_mask<>(SB), BYTE_FLIP_MASK

	// Apply Byte Flip Mask: LE -> BE
	VPSHUFB BYTE_FLIP_MASK, XTMP0, XTMP0
	VPSHUFB BYTE_FLIP_MASK, XTMP1, XTMP1
	VPSHUFB BYTE_FLIP_MASK, XTMP2, XTMP2
	VPSHUFB BYTE_FLIP_MASK, XTMP3, XTMP3

    // XTMP0:XTMP1 - first block
    // XTMP0: w7, w6, w5, w4, w3, w2, w1,w0
    // XTMP1: w15,w14,w13,w12,w11,w10,w9,w8 
    // XTMP3:XTMP2 - second block
    // XTMP2: u7, u6, u5, u4, u3, u2, u1,u0
    // XTMP3: u15,u14,u13,u12,u11,u10,u9,u8 

    // XDWORD0: u3, u2, u1, u0, w3, w2, w1,w0
    // XDWORD1: u7, u6, u5, u4, w7, w6, w5, w4
    // XDWORD2: u11,u10,u9, u8, w11,w10,w9,w8 
    // XDWORD3: u15,u14,u13,u12,w15,w14,w13,w12

	// Transpose data into high/low parts
	VPERM2I128 $0x20, XTMP2, XTMP0, XDWORD0 // w3, w2, w1, w0
	VPERM2I128 $0x31, XTMP2, XTMP0, XDWORD1 // w7, w6, w5, w4
	VPERM2I128 $0x20, XTMP3, XTMP1, XDWORD2 // w11, w10, w9, w8
	VPERM2I128 $0x31, XTMP3, XTMP1, XDWORD3 // w15, w14, w13, w12
	MOVQ $T256<>(SB), TBL // Loading address of table with round-specific constants

avx2_last_block_enter:
	ADDQ $64, INP
	MOVQ INP, _INP(SP)
	XORQ SRND, SRND  // SRND increace 16 of each 4 rounds (dist of T)

    // for w0 - w15
	// Do 4 rounds and scheduling
	VMOVDQU XDWORD0, (_XFER + 0*32)(SP)(SRND*2)
    VPXOR   XDWORD0, XDWORD1, XFER
	VMOVDQU XFER, (_XFER + 0*32+68*4*2)(SP)(SRND*2)
    SM3_SCHED(XDWORD0, XDWORD1, XDWORD2, XDWORD3)
    DO_ROUND16(0*4, 0*32+0, a, b, c, d, e, f, g, h)
    DO_ROUND16(1*4, 0*32+4, d, a, b, c, h, e, f, g)
    DO_ROUND16(2*4, 0*32+8, c, d, a, b, g, h, e, f)
    DO_ROUND16(3*4, 0*32+12, b, c, d, a, f, g, h, e)

    VMOVDQU XDWORD1, (_XFER + 1*32)(SP)(SRND*2)
    VPXOR   XDWORD1, XDWORD2, XFER
	VMOVDQU XFER, (_XFER + 1*32+68*4*2)(SP)(SRND*2)
    SM3_SCHED(XDWORD1, XDWORD2, XDWORD3, XDWORD0)
    DO_ROUND16(4*4, 1*32+0, a, b, c, d, e, f, g, h)
    DO_ROUND16(5*4, 1*32+4, d, a, b, c, h, e, f, g)
    DO_ROUND16(6*4, 1*32+8, c, d, a, b, g, h, e, f)
    DO_ROUND16(7*4, 1*32+12, b, c, d, a, f, g, h, e)

    VMOVDQU XDWORD2, (_XFER + 2*32)(SP)(SRND*2)
    VPXOR   XDWORD2, XDWORD3, XFER
	VMOVDQU XFER, (_XFER + 2*32+68*4*2)(SP)(SRND*2)
    SM3_SCHED(XDWORD2, XDWORD3, XDWORD0, XDWORD1)
    DO_ROUND16(8*4, 2*32+0, a, b, c, d, e, f, g, h)
    DO_ROUND16(9*4, 2*32+4, d, a, b, c, h, e, f, g)
    DO_ROUND16(10*4,2*32+8, c, d, a, b, g, h, e, f)
    DO_ROUND16(11*4,2*32+12, b, c, d, a, f, g, h, e)

    VMOVDQU XDWORD3, (_XFER + 3*32)(SP)(SRND*2)
    VPXOR   XDWORD3, XDWORD0, XFER
	VMOVDQU XFER, (_XFER + 3*32+68*4*2)(SP)(SRND*2)
    SM3_SCHED(XDWORD3, XDWORD0, XDWORD1, XDWORD2)
    DO_ROUND16(12*4, 3*32+0, a, b, c, d, e, f, g, h)
    DO_ROUND16(13*4, 3*32+4, d, a, b, c, h, e, f, g)
    DO_ROUND16(14*4, 3*32+8, c, d, a, b, g, h, e, f)
    DO_ROUND16(15*4, 3*32+12, b, c, d, a, f, g, h, e)
	
    ADDQ $4*16, SRND

avx2_loop1: 
    // for w16 - w47 with scheduling (32 rounds)
    VMOVDQU XDWORD0, (_XFER + 0*32)(SP)(SRND*2)
    VPXOR   XDWORD0, XDWORD1, XFER
	VMOVDQU XFER, (_XFER + 0*32+68*4*2)(SP)(SRND*2)
    SM3_SCHED(XDWORD0, XDWORD1, XDWORD2, XDWORD3)
    DO_ROUND48(0*4, 0*32+0, a, b, c, d, e, f, g, h)
    DO_ROUND48(1*4, 0*32+4, d, a, b, c, h, e, f, g)
    DO_ROUND48(2*4, 0*32+8, c, d, a, b, g, h, e, f)
    DO_ROUND48(3*4, 0*32+12, b, c, d, a, f, g, h, e)

    VMOVDQU XDWORD1, (_XFER + 1*32)(SP)(SRND*2)
    VPXOR   XDWORD1, XDWORD2, XFER
	VMOVDQU XFER, (_XFER + 1*32+68*4*2)(SP)(SRND*2)
    SM3_SCHED(XDWORD1, XDWORD2, XDWORD3, XDWORD0)
    DO_ROUND48(4*4, 1*32+0, a, b, c, d, e, f, g, h)
    DO_ROUND48(5*4, 1*32+4, d, a, b, c, h, e, f, g)
    DO_ROUND48(6*4, 1*32+8, c, d, a, b, g, h, e, f)
    DO_ROUND48(7*4, 1*32+12, b, c, d, a, f, g, h, e)

    VMOVDQU XDWORD2, (_XFER + 2*32)(SP)(SRND*2)
    VPXOR   XDWORD2, XDWORD3, XFER
	VMOVDQU XFER, (_XFER + 2*32+68*4*2)(SP)(SRND*2)
    SM3_SCHED(XDWORD2, XDWORD3, XDWORD0, XDWORD1)
    DO_ROUND48(8*4, 2*32+0, a, b, c, d, e, f, g, h)
    DO_ROUND48(9*4, 2*32+4, d, a, b, c, h, e, f, g)
    DO_ROUND48(10*4,2*32+8, c, d, a, b, g, h, e, f)
    DO_ROUND48(11*4,2*32+12, b, c, d, a, f, g, h, e)

    VMOVDQU XDWORD3, (_XFER + 3*32)(SP)(SRND*2)
    VPXOR   XDWORD3, XDWORD0, XFER
	VMOVDQU XFER, (_XFER + 3*32+68*4*2)(SP)(SRND*2)
    SM3_SCHED(XDWORD3, XDWORD0, XDWORD1, XDWORD2)
    DO_ROUND48(12*4,3*32+0, a, b, c, d, e, f, g, h)
    DO_ROUND48(13*4,3*32+4, d, a, b, c, h, e, f, g)
    DO_ROUND48(14*4,3*32+8, c, d, a, b, g, h, e, f)
    DO_ROUND48(15*4,3*32+12, b, c, d, a, f, g, h, e)

    ADDQ $4*16, SRND
	CMPQ SRND, $12*16
	JB   avx2_loop1

	// w48 - w63 processed with one scheduling (last 16 rounds)
    VMOVDQU XDWORD0, (_XFER + 0*32)(SP)(SRND*2)
    VPXOR   XDWORD0, XDWORD1, XFER
	VMOVDQU XFER, (_XFER + 0*32+68*4*2)(SP)(SRND*2)
    SM3_SCHED(XDWORD0, XDWORD1, XDWORD2, XDWORD3) // scheduling XDWORD0 for W64-W67
    DO_ROUND48(0*4, 0*32+0, a, b, c, d, e, f, g, h)
    DO_ROUND48(1*4, 0*32+4, d, a, b, c, h, e, f, g)
    DO_ROUND48(2*4, 0*32+8, c, d, a, b, g, h, e, f)
    DO_ROUND48(3*4, 0*32+12, b, c, d, a, f, g, h, e)

    VMOVDQU XDWORD1, (_XFER + 1*32)(SP)(SRND*2)
    VPXOR   XDWORD1, XDWORD2, XFER
	VMOVDQU XFER, (_XFER + 1*32+68*4*2)(SP)(SRND*2)
    DO_ROUND48(4*4, 1*32+0, a, b, c, d, e, f, g, h)
    DO_ROUND48(5*4, 1*32+4, d, a, b, c, h, e, f, g)
    DO_ROUND48(6*4, 1*32+8, c, d, a, b, g, h, e, f)
    DO_ROUND48(7*4, 1*32+12, b, c, d, a, f, g, h, e)

    VMOVDQU XDWORD2, (_XFER + 2*32)(SP)(SRND*2)
    VPXOR   XDWORD2, XDWORD3, XFER
	VMOVDQU XFER, (_XFER + 2*32+68*4*2)(SP)(SRND*2)
    DO_ROUND48(8*4, 2*32+0, a, b, c, d, e, f, g, h)
    DO_ROUND48(9*4, 2*32+4, d, a, b, c, h, e, f, g)
    DO_ROUND48(10*4,2*32+8, c, d, a, b, g, h, e, f)
    DO_ROUND48(11*4,2*32+12, b, c, d, a, f, g, h, e)

    VMOVDQU XDWORD3, (_XFER + 3*32)(SP)(SRND*2)
    VPXOR   XDWORD3, XDWORD0, XFER
	VMOVDQU XFER, (_XFER + 3*32+68*4*2)(SP)(SRND*2)
    DO_ROUND48(12*4, 3*32+0, a, b, c, d, e, f, g, h)
    DO_ROUND48(13*4, 3*32+4, d, a, b, c, h, e, f, g)
    DO_ROUND48(14*4, 3*32+8, c, d, a, b, g, h, e, f)
    DO_ROUND48(15*4, 3*32+12, b, c, d, a, f, g, h, e)

	MOVQ dig+0(FP), CTX // d.h[8]
	MOVQ _INP(SP), INP

	xorm(  0(CTX), a)
	xorm(  4(CTX), b)
	xorm(  8(CTX), c)
	xorm( 12(CTX), d)
	xorm( 16(CTX), e)
	xorm( 20(CTX), f)
	xorm( 24(CTX), g)
	xorm( 28(CTX), h)

	CMPQ _INP_END(SP), INP
	JB   done_hash

	XORQ SRND, SRND

avx2_loop3: // Do second block using previously scheduled results
	DO_ROUND16(0, 16+0, a, b, c, d, e, f, g, h)
    DO_ROUND16(4, 16+4, d, a, b, c, h, e, f, g)
    DO_ROUND16(8, 16+8, c, d, a, b, g, h, e, f)
    DO_ROUND16(12, 16+12, b, c, d, a, f, g, h, e)

	ADDQ $16, SRND
	CMPQ SRND, $4*16
	JB   avx2_loop3
avx2_loop4:  
	DO_ROUND48(0, 16+0,a, b, c, d, e, f, g, h)
    DO_ROUND48(4, 16+4,d, a, b, c, h, e, f, g)
    DO_ROUND48(8, 16+8,c, d, a, b, g, h, e, f)
    DO_ROUND48(12, 16+12, b, c, d, a, f, g, h, e)

	ADDQ $16, SRND
	CMPQ SRND, $16*16
	JB   avx2_loop4

	MOVQ dig+0(FP), CTX // d.h[8]
	MOVQ _INP(SP), INP
	ADDQ $64, INP

	xorm(  0(CTX), a)
	xorm(  4(CTX), b)
	xorm(  8(CTX), c)
	xorm( 12(CTX), d)
	xorm( 16(CTX), e)
	xorm( 20(CTX), f)
	xorm( 24(CTX), g)
	xorm( 28(CTX), h)

	CMPQ _INP_END(SP), INP
	JA   avx2_loop0
	JB   done_hash

avx2_do_last_block:

	VMOVDQU 0(INP), XWORD0
	VMOVDQU 16(INP), XWORD1
	VMOVDQU 32(INP), XWORD2
	VMOVDQU 48(INP), XWORD3

	VMOVDQU flip_mask<>(SB), BYTE_FLIP_MASK

	VPSHUFB X_BYTE_FLIP_MASK, XWORD0, XWORD0
	VPSHUFB X_BYTE_FLIP_MASK, XWORD1, XWORD1
	VPSHUFB X_BYTE_FLIP_MASK, XWORD2, XWORD2
	VPSHUFB X_BYTE_FLIP_MASK, XWORD3, XWORD3
	MOVQ $T256<>(SB), TBL

	JMP avx2_last_block_enter

avx2_only_one_block:
	// Load initial digest
	MOVL 0(CTX), a  // a = H0
	MOVL 4(CTX), b  // b = H1
	MOVL 8(CTX), c  // c = H2
	MOVL 12(CTX), d // d = H3
	MOVL 16(CTX), e // e = H4
	MOVL 20(CTX), f // f = H5
	MOVL 24(CTX), g // g = H6
	MOVL 28(CTX), h // h = H7

	JMP avx2_do_last_block

done_hash:
	VZEROUPPER
	RET

// shuffle byte order from LE to BE
DATA flip_mask<>+0x00(SB)/8, $0x0405060700010203
DATA flip_mask<>+0x08(SB)/8, $0x0c0d0e0f08090a0b
DATA flip_mask<>+0x10(SB)/8, $0x0405060700010203
DATA flip_mask<>+0x18(SB)/8, $0x0c0d0e0f08090a0b
GLOBL flip_mask<>(SB), 8, $32

// shuffle xBxA -> 00BA
DATA shuff_00BA<>+0x00(SB)/8, $0x0b0a090803020100
DATA shuff_00BA<>+0x08(SB)/8, $0xFFFFFFFFFFFFFFFF
DATA shuff_00BA<>+0x10(SB)/8, $0x0b0a090803020100
DATA shuff_00BA<>+0x18(SB)/8, $0xFFFFFFFFFFFFFFFF
GLOBL shuff_00BA<>(SB), 8, $32

// shuffle xDxC -> DC00
DATA shuff_DC00<>+0x00(SB)/8, $0xFFFFFFFFFFFFFFFF
DATA shuff_DC00<>+0x08(SB)/8, $0x0b0a090803020100
DATA shuff_DC00<>+0x10(SB)/8, $0xFFFFFFFFFFFFFFFF
DATA shuff_DC00<>+0x18(SB)/8, $0x0b0a090803020100
GLOBL shuff_DC00<>(SB), 8, $32

// rotate of Tj: rotT[i] = Tj << (j mod 32)
DATA T256<>+0x0(SB)/4, $0x79cc4519
DATA T256<>+0x4(SB)/4, $0xf3988a32
DATA T256<>+0x8(SB)/4, $0xe7311465
DATA T256<>+0xc(SB)/4, $0xce6228cb
DATA T256<>+0x10(SB)/4, $0x9cc45197
DATA T256<>+0x14(SB)/4, $0x3988a32f
DATA T256<>+0x18(SB)/4, $0x7311465e
DATA T256<>+0x1c(SB)/4, $0xe6228cbc
DATA T256<>+0x20(SB)/4, $0xcc451979
DATA T256<>+0x24(SB)/4, $0x988a32f3
DATA T256<>+0x28(SB)/4, $0x311465e7
DATA T256<>+0x2c(SB)/4, $0x6228cbce
DATA T256<>+0x30(SB)/4, $0xc451979c
DATA T256<>+0x34(SB)/4, $0x88a32f39
DATA T256<>+0x38(SB)/4, $0x11465e73
DATA T256<>+0x3c(SB)/4, $0x228cbce6
DATA T256<>+0x40(SB)/4, $0x9d8a7a87
DATA T256<>+0x44(SB)/4, $0x3b14f50f
DATA T256<>+0x48(SB)/4, $0x7629ea1e
DATA T256<>+0x4c(SB)/4, $0xec53d43c
DATA T256<>+0x50(SB)/4, $0xd8a7a879
DATA T256<>+0x54(SB)/4, $0xb14f50f3
DATA T256<>+0x58(SB)/4, $0x629ea1e7
DATA T256<>+0x5c(SB)/4, $0xc53d43ce
DATA T256<>+0x60(SB)/4, $0x8a7a879d
DATA T256<>+0x64(SB)/4, $0x14f50f3b
DATA T256<>+0x68(SB)/4, $0x29ea1e76
DATA T256<>+0x6c(SB)/4, $0x53d43cec
DATA T256<>+0x70(SB)/4, $0xa7a879d8
DATA T256<>+0x74(SB)/4, $0x4f50f3b1
DATA T256<>+0x78(SB)/4, $0x9ea1e762
DATA T256<>+0x7c(SB)/4, $0x3d43cec5
DATA T256<>+0x80(SB)/4, $0x7a879d8a
DATA T256<>+0x84(SB)/4, $0xf50f3b14
DATA T256<>+0x88(SB)/4, $0xea1e7629
DATA T256<>+0x8c(SB)/4, $0xd43cec53
DATA T256<>+0x90(SB)/4, $0xa879d8a7
DATA T256<>+0x94(SB)/4, $0x50f3b14f
DATA T256<>+0x98(SB)/4, $0xa1e7629e
DATA T256<>+0x9c(SB)/4, $0x43cec53d
DATA T256<>+0xa0(SB)/4, $0x879d8a7a
DATA T256<>+0xa4(SB)/4, $0x0f3b14f5
DATA T256<>+0xa8(SB)/4, $0x1e7629ea
DATA T256<>+0xac(SB)/4, $0x3cec53d4
DATA T256<>+0xb0(SB)/4, $0x79d8a7a8
DATA T256<>+0xb4(SB)/4, $0xf3b14f50
DATA T256<>+0xb8(SB)/4, $0xe7629ea1
DATA T256<>+0xbc(SB)/4, $0xcec53d43
DATA T256<>+0xc0(SB)/4, $0x9d8a7a87
DATA T256<>+0xc4(SB)/4, $0x3b14f50f
DATA T256<>+0xc8(SB)/4, $0x7629ea1e
DATA T256<>+0xcc(SB)/4, $0xec53d43c
DATA T256<>+0xd0(SB)/4, $0xd8a7a879
DATA T256<>+0xd4(SB)/4, $0xb14f50f3
DATA T256<>+0xd8(SB)/4, $0x629ea1e7
DATA T256<>+0xdc(SB)/4, $0xc53d43ce
DATA T256<>+0xe0(SB)/4, $0x8a7a879d
DATA T256<>+0xe4(SB)/4, $0x14f50f3b
DATA T256<>+0xe8(SB)/4, $0x29ea1e76
DATA T256<>+0xec(SB)/4, $0x53d43cec
DATA T256<>+0xf0(SB)/4, $0xa7a879d8
DATA T256<>+0xf4(SB)/4, $0x4f50f3b1
DATA T256<>+0xf8(SB)/4, $0x9ea1e762
DATA T256<>+0xfc(SB)/4, $0x3d43cec5
GLOBL T256<>(SB), (NOPTR + RODATA), $256
