Hu Chunming / FFNvDecoder

;*******************************************************************************
;* SIMD-optimized IDCT functions for HEVC decoding
;* Copyright (c) 2014 Pierre-Edouard LEPERE
;* Copyright (c) 2014 James Almer
;* Copyright (c) 2016 Alexandra Hájková
;*
;* This file is part of FFmpeg.
;*
;* FFmpeg is free software; you can redistribute it and/or
;* modify it under the terms of the GNU Lesser General Public
;* License as published by the Free Software Foundation; either
;* version 2.1 of the License, or (at your option) any later version.
;*
;* FFmpeg is distributed in the hope that it will be useful,
;* but WITHOUT ANY WARRANTY; without even the implied warranty of
;* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
;* Lesser General Public License for more details.
;*
;* You should have received a copy of the GNU Lesser General Public
;* License along with FFmpeg; if not, write to the Free Software
;* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
;******************************************************************************

%include "libavutil/x86/x86util.asm"

SECTION_RODATA

pd_64: times 4 dd 64
pd_2048: times 4 dd 2048
pd_512: times 4 dd 512

; 4x4 transform coeffs
cextern pw_64
pw_64_m64: times 4 dw 64, -64
pw_83_36: times 4 dw 83, 36
pw_36_m83: times 4 dw 36, -83

; 8x8 transform coeffs
pw_89_75: times 4 dw 89, 75
pw_50_18: times 4 dw 50, 18

pw_75_m18: times 4 dw 75, -18
pw_m89_m50: times 4 dw -89, -50

pw_50_m89: times 4 dw 50, -89
pw_18_75: times 4 dw 18, 75

pw_18_m50: times 4 dw 18, -50
pw_75_m89: times 4 dw 75, -89

; 16x16 transformation coeffs
trans_coeffs16: times 4 dw 90, 87
times 4 dw 80, 70
times 4 dw 57, 43
times 4 dw 25, 9

times 4 dw 87, 57
times 4 dw 9, -43
times 4 dw -80, -90
times 4 dw -70, -25

times 4 dw 80, 9
times 4 dw -70, -87
times 4 dw -25, 57
times 4 dw 90, 43

times 4 dw 70, -43
times 4 dw -87, 9
times 4 dw 90, 25
times 4 dw -80, -57

times 4 dw 57, -80
times 4 dw -25, 90
times 4 dw -9, -87
times 4 dw 43, 70

times 4 dw 43, -90
times 4 dw 57, 25
times 4 dw -87, 70
times 4 dw 9, -80

times 4 dw 25, -70
times 4 dw 90, -80
times 4 dw 43, 9
times 4 dw -57, 87

times 4 dw 9, -25
times 4 dw 43, -57
times 4 dw 70, -80
times 4 dw 87, -90

; 32x32 transform coeffs
trans_coeff32: times 8 dw 90
times 4 dw 88, 85
times 4 dw 82, 78
times 4 dw 73, 67
times 4 dw 61, 54
times 4 dw 46, 38
times 4 dw 31, 22
times 4 dw 13, 4

times 4 dw 90, 82
times 4 dw 67, 46
times 4 dw 22, -4
times 4 dw -31, -54
times 4 dw -73, -85
times 4 dw -90, -88
times 4 dw -78, -61
times 4 dw -38, -13

times 4 dw 88, 67
times 4 dw 31, -13
times 4 dw -54, -82
times 4 dw -90, -78
times 4 dw -46, -4
times 4 dw 38, 73
times 4 dw 90, 85
times 4 dw 61, 22

times 4 dw 85, 46
times 4 dw -13, -67
times 4 dw -90, -73
times 4 dw -22, 38
times 4 dw 82, 88
times 4 dw 54, -4
times 4 dw -61, -90
times 4 dw -78, -31

times 4 dw 82, 22
times 4 dw -54, -90
times 4 dw -61, 13
times 4 dw 78, 85
times 4 dw 31, -46
times 4 dw -90, -67
times 4 dw 4, 73
times 4 dw 88, 38

times 4 dw 78, -4
times 4 dw -82, -73
times 4 dw 13, 85
times 4 dw 67, -22
times 4 dw -88, -61
times 4 dw 31, 90
times 4 dw 54, -38
times 4 dw -90, -46

times 4 dw 73, -31
times 4 dw -90, -22
times 4 dw 78, 67
times 4 dw -38, -90
times 4 dw -13, 82
times 4 dw 61, -46
times 4 dw -88, -4
times 4 dw 85, 54

times 4 dw 67, -54
times 4 dw -78, 38
times 4 dw 85, -22
times 4 dw -90, 4
times 4 dw 90, 13
times 4 dw -88, -31
times 4 dw 82, 46
times 4 dw -73, -61

times 4 dw 61, -73
times 4 dw -46, 82
times 4 dw 31, -88
times 4 dw -13, 90
times 4 dw -4, -90
times 4 dw 22, 85
times 4 dw -38, -78
times 4 dw 54, 67

times 4 dw 54, -85
times 4 dw -4, 88
times 4 dw -46, -61
times 4 dw 82, 13
times 4 dw -90, 38
times 4 dw 67, -78
times 4 dw -22, 90
times 4 dw -31, -73

times 4 dw 46, -90
times 4 dw 38, 54
times 4 dw -90, 31
times 4 dw 61, -88
times 4 dw 22, 67
times 4 dw -85, 13
times 4 dw 73, -82
times 4 dw 4, 78

times 4 dw 38, -88
times 4 dw 73, -4
times 4 dw -67, 90
times 4 dw -46, -31
times 4 dw 85, -78
times 4 dw 13, 61
times 4 dw -90, 54
times 4 dw 22, -82

times 4 dw 31, -78
times 4 dw 90, -61
times 4 dw 4, 54
times 4 dw -88, 82
times 4 dw -38, -22
times 4 dw 73, -90
times 4 dw 67, -13
times 4 dw -46, 85

times 4 dw 22, -61
times 4 dw 85, -90
times 4 dw 73, -38
times 4 dw -4, 46
times 4 dw -78, 90
times 4 dw -82, 54
times 4 dw -13, -31
times 4 dw 67, -88

times 4 dw 13, -38
times 4 dw 61, -78
times 4 dw 88, -90
times 4 dw 85, -73
times 4 dw 54, -31
times 4 dw 4, 22
times 4 dw -46, 67
times 4 dw -82, 90

times 4 dw 4, -13
times 4 dw 22, -31
times 4 dw 38, -46
times 4 dw 54, -61
times 4 dw 67, -73
times 4 dw 78, -82
times 4 dw 85, -88
times 4 dw 90, -90

SECTION .text

; void ff_hevc_idct_HxW_dc_{8,10}_<opt>(int16_t *coeffs)
; %1 = HxW
; %2 = number of loops
; %3 = bitdepth
%macro IDCT_DC 3
cglobal hevc_idct_%1x%1_dc_%3, 1, 2, 1, coeff, tmp
    movsx             tmpd, word [coeffq]
    add               tmpd, (1 << (14 - %3)) + 1
    sar               tmpd, (15 - %3)
    movd               xm0, tmpd
    SPLATW              m0, xm0
    DEFINE_ARGS coeff, cnt
    mov               cntd, %2
.loop:
    mova [coeffq+mmsize*0], m0
    mova [coeffq+mmsize*1], m0
    mova [coeffq+mmsize*2], m0
    mova [coeffq+mmsize*3], m0
    add  coeffq, mmsize*8
    mova [coeffq+mmsize*-4], m0
    mova [coeffq+mmsize*-3], m0
    mova [coeffq+mmsize*-2], m0
    mova [coeffq+mmsize*-1], m0
    dec  cntd
    jg  .loop
    RET
%endmacro

; %1 = HxW
; %2 = bitdepth
%macro IDCT_DC_NL 2 ; No loop
cglobal hevc_idct_%1x%1_dc_%2, 1, 2, 1, coeff, tmp
    movsx             tmpd, word [coeffq]
    add               tmpd, (1 << (14 - %2)) + 1
    sar               tmpd, (15 - %2)
    movd                m0, tmpd
    SPLATW              m0, xm0
    mova [coeffq+mmsize*0], m0
    mova [coeffq+mmsize*1], m0
    mova [coeffq+mmsize*2], m0
    mova [coeffq+mmsize*3], m0
%if mmsize == 16
    mova [coeffq+mmsize*4], m0
    mova [coeffq+mmsize*5], m0
    mova [coeffq+mmsize*6], m0
    mova [coeffq+mmsize*7], m0
%endif
    RET
%endmacro

; IDCT 4x4, expects input in m0, m1
; %1 - shift
; %2 - 1/0 - SCALE and Transpose or not
; %3 - 1/0 add constant or not
%macro TR_4x4 3
    ; interleaves src0 with src2 to m0
    ;         and src1 with scr3 to m2
    ; src0: 00 01 02 03     m0: 00 20 01 21 02 22 03 23
    ; src1: 10 11 12 13 -->
    ; src2: 20 21 22 23     m1: 10 30 11 31 12 32 13 33
    ; src3: 30 31 32 33

    SBUTTERFLY wd, 0, 1, 2

    pmaddwd m2, m0, [pw_64]    ; e0
    pmaddwd m3, m1, [pw_83_36] ; o0
    pmaddwd m0, [pw_64_m64]    ; e1
    pmaddwd m1, [pw_36_m83]    ; o1

%if %3 == 1
    %assign %%add 1 << (%1 - 1)
    mova  m4, [pd_ %+ %%add]
    paddd m2, m4
    paddd m0, m4
%endif

    SUMSUB_BADC d, 3, 2, 1, 0, 4

%if %2 == 1
    psrad m3, %1 ; e0 + o0
    psrad m1, %1 ; e1 + o1
    psrad m2, %1 ; e0 - o0
    psrad m0, %1 ; e1 - o1
    ;clip16
    packssdw m3, m1
    packssdw m0, m2
    ; Transpose
    SBUTTERFLY wd, 3, 0, 1
    SBUTTERFLY wd, 3, 0, 1
    SWAP 3, 1, 0
%else
    SWAP 3, 2, 0
%endif
%endmacro

%macro DEFINE_BIAS 1
    %assign shift (20 - %1)
    %assign c_add (1 << (shift - 1))
    %define arr_add pd_ %+ c_add
%endmacro

; %1 - bit_depth
; %2 - register add constant
; is loaded to
; shift = 20 - bit_depth
%macro LOAD_BIAS 2
    DEFINE_BIAS %1
    mova %2, [arr_add]
%endmacro

; %1, %2 - registers to load packed 16 bit values to
; %3, %4, %5, %6 - vertical offsets
; %7 - horizontal offset
%macro LOAD_BLOCK 7
    movq   %1, [r0 + %3 + %7]
    movhps %1, [r0 + %5 + %7]
    movq   %2, [r0 + %4 + %7]
    movhps %2, [r0 + %6 + %7]
%endmacro

; void ff_hevc_idct_4x4__{8,10}_<opt>(int16_t *coeffs, int col_limit)
; %1 = bitdepth
%macro IDCT_4x4 1
cglobal hevc_idct_4x4_%1, 1, 1, 5, coeffs
    mova m0, [coeffsq]
    mova m1, [coeffsq + 16]

    TR_4x4 7, 1, 1
    TR_4x4 20 - %1, 1, 1

    mova [coeffsq],      m0
    mova [coeffsq + 16], m1
    RET
%endmacro

; scale, pack (clip16) and store the residuals     0 e8[0] + o8[0] --> + %1
; 4 at one time (4 columns)                        1 e8[1] + o8[1]
; from %5: e8/16 + o8/16, with %1 offset                  ...
; and  %3: e8/16 - o8/16, with %2 offset           6 e8[1] - o8[1]
; %4 - shift                                       7 e8[0] - o8[0] --> + %2
%macro STORE_8 7
    psrad    %5, %4
    psrad    %3, %4
    packssdw %5, %3
    movq     [coeffsq + %1], %5
    movhps   [coeffsq + %2], %5
%endmacro

; %1 - horizontal offset
; %2 - shift
; %3, %4 - transform coeffs
; %5 - vertical offset for e8 + o8
; %6 - vertical offset for e8 - o8
; %7 - register with e8 inside
; %8 - block_size
; %9 - register to store e8 +o8
; %10 - register to store e8 - o8
%macro E8_O8 10
    pmaddwd m6, m4, %3
    pmaddwd m7, m5, %4

    paddd m6, m7
    paddd m7, m6, %7 ; o8 + e8
    psubd %7, m6     ; e8 - o8
%if %8 == 8
    STORE_8 %5 + %1, %6 + %1, %7, %2, m7, 0, 0
%else
    SWAP m7, %9
    SWAP %7, %10
%endif
%endmacro

; 8x4 residuals are processed and stored
; %1 - horizontal offset
; %2 - shift
; %3 - offset of the even row
; %4 - step: 1 for 8x8, 2 for 16x16, 4 for 32x32
; %5 - offset of the odd row
; %6 - block size
; %7 - 1/0 add a constant in TR_4x4 or not
; I want to add a constant for 8x8 transform but not for 16x16 and 32x32
%macro TR_8x4 7
    ; load 4 columns of even rows
    LOAD_BLOCK  m0, m1, 0, 2 * %4 * %3, %4 * %3, 3 * %4 * %3, %1

    TR_4x4 %2, 0, %7 ; e8: m0, m1, m2, m3, for 4 columns only

    ; load 4 columns of odd rows
    LOAD_BLOCK m4, m5, %4 * %5, 3 * %4 * %5, 5 * %4 * %5, 7 * %4 * %5, %1

    ; 00 01 02 03
    ; 10 11 12 13      m4: 10 30 11 31 12 32 13 33

    ; ...        -- >
    ;                  m5: 50 70 51 71 52 72 53 73
    ; 70 71 72 73
    SBUTTERFLY wd, 4, 5, 6

    E8_O8 %1, %2, [pw_89_75],  [pw_50_18],   0,      %5 * 7, m0, %6, m8, m15
    E8_O8 %1, %2, [pw_75_m18], [pw_m89_m50], %5,     %5 * 6, m1, %6, m9, m14
    E8_O8 %1, %2, [pw_50_m89], [pw_18_75],   %5 * 2, %5 * 5, m2, %6, m10, m13
    E8_O8 %1, %2, [pw_18_m50], [pw_75_m89],  %5 * 3, %5 * 4, m3, %6, m11, m12
%endmacro

%macro STORE_PACKED 7
    movq   [r0 + %3 + %7], %1
    movhps [r0 + %4 + %7], %1
    movq   [r0 + %5 + %7], %2
    movhps [r0 + %6 + %7], %2
%endmacro

; transpose 4x4 block packed
; in %1 and %2 registers
; %3 - temporary register
%macro TRANSPOSE_4x4 3
    SBUTTERFLY wd, %1, %2, %3
    SBUTTERFLY dq, %1, %2, %3
%endmacro

; %1 - horizontal offset of the block i
; %2 - vertical offset of the block i
; %3 - width in bytes
; %4 - vertical offset for the block j
; %5 - horizontal offset for the block j
%macro SWAP_BLOCKS 5
    ; M_j
    LOAD_BLOCK m4, m5, %4, %4 + %3, %4 + 2 * %3, %4 + 3 * %3, %5
    TRANSPOSE_4x4 4, 5, 6

    ; M_i
    LOAD_BLOCK m6, m7, %2, %2 + %3, %2 + 2 * %3, %2 + 3 * %3, %1

    STORE_PACKED m4, m5, %2, %2 + %3, %2 + 2 * %3, %2 + 3 * %3, %1

    ; transpose and store M_i
    SWAP m6, m4
    SWAP m7, m5
    TRANSPOSE_4x4 4, 5, 6
    STORE_PACKED m4, m5, %4, %4 + %3, %4 + 2 * %3, %4 + 3 * %3, %5
%endmacro

; %1 - horizontal offset
; %2 - vertical offset of the block
; %3 - width in bytes
%macro TRANSPOSE_BLOCK 3
    LOAD_BLOCK m4, m5, %2, %2 + %3, %2 + 2 * %3, %2 + 3 * %3, %1
    TRANSPOSE_4x4 4, 5, 6
    STORE_PACKED m4, m5, %2, %2 + %3, %2 + 2 * %3, %2 + 3 * %3, %1
%endmacro

%macro TRANSPOSE_8x8 0
cglobal hevc_idct_transpose_8x8, 0, 0, 0
    ; M1 M2 ^T = M1^t M3^t
    ; M3 M4      M2^t M4^t

    ; M1 4x4 block
    TRANSPOSE_BLOCK 0, 0, 16

    ; M2 and M3
    SWAP_BLOCKS 0, 64, 16, 0, 8

    ; M4
    TRANSPOSE_BLOCK 8, 64, 16

    ret
%endmacro

; void ff_hevc_idct_8x8_{8,10}_<opt>(int16_t *coeffs, int col_limit)
; %1 = bitdepth
%macro IDCT_8x8 1
cglobal hevc_idct_8x8_%1, 1, 1, 8, coeffs
    TR_8x4 0, 7, 32, 1, 16, 8, 1
    TR_8x4 8, 7, 32, 1, 16, 8, 1

    call hevc_idct_transpose_8x8_ %+ cpuname

    DEFINE_BIAS %1
    TR_8x4 0, shift, 32, 1, 16, 8, 1
    TR_8x4 8, shift, 32, 1, 16, 8, 1

    TAIL_CALL hevc_idct_transpose_8x8_ %+ cpuname, 1
%endmacro

; store intermedite e32 coeffs on stack
; as 16x4 matrix
; from m10: e8 + o8, with %6 offset
; and  %3:  e8 - o8, with %7 offset
; %4 - shift, unused here
%macro STORE_16 7
    mova [rsp + %6], %5
    mova [rsp + %7], %3
%endmacro

; %1, %2 - transform constants
; %3, %4 - regs with interleaved coeffs
; %5 - 1/0 SWAP or add
; %6, %7 - registers for intermidiate sums
; %8 - accumulator register
%macro ADD_ROWS 8
    pmaddwd %6, %3, %1
    pmaddwd %7, %4, %2
    paddd   %6, %7
%if %5 == 1
    SWAP %6, %8
%else
    paddd %8, %6
%endif
%endmacro

; %1 - transform coeffs
; %2, %3 offsets for storing e+o/e-o back to coeffsq
; %4 - shift
; %5 - add
; %6 - block_size
; %7 - register with e16
; %8, %9 - stack offsets for storing e+o/e-o
%macro E16_O16 9
    ADD_ROWS [%1],          [%1 +     16], m0, m1, 1, m5, m6, m7
    ADD_ROWS [%1 + 2 * 16], [%1 + 3 * 16], m2, m3, 0, m5, m6, m7

%if %6 == 8
    paddd %7, %5
%endif

    paddd m4, m7, %7 ; o16 + e16
    psubd %7, m7     ; e16 - o16
    STORE_%6 %2, %3, %7, %4, m4, %8, %9
%endmacro

%macro TR_16x4 10
    ; produce 8x4 matrix of e16 coeffs
    ; for 4 first rows and store it on stack (128 bytes)
    TR_8x4 %1, 7, %4, %5, %6, %8, 0

    ; load 8 even rows
    LOAD_BLOCK m0, m1, %9 * %6, %9 * 3 * %6, %9 * 5 * %6, %9 * 7 * %6, %1
    LOAD_BLOCK m2, m3, %9 * 9 * %6, %9 * 11 * %6, %9 * 13 * %6, %9 * 15 * %6, %1

    SBUTTERFLY wd, 0, 1, 4
    SBUTTERFLY wd, 2, 3, 4

    E16_O16 trans_coeffs16,               0 + %1, 15 * %6 + %1, %2, %3, %7, m8,       0, 15 * 16
    mova m8, %3
    E16_O16 trans_coeffs16 +     64,     %6 + %1, 14 * %6 + %1, %2, m8, %7, m9,      16, 14 * 16
    E16_O16 trans_coeffs16 + 2 * 64, 2 * %6 + %1, 13 * %6 + %1, %2, m8, %7, m10, 2 * 16, 13 * 16
    E16_O16 trans_coeffs16 + 3 * 64, 3 * %6 + %1, 12 * %6 + %1, %2, m8, %7, m11, 3 * 16, 12 * 16
    E16_O16 trans_coeffs16 + 4 * 64, 4 * %6 + %1, 11 * %6 + %1, %2, m8, %7, m12, 4 * 16, 11 * 16
    E16_O16 trans_coeffs16 + 5 * 64, 5 * %6 + %1, 10 * %6 + %1, %2, m8, %7, m13, 5 * 16, 10 * 16
    E16_O16 trans_coeffs16 + 6 * 64, 6 * %6 + %1,  9 * %6 + %1, %2, m8, %7, m14, 6 * 16,  9 * 16
    E16_O16 trans_coeffs16 + 7 * 64, 7 * %6 + %1,  8 * %6 + %1, %2, m8, %7, m15, 7 * 16,  8 * 16
%endmacro

%macro TRANSPOSE_16x16 0
cglobal hevc_idct_transpose_16x16, 0, 0, 0
; M1  M2  M3  M4 ^T      m1 m5 m9  m13   M_i^T = m_i
; M5  M6  M7  M8    -->  m2 m6 m10 m14
; M9  M10 M11 M12        m3 m7 m11 m15
; M13 M14 M15 M16        m4 m8 m12 m16

    ; M1 4x4 block
    TRANSPOSE_BLOCK 0, 0, 32

    ; M5, M2
    SWAP_BLOCKS 0, 128, 32, 0, 8
    ; M9, M3
    SWAP_BLOCKS 0, 256, 32, 0, 16
    ; M13, M4
    SWAP_BLOCKS 0, 384, 32, 0, 24

    ;M6
    TRANSPOSE_BLOCK 8, 128, 32

    ; M10, M7
    SWAP_BLOCKS 8, 256, 32, 128, 16
    ; M14, M8
    SWAP_BLOCKS 8, 384, 32, 128, 24

    ;M11
    TRANSPOSE_BLOCK 16, 256, 32

    ; M15, M12
    SWAP_BLOCKS 16, 384, 32, 256, 24

    ;M16
    TRANSPOSE_BLOCK 24, 384, 32

    ret
%endmacro

; void ff_hevc_idct_16x16_{8,10}_<opt>(int16_t *coeffs, int col_limit)
; %1 = bitdepth
%macro IDCT_16x16 1
cglobal hevc_idct_16x16_%1, 1, 2, 16, coeffs
    mov r1d, 3
.loop16:
    TR_16x4 8 * r1, 7, [pd_64], 64, 2, 32, 8, 16, 1, 0
    dec r1d
    jge .loop16

    call hevc_idct_transpose_16x16_ %+ cpuname

    DEFINE_BIAS %1
    mov r1d, 3
.loop16_2:
    TR_16x4 8 * r1, shift, [arr_add], 64, 2, 32, 8, 16, 1, 1
    dec r1d
    jge .loop16_2

    TAIL_CALL hevc_idct_transpose_16x16_ %+ cpuname, 1
%endmacro

; scale, pack (clip16) and store the residuals     0 e32[0] + o32[0] --> %1
; 4 at one time (4 columns)                        1 e32[1] + o32[1]
; %1 - address to store e32 + o32
; %2 - address to store e32 - e32
; %5 - reg with e32 + o32                                  ...
; %3 - reg with e32 - o32                          30 e32[1] - o32[1]
; %4 - shift                                       31 e32[0] - o32[0] --> %2
%macro STORE_32 5
    psrad    %5, %4
    psrad    %3, %4
    packssdw %5, %3
    movq     [%1], %5
    movhps   [%2], %5
%endmacro

; %1 - transform coeffs
; %2 - stack offset for e32
; %2, %3 offsets for storing e+o/e-o back to coeffsq
; %4 - shift
; %5 - stack offset of e32
%macro E32_O32 5
    ADD_ROWS [%1],          [%1 +     16], m0, m1, 1, m8, m9, m10
    ADD_ROWS [%1 + 2 * 16], [%1 + 3 * 16], m2, m3, 0, m8, m9, m10
    ADD_ROWS [%1 + 4 * 16], [%1 + 5 * 16], m4, m5, 0, m8, m9, m10
    ADD_ROWS [%1 + 6 * 16], [%1 + 7 * 16], m6, m7, 0, m8, m9, m10

    paddd m11, m14, [rsp + %5]
    paddd m12, m10, m11 ; o32 + e32
    psubd m11, m10      ; e32 - o32
    STORE_32 %2, %3, m11, %4, m12
%endmacro

; %1 - horizontal offset
; %2 - bitdepth
%macro TR_32x4 3
    TR_16x4 %1, 7, [pd_64], 128, 4, 64, 16, 16, 2, 0

    LOAD_BLOCK m0, m1,      64,  3 * 64,  5 * 64,  7 * 64, %1
    LOAD_BLOCK m2, m3,  9 * 64, 11 * 64, 13 * 64, 15 * 64, %1
    LOAD_BLOCK m4, m5, 17 * 64, 19 * 64, 21 * 64, 23 * 64, %1
    LOAD_BLOCK m6, m7, 25 * 64, 27 * 64, 29 * 64, 31 * 64, %1

    SBUTTERFLY wd, 0, 1, 8
    SBUTTERFLY wd, 2, 3, 8
    SBUTTERFLY wd, 4, 5, 8
    SBUTTERFLY wd, 6, 7, 8

%if %3 == 1
    %assign shift 7
    mova m14, [pd_64]
%else
    LOAD_BIAS %2, m14
%endif

    lea r2, [trans_coeff32 + 15 * 128]
    lea r3, [coeffsq + %1]
    lea r4, [r3 + 16 * 64]
    mov r5d, 15 * 16
%%loop:
    E32_O32 r2, r3 + r5 * 4, r4, shift, r5
    sub r2, 128
    add r4, 64
    sub r5d, 16
    jge %%loop
%endmacro

%macro TRANSPOSE_32x32 0
cglobal hevc_idct_transpose_32x32, 0, 0, 0
    ; M0  M1 ... M7
    ; M8         M15
    ;
    ; ...
    ;
    ; M56        M63

    TRANSPOSE_BLOCK 0, 0, 64 ; M1
    mov r1d, 7
    mov r2d, 7 * 256
.loop_transpose:
    SWAP_BLOCKS 0, r2, 64, 0, r1 * 8
    sub r2d, 256
    dec r1d
    jg .loop_transpose

    TRANSPOSE_BLOCK 8, 256, 64 ; M9
    mov r1d, 6
    mov r2d, 512
    mov r3d, 16
.loop_transpose2:
    SWAP_BLOCKS 8, r2, 64, 256, r3
    add r3d, 8
    add r2d, 256
    dec r1d
    jg .loop_transpose2

    TRANSPOSE_BLOCK 2 * 8, 2 * 256, 64 ; M9
    mov r1d, 5
    mov r2d, 768
    mov r3d, 24
.loop_transpose3:
    SWAP_BLOCKS 2 * 8, r2, 64, 2 * 256, r3
    add r3d, 8
    add r2d, 256
    dec r1d
    jg .loop_transpose3

    TRANSPOSE_BLOCK 3 * 8, 3 * 256, 64 ; M27
    mov r1d, 4
    mov r2d, 1024
    mov r3d, 32
.loop_transpose4:
    SWAP_BLOCKS 3 * 8, r2, 64, 3 * 256, r3
    add r3d, 8
    add r2d, 256
    dec r1d
    jg .loop_transpose4

    TRANSPOSE_BLOCK 4 * 8, 4 * 256, 64 ; M36
    mov r1d, 3
    mov r2d, 1280
    mov r3d, 40
.loop_transpose5:
    SWAP_BLOCKS 4 * 8, r2, 64, 4 * 256, r3
    add r3d, 8
    add r2d, 256
    dec r1d
    jg .loop_transpose5

    TRANSPOSE_BLOCK 5 * 8, 5 * 256, 64 ; M45
    SWAP_BLOCKS 5 * 8, 6 * 256, 64, 5 * 256, 6 * 8
    SWAP_BLOCKS 5 * 8, 7 * 256, 64, 5 * 256, 7 * 8

    TRANSPOSE_BLOCK 6 * 8, 6 * 256, 64 ; M54
    SWAP_BLOCKS 6 * 8, 7 * 256, 64, 6 * 256, 7 * 8

    TRANSPOSE_BLOCK 7 * 8, 7 * 256, 64 ; M63

    ret
%endmacro

; void ff_hevc_idct_32x32_{8,10}_<opt>(int16_t *coeffs, int col_limit)
; %1 = bitdepth
%macro IDCT_32x32 1
cglobal hevc_idct_32x32_%1, 1, 6, 16, 256, coeffs
    mov r1d, 7
.loop32:
    TR_32x4 8 * r1, %1, 1
    dec r1d
    jge .loop32

    call hevc_idct_transpose_32x32_ %+ cpuname

    mov r1d, 7
.loop32_2:
    TR_32x4 8 * r1, %1, 0
    dec r1d
    jge .loop32_2

    TAIL_CALL hevc_idct_transpose_32x32_ %+ cpuname, 1
%endmacro

%macro INIT_IDCT_DC 1
INIT_MMX mmxext
IDCT_DC_NL  4,      %1
IDCT_DC     8,  2,  %1

INIT_XMM sse2
IDCT_DC_NL  8,      %1
IDCT_DC    16,  4,  %1
IDCT_DC    32, 16,  %1

%if HAVE_AVX2_EXTERNAL
    INIT_YMM avx2
    IDCT_DC    16,  2,  %1
    IDCT_DC    32,  8,  %1
%endif ;HAVE_AVX2_EXTERNAL
%endmacro

%macro INIT_IDCT 2
INIT_XMM %2
%if %1 == 8
    TRANSPOSE_8x8
    %if ARCH_X86_64
        TRANSPOSE_16x16
        TRANSPOSE_32x32
    %endif
%endif
%if ARCH_X86_64
    IDCT_32x32 %1
    IDCT_16x16 %1
%endif
IDCT_8x8 %1
IDCT_4x4 %1
%endmacro

INIT_IDCT_DC 8
INIT_IDCT_DC 10
INIT_IDCT_DC 12
INIT_IDCT 8, sse2
INIT_IDCT 8, avx
INIT_IDCT 10, sse2
INIT_IDCT 10, avx
;INIT_IDCT 12, sse2
;INIT_IDCT 12, avx