fft.asm 25.3 KB

Edit Raw Blame History

;******************************************************************************
;* FFT transform with SSE/3DNow optimizations
;* Copyright (c) 2008 Loren Merritt
;* Copyright (c) 2011 Vitor Sessak
;*
;* This algorithm (though not any of the implementation details) is
;* based on libdjbfft by D. J. Bernstein.
;*
;* This file is part of FFmpeg.
;*
;* FFmpeg is free software; you can redistribute it and/or
;* modify it under the terms of the GNU Lesser General Public
;* License as published by the Free Software Foundation; either
;* version 2.1 of the License, or (at your option) any later version.
;*
;* FFmpeg is distributed in the hope that it will be useful,
;* but WITHOUT ANY WARRANTY; without even the implied warranty of
;* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
;* Lesser General Public License for more details.
;*
;* You should have received a copy of the GNU Lesser General Public
;* License along with FFmpeg; if not, write to the Free Software
;* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
;******************************************************************************

; These functions are not individually interchangeable with the C versions.
; While C takes arrays of FFTComplex, SSE/3DNow leave intermediate results
; in blocks as conventient to the vector size.
; i.e. {4x real, 4x imaginary, 4x real, ...} (or 2x respectively)

%include "libavutil/x86/x86util.asm"

%if ARCH_X86_64
%define pointer resq
%else
%define pointer resd
%endif

struc FFTContext
    .nbits:    resd 1
    .reverse:  resd 1
    .revtab:   pointer 1
    .tmpbuf:   pointer 1
    .mdctsize: resd 1
    .mdctbits: resd 1
    .tcos:     pointer 1
    .tsin:     pointer 1
    .fftperm:  pointer 1
    .fftcalc:  pointer 1
    .imdctcalc:pointer 1
    .imdcthalf:pointer 1
endstruc

SECTION_RODATA 32

%define M_SQRT1_2 0.70710678118654752440
%define M_COS_PI_1_8 0.923879532511287
%define M_COS_PI_3_8 0.38268343236509

ps_cos16_1: dd 1.0, M_COS_PI_1_8, M_SQRT1_2, M_COS_PI_3_8, 1.0, M_COS_PI_1_8, M_SQRT1_2, M_COS_PI_3_8
ps_cos16_2: dd 0, M_COS_PI_3_8, M_SQRT1_2, M_COS_PI_1_8, 0, -M_COS_PI_3_8, -M_SQRT1_2, -M_COS_PI_1_8

ps_root2: times 8 dd M_SQRT1_2
ps_root2mppm: dd -M_SQRT1_2, M_SQRT1_2, M_SQRT1_2, -M_SQRT1_2, -M_SQRT1_2, M_SQRT1_2, M_SQRT1_2, -M_SQRT1_2
ps_p1p1m1p1: dd 0, 0, 1<<31, 0, 0, 0, 1<<31, 0

perm1: dd 0x00, 0x02, 0x03, 0x01, 0x03, 0x00, 0x02, 0x01
perm2: dd 0x00, 0x01, 0x02, 0x03, 0x01, 0x00, 0x02, 0x03
ps_p1p1m1p1root2: dd 1.0, 1.0, -1.0, 1.0, M_SQRT1_2, M_SQRT1_2, M_SQRT1_2, M_SQRT1_2
ps_m1m1p1m1p1m1m1m1: dd 1<<31, 1<<31, 0, 1<<31, 0, 1<<31, 1<<31, 1<<31
ps_m1p1: dd 1<<31, 0

cextern ps_neg

%assign i 16
%rep 14
cextern cos_ %+ i
%assign i i<<1
%endrep

%if ARCH_X86_64
    %define pointer dq
%else
    %define pointer dd
%endif

%macro IF0 1+
%endmacro
%macro IF1 1+
    %1
%endmacro

SECTION .text

%macro T2_3DNOW 4 ; z0, z1, mem0, mem1
    mova     %1, %3
    mova     %2, %1
    pfadd    %1, %4
    pfsub    %2, %4
%endmacro

%macro T4_3DNOW 6 ; z0, z1, z2, z3, tmp0, tmp1
    mova     %5, %3
    pfsub    %3, %4
    pfadd    %5, %4 ; {t6,t5}
    pxor     %3, [ps_m1p1] ; {t8,t7}
    mova     %6, %1
    movd [r0+12], %3
    punpckhdq %3, [r0+8]
    pfadd    %1, %5 ; {r0,i0}
    pfsub    %6, %5 ; {r2,i2}
    mova     %4, %2
    pfadd    %2, %3 ; {r1,i1}
    pfsub    %4, %3 ; {r3,i3}
    SWAP     %3, %6
%endmacro

;  in: %1 = {r0,i0,r2,i2,r4,i4,r6,i6}
;      %2 = {r1,i1,r3,i3,r5,i5,r7,i7}
;      %3, %4, %5 tmp
; out: %1 = {r0,r1,r2,r3,i0,i1,i2,i3}
;      %2 = {r4,r5,r6,r7,i4,i5,i6,i7}
%macro T8_AVX 5
    vsubps     %5, %1, %2       ; v  = %1 - %2
    vaddps     %3, %1, %2       ; w  = %1 + %2
    vmulps     %2, %5, [ps_p1p1m1p1root2]  ; v *= vals1
    vpermilps  %2, %2, [perm1]
    vblendps   %1, %2, %3, 0x33 ; q = {w1,w2,v4,v2,w5,w6,v7,v6}
    vshufps    %5, %3, %2, 0x4e ; r = {w3,w4,v1,v3,w7,w8,v8,v5}
    vsubps     %4, %5, %1       ; s = r - q
    vaddps     %1, %5, %1       ; u = r + q
    vpermilps  %1, %1, [perm2]  ; k  = {u1,u2,u3,u4,u6,u5,u7,u8}
    vshufps    %5, %4, %1, 0xbb
    vshufps    %3, %4, %1, 0xee
    vperm2f128 %3, %3, %5, 0x13
    vxorps     %4, %4, [ps_m1m1p1m1p1m1m1m1]  ; s *= {1,1,-1,-1,1,-1,-1,-1}
    vshufps    %2, %1, %4, 0xdd
    vshufps    %1, %1, %4, 0x88
    vperm2f128 %4, %2, %1, 0x02 ; v  = {k1,k3,s1,s3,k2,k4,s2,s4}
    vperm2f128 %1, %1, %2, 0x13 ; w  = {k6,k8,s6,s8,k5,k7,s5,s7}
    vsubps     %5, %1, %3
    vblendps   %1, %5, %1, 0x55 ; w -= {0,s7,0,k7,0,s8,0,k8}
    vsubps     %2, %4, %1       ; %2 = v - w
    vaddps     %1, %4, %1       ; %1 = v + w
%endmacro

; In SSE mode do one fft4 transforms
; in:  %1={r0,i0,r2,i2} %2={r1,i1,r3,i3}
; out: %1={r0,r1,r2,r3} %2={i0,i1,i2,i3}
;
; In AVX mode do two fft4 transforms
; in:  %1={r0,i0,r2,i2,r4,i4,r6,i6} %2={r1,i1,r3,i3,r5,i5,r7,i7}
; out: %1={r0,r1,r2,r3,r4,r5,r6,r7} %2={i0,i1,i2,i3,i4,i5,i6,i7}
%macro T4_SSE 3
    subps    %3, %1, %2       ; {t3,t4,-t8,t7}
    addps    %1, %1, %2       ; {t1,t2,t6,t5}
    xorps    %3, %3, [ps_p1p1m1p1]
    shufps   %2, %1, %3, 0xbe ; {t6,t5,t7,t8}
    shufps   %1, %1, %3, 0x44 ; {t1,t2,t3,t4}
    subps    %3, %1, %2       ; {r2,i2,r3,i3}
    addps    %1, %1, %2       ; {r0,i0,r1,i1}
    shufps   %2, %1, %3, 0xdd ; {i0,i1,i2,i3}
    shufps   %1, %1, %3, 0x88 ; {r0,r1,r2,r3}
%endmacro

; In SSE mode do one FFT8
; in:  %1={r0,r1,r2,r3} %2={i0,i1,i2,i3} %3={r4,i4,r6,i6} %4={r5,i5,r7,i7}
; out: %1={r0,r1,r2,r3} %2={i0,i1,i2,i3} %1={r4,r5,r6,r7} %2={i4,i5,i6,i7}
;
; In AVX mode do two FFT8
; in:  %1={r0,i0,r2,i2,r8, i8, r10,i10} %2={r1,i1,r3,i3,r9, i9, r11,i11}
;      %3={r4,i4,r6,i6,r12,i12,r14,i14} %4={r5,i5,r7,i7,r13,i13,r15,i15}
; out: %1={r0,r1,r2,r3,r8, r9, r10,r11} %2={i0,i1,i2,i3,i8, i9, i10,i11}
;      %3={r4,r5,r6,r7,r12,r13,r14,r15} %4={i4,i5,i6,i7,i12,i13,i14,i15}
%macro T8_SSE 6
    addps    %6, %3, %4       ; {t1,t2,t3,t4}
    subps    %3, %3, %4       ; {r5,i5,r7,i7}
    shufps   %4, %3, %3, 0xb1 ; {i5,r5,i7,r7}
    mulps    %3, %3, [ps_root2mppm] ; {-r5,i5,r7,-i7}
    mulps    %4, %4, [ps_root2]
    addps    %3, %3, %4       ; {t8,t7,ta,t9}
    shufps   %4, %6, %3, 0x9c ; {t1,t4,t7,ta}
    shufps   %6, %6, %3, 0x36 ; {t3,t2,t9,t8}
    subps    %3, %6, %4       ; {t6,t5,tc,tb}
    addps    %6, %6, %4       ; {t1,t2,t9,ta}
    shufps   %5, %6, %3, 0x8d ; {t2,ta,t6,tc}
    shufps   %6, %6, %3, 0xd8 ; {t1,t9,t5,tb}
    subps    %3, %1, %6       ; {r4,r5,r6,r7}
    addps    %1, %1, %6       ; {r0,r1,r2,r3}
    subps    %4, %2, %5       ; {i4,i5,i6,i7}
    addps    %2, %2, %5       ; {i0,i1,i2,i3}
%endmacro

%macro INTERL 5
%if cpuflag(avx)
    vunpckhps      %3, %2, %1
    vunpcklps      %2, %2, %1
    vextractf128   %4(%5), %2, 0
    vextractf128  %4 %+ H(%5), %3, 0
    vextractf128   %4(%5 + 1), %2, 1
    vextractf128  %4 %+ H(%5 + 1), %3, 1
%elif cpuflag(sse) || cpuflag(3dnow)
    mova     %3, %2
    unpcklps %2, %1
    unpckhps %3, %1
    mova  %4(%5), %2
    mova  %4(%5+1), %3
%endif
%endmacro

; scheduled for cpu-bound sizes
%macro PASS_SMALL 3 ; (to load m4-m7), wre, wim
IF%1 mova    m4, Z(4)
IF%1 mova    m5, Z(5)
    mova     m0, %2 ; wre
    mova     m1, %3 ; wim
    mulps    m2, m4, m0 ; r2*wre
IF%1 mova    m6, Z2(6)
    mulps    m3, m5, m1 ; i2*wim
IF%1 mova    m7, Z2(7)
    mulps    m4, m4, m1 ; r2*wim
    mulps    m5, m5, m0 ; i2*wre
    addps    m2, m2, m3 ; r2*wre + i2*wim
    mulps    m3, m1, m7 ; i3*wim
    subps    m5, m5, m4 ; i2*wre - r2*wim
    mulps    m1, m1, m6 ; r3*wim
    mulps    m4, m0, m6 ; r3*wre
    mulps    m0, m0, m7 ; i3*wre
    subps    m4, m4, m3 ; r3*wre - i3*wim
    mova     m3, Z(0)
    addps    m0, m0, m1 ; i3*wre + r3*wim
    subps    m1, m4, m2 ; t3
    addps    m4, m4, m2 ; t5
    subps    m3, m3, m4 ; r2
    addps    m4, m4, Z(0) ; r0
    mova     m6, Z(2)
    mova   Z(4), m3
    mova   Z(0), m4
    subps    m3, m5, m0 ; t4
    subps    m4, m6, m3 ; r3
    addps    m3, m3, m6 ; r1
    mova  Z2(6), m4
    mova   Z(2), m3
    mova     m2, Z(3)
    addps    m3, m5, m0 ; t6
    subps    m2, m2, m1 ; i3
    mova     m7, Z(1)
    addps    m1, m1, Z(3) ; i1
    mova  Z2(7), m2
    mova   Z(3), m1
    subps    m4, m7, m3 ; i2
    addps    m3, m3, m7 ; i0
    mova   Z(5), m4
    mova   Z(1), m3
%endmacro

; scheduled to avoid store->load aliasing
%macro PASS_BIG 1 ; (!interleave)
    mova     m4, Z(4) ; r2
    mova     m5, Z(5) ; i2
    mova     m0, [wq] ; wre
    mova     m1, [wq+o1q] ; wim
    mulps    m2, m4, m0 ; r2*wre
    mova     m6, Z2(6) ; r3
    mulps    m3, m5, m1 ; i2*wim
    mova     m7, Z2(7) ; i3
    mulps    m4, m4, m1 ; r2*wim
    mulps    m5, m5, m0 ; i2*wre
    addps    m2, m2, m3 ; r2*wre + i2*wim
    mulps    m3, m1, m7 ; i3*wim
    mulps    m1, m1, m6 ; r3*wim
    subps    m5, m5, m4 ; i2*wre - r2*wim
    mulps    m4, m0, m6 ; r3*wre
    mulps    m0, m0, m7 ; i3*wre
    subps    m4, m4, m3 ; r3*wre - i3*wim
    mova     m3, Z(0)
    addps    m0, m0, m1 ; i3*wre + r3*wim
    subps    m1, m4, m2 ; t3
    addps    m4, m4, m2 ; t5
    subps    m3, m3, m4 ; r2
    addps    m4, m4, Z(0) ; r0
    mova     m6, Z(2)
    mova   Z(4), m3
    mova   Z(0), m4
    subps    m3, m5, m0 ; t4
    subps    m4, m6, m3 ; r3
    addps    m3, m3, m6 ; r1
IF%1 mova Z2(6), m4
IF%1 mova  Z(2), m3
    mova     m2, Z(3)
    addps    m5, m5, m0 ; t6
    subps    m2, m2, m1 ; i3
    mova     m7, Z(1)
    addps    m1, m1, Z(3) ; i1
IF%1 mova Z2(7), m2
IF%1 mova  Z(3), m1
    subps    m6, m7, m5 ; i2
    addps    m5, m5, m7 ; i0
IF%1 mova  Z(5), m6
IF%1 mova  Z(1), m5
%if %1==0
    INTERL m1, m3, m7, Z, 2
    INTERL m2, m4, m0, Z2, 6

    mova     m1, Z(0)
    mova     m2, Z(4)

    INTERL m5, m1, m3, Z, 0
    INTERL m6, m2, m7, Z, 4
%endif
%endmacro

%macro PUNPCK 3
    mova      %3, %1
    punpckldq %1, %2
    punpckhdq %3, %2
%endmacro

%define Z(x) [r0+mmsize*x]
%define Z2(x) [r0+mmsize*x]
%define ZH(x) [r0+mmsize*x+mmsize/2]

INIT_YMM avx

%if HAVE_AVX_EXTERNAL
align 16
fft8_avx:
    mova      m0, Z(0)
    mova      m1, Z(1)
    T8_AVX    m0, m1, m2, m3, m4
    mova      Z(0), m0
    mova      Z(1), m1
    ret


align 16
fft16_avx:
    mova       m2, Z(2)
    mova       m3, Z(3)
    T4_SSE     m2, m3, m7

    mova       m0, Z(0)
    mova       m1, Z(1)
    T8_AVX     m0, m1, m4, m5, m7

    mova       m4, [ps_cos16_1]
    mova       m5, [ps_cos16_2]
    vmulps     m6, m2, m4
    vmulps     m7, m3, m5
    vaddps     m7, m7, m6
    vmulps     m2, m2, m5
    vmulps     m3, m3, m4
    vsubps     m3, m3, m2
    vblendps   m2, m7, m3, 0xf0
    vperm2f128 m3, m7, m3, 0x21
    vaddps     m4, m2, m3
    vsubps     m2, m3, m2
    vperm2f128 m2, m2, m2, 0x01
    vsubps     m3, m1, m2
    vaddps     m1, m1, m2
    vsubps     m5, m0, m4
    vaddps     m0, m0, m4
    vextractf128   Z(0), m0, 0
    vextractf128  ZH(0), m1, 0
    vextractf128   Z(1), m0, 1
    vextractf128  ZH(1), m1, 1
    vextractf128   Z(2), m5, 0
    vextractf128  ZH(2), m3, 0
    vextractf128   Z(3), m5, 1
    vextractf128  ZH(3), m3, 1
    ret

align 16
fft32_avx:
    call fft16_avx

    mova m0, Z(4)
    mova m1, Z(5)

    T4_SSE      m0, m1, m4

    mova m2, Z(6)
    mova m3, Z(7)

    T8_SSE      m0, m1, m2, m3, m4, m6
    ; m0={r0,r1,r2,r3,r8, r9, r10,r11} m1={i0,i1,i2,i3,i8, i9, i10,i11}
    ; m2={r4,r5,r6,r7,r12,r13,r14,r15} m3={i4,i5,i6,i7,i12,i13,i14,i15}

    vperm2f128  m4, m0, m2, 0x20
    vperm2f128  m5, m1, m3, 0x20
    vperm2f128  m6, m0, m2, 0x31
    vperm2f128  m7, m1, m3, 0x31

    PASS_SMALL 0, [cos_32], [cos_32+32]

    ret

fft32_interleave_avx:
    call fft32_avx
    mov r2d, 32
.deint_loop:
    mova     m2, Z(0)
    mova     m3, Z(1)
    vunpcklps      m0, m2, m3
    vunpckhps      m1, m2, m3
    vextractf128   Z(0), m0, 0
    vextractf128  ZH(0), m1, 0
    vextractf128   Z(1), m0, 1
    vextractf128  ZH(1), m1, 1
    add r0, mmsize*2
    sub r2d, mmsize/4
    jg .deint_loop
    ret

%endif

INIT_XMM sse

align 16
fft4_avx:
fft4_sse:
    mova     m0, Z(0)
    mova     m1, Z(1)
    T4_SSE   m0, m1, m2
    mova   Z(0), m0
    mova   Z(1), m1
    ret

align 16
fft8_sse:
    mova     m0, Z(0)
    mova     m1, Z(1)
    T4_SSE   m0, m1, m2
    mova     m2, Z(2)
    mova     m3, Z(3)
    T8_SSE   m0, m1, m2, m3, m4, m5
    mova   Z(0), m0
    mova   Z(1), m1
    mova   Z(2), m2
    mova   Z(3), m3
    ret

align 16
fft16_sse:
    mova     m0, Z(0)
    mova     m1, Z(1)
    T4_SSE   m0, m1, m2
    mova     m2, Z(2)
    mova     m3, Z(3)
    T8_SSE   m0, m1, m2, m3, m4, m5
    mova     m4, Z(4)
    mova     m5, Z(5)
    mova   Z(0), m0
    mova   Z(1), m1
    mova   Z(2), m2
    mova   Z(3), m3
    T4_SSE   m4, m5, m6
    mova     m6, Z2(6)
    mova     m7, Z2(7)
    T4_SSE   m6, m7, m0
    PASS_SMALL 0, [cos_16], [cos_16+16]
    ret


%macro FFT48_3DNOW 0
align 16
fft4 %+ SUFFIX:
    T2_3DNOW m0, m1, Z(0), Z(1)
    mova     m2, Z(2)
    mova     m3, Z(3)
    T4_3DNOW m0, m1, m2, m3, m4, m5
    PUNPCK   m0, m1, m4
    PUNPCK   m2, m3, m5
    mova   Z(0), m0
    mova   Z(1), m4
    mova   Z(2), m2
    mova   Z(3), m5
    ret

align 16
fft8 %+ SUFFIX:
    T2_3DNOW m0, m1, Z(0), Z(1)
    mova     m2, Z(2)
    mova     m3, Z(3)
    T4_3DNOW m0, m1, m2, m3, m4, m5
    mova   Z(0), m0
    mova   Z(2), m2
    T2_3DNOW m4, m5,  Z(4),  Z(5)
    T2_3DNOW m6, m7, Z2(6), Z2(7)
    PSWAPD   m0, m5
    PSWAPD   m2, m7
    pxor     m0, [ps_m1p1]
    pxor     m2, [ps_m1p1]
    pfsub    m5, m0
    pfadd    m7, m2
    pfmul    m5, [ps_root2]
    pfmul    m7, [ps_root2]
    T4_3DNOW m1, m3, m5, m7, m0, m2
    mova   Z(5), m5
    mova  Z2(7), m7
    mova     m0, Z(0)
    mova     m2, Z(2)
    T4_3DNOW m0, m2, m4, m6, m5, m7
    PUNPCK   m0, m1, m5
    PUNPCK   m2, m3, m7
    mova   Z(0), m0
    mova   Z(1), m5
    mova   Z(2), m2
    mova   Z(3), m7
    PUNPCK   m4,  Z(5), m5
    PUNPCK   m6, Z2(7), m7
    mova   Z(4), m4
    mova   Z(5), m5
    mova  Z2(6), m6
    mova  Z2(7), m7
    ret
%endmacro

%if ARCH_X86_32
INIT_MMX 3dnowext
FFT48_3DNOW

INIT_MMX 3dnow
FFT48_3DNOW
%endif

%define Z(x) [zcq + o1q*(x&6) + mmsize*(x&1)]
%define Z2(x) [zcq + o3q + mmsize*(x&1)]
%define ZH(x) [zcq + o1q*(x&6) + mmsize*(x&1) + mmsize/2]
%define Z2H(x) [zcq + o3q + mmsize*(x&1) + mmsize/2]

%macro DECL_PASS 2+ ; name, payload
align 16
%1:
DEFINE_ARGS zc, w, n, o1, o3
    lea o3q, [nq*3]
    lea o1q, [nq*8]
    shl o3q, 4
.loop:
    %2
    add zcq, mmsize*2
    add  wq, mmsize
    sub  nd, mmsize/8
    jg .loop
    rep ret
%endmacro

%macro FFT_DISPATCH 2; clobbers 5 GPRs, 8 XMMs
    lea r2, [dispatch_tab%1]
    mov r2, [r2 + (%2q-2)*gprsize]
%ifdef PIC
    lea r3, [$$]
    add r2, r3
%endif
    call r2
%endmacro ; FFT_DISPATCH

INIT_YMM avx

%if HAVE_AVX_EXTERNAL
DECL_PASS pass_avx, PASS_BIG 1
DECL_PASS pass_interleave_avx, PASS_BIG 0

cglobal fft_calc, 2,5,8
    mov     r3d, [r0 + FFTContext.nbits]
    mov     r0, r1
    mov     r1, r3
    FFT_DISPATCH _interleave %+ SUFFIX, r1
    REP_RET

%endif

INIT_XMM sse

DECL_PASS pass_sse, PASS_BIG 1
DECL_PASS pass_interleave_sse, PASS_BIG 0

%macro FFT_CALC_FUNC 0
cglobal fft_calc, 2,5,8
    mov     r3d, [r0 + FFTContext.nbits]
    PUSH    r1
    PUSH    r3
    mov     r0, r1
    mov     r1, r3
    FFT_DISPATCH _interleave %+ SUFFIX, r1
    POP     rcx
    POP     r4
    cmp     rcx, 3+(mmsize/16)
    jg      .end
    mov     r2, -1
    add     rcx, 3
    shl     r2, cl
    sub     r4, r2
.loop:
%if mmsize == 8
    PSWAPD  m0, [r4 + r2 + 4]
    mova [r4 + r2 + 4], m0
%else
    movaps   xmm0, [r4 + r2]
    movaps   xmm1, xmm0
    unpcklps xmm0, [r4 + r2 + 16]
    unpckhps xmm1, [r4 + r2 + 16]
    movaps   [r4 + r2],      xmm0
    movaps   [r4 + r2 + 16], xmm1
%endif
    add      r2, mmsize*2
    jl       .loop
.end:
%if cpuflag(3dnow)
    femms
    RET
%else
    REP_RET
%endif
%endmacro

%if ARCH_X86_32
INIT_MMX 3dnow
FFT_CALC_FUNC
INIT_MMX 3dnowext
FFT_CALC_FUNC
%endif
INIT_XMM sse
FFT_CALC_FUNC

cglobal fft_permute, 2,7,1
    mov     r4,  [r0 + FFTContext.revtab]
    mov     r5,  [r0 + FFTContext.tmpbuf]
    mov     ecx, [r0 + FFTContext.nbits]
    mov     r2, 1
    shl     r2, cl
    xor     r0, r0
%if ARCH_X86_32
    mov     r1, r1m
%endif
.loop:
    movaps  xmm0, [r1 + 8*r0]
    movzx   r6, word [r4 + 2*r0]
    movzx   r3, word [r4 + 2*r0 + 2]
    movlps  [r5 + 8*r6], xmm0
    movhps  [r5 + 8*r3], xmm0
    add     r0, 2
    cmp     r0, r2
    jl      .loop
    shl     r2, 3
    add     r1, r2
    add     r5, r2
    neg     r2
; nbits >= 2 (FFT4) and sizeof(FFTComplex)=8 => at least 32B
.loopcopy:
    movaps  xmm0, [r5 + r2]
    movaps  xmm1, [r5 + r2 + 16]
    movaps  [r1 + r2], xmm0
    movaps  [r1 + r2 + 16], xmm1
    add     r2, 32
    jl      .loopcopy
    REP_RET

%macro IMDCT_CALC_FUNC 0
cglobal imdct_calc, 3,5,3
    mov     r3d, [r0 + FFTContext.mdctsize]
    mov     r4,  [r0 + FFTContext.imdcthalf]
    add     r1,  r3
    PUSH    r3
    PUSH    r1
%if ARCH_X86_32
    push    r2
    push    r1
    push    r0
%else
    sub     rsp, 8+32*WIN64 ; allocate win64 shadow space
%endif
    call    r4
%if ARCH_X86_32
    add     esp, 12
%else
    add     rsp, 8+32*WIN64
%endif
    POP     r1
    POP     r3
    lea     r0, [r1 + 2*r3]
    mov     r2, r3
    sub     r3, mmsize
    neg     r2
    mova    m2, [ps_neg]
.loop:
%if mmsize == 8
    PSWAPD  m0, [r1 + r3]
    PSWAPD  m1, [r0 + r2]
    pxor    m0, m2
%else
    mova    m0, [r1 + r3]
    mova    m1, [r0 + r2]
    shufps  m0, m0, 0x1b
    shufps  m1, m1, 0x1b
    xorps   m0, m2
%endif
    mova [r0 + r3], m1
    mova [r1 + r2], m0
    sub     r3, mmsize
    add     r2, mmsize
    jl      .loop
%if cpuflag(3dnow)
    femms
    RET
%else
    REP_RET
%endif
%endmacro

%if ARCH_X86_32
INIT_MMX 3dnow
IMDCT_CALC_FUNC
INIT_MMX 3dnowext
IMDCT_CALC_FUNC
%endif

INIT_XMM sse
IMDCT_CALC_FUNC

%if ARCH_X86_32
INIT_MMX 3dnow
%define mulps pfmul
%define addps pfadd
%define subps pfsub
%define unpcklps punpckldq
%define unpckhps punpckhdq
DECL_PASS pass_3dnow, PASS_SMALL 1, [wq], [wq+o1q]
DECL_PASS pass_interleave_3dnow, PASS_BIG 0
%define pass_3dnowext pass_3dnow
%define pass_interleave_3dnowext pass_interleave_3dnow
%endif

%ifdef PIC
%define SECTION_REL - $$
%else
%define SECTION_REL
%endif

%macro DECL_FFT 1-2 ; nbits, suffix
%ifidn %0, 1
%xdefine fullsuffix SUFFIX
%else
%xdefine fullsuffix %2 %+ SUFFIX
%endif
%xdefine list_of_fft fft4 %+ SUFFIX SECTION_REL, fft8 %+ SUFFIX SECTION_REL
%if %1>=5
%xdefine list_of_fft list_of_fft, fft16 %+ SUFFIX SECTION_REL
%endif
%if %1>=6
%xdefine list_of_fft list_of_fft, fft32 %+ fullsuffix SECTION_REL
%endif

%assign n 1<<%1
%rep 18-%1
%assign n2 n/2
%assign n4 n/4
%xdefine list_of_fft list_of_fft, fft %+ n %+ fullsuffix SECTION_REL

align 16
fft %+ n %+ fullsuffix:
    call fft %+ n2 %+ SUFFIX
    add r0, n*4 - (n&(-2<<%1))
    call fft %+ n4 %+ SUFFIX
    add r0, n*2 - (n2&(-2<<%1))
    call fft %+ n4 %+ SUFFIX
    sub r0, n*6 + (n2&(-2<<%1))
    lea r1, [cos_ %+ n]
    mov r2d, n4/2
    jmp pass %+ fullsuffix

%assign n n*2
%endrep
%undef n

align 8
dispatch_tab %+ fullsuffix: pointer list_of_fft
%endmacro ; DECL_FFT

%if HAVE_AVX_EXTERNAL
INIT_YMM avx
DECL_FFT 6
DECL_FFT 6, _interleave
%endif
INIT_XMM sse
DECL_FFT 5
DECL_FFT 5, _interleave
%if ARCH_X86_32
INIT_MMX 3dnow
DECL_FFT 4
DECL_FFT 4, _interleave
INIT_MMX 3dnowext
DECL_FFT 4
DECL_FFT 4, _interleave
%endif

INIT_XMM sse
%undef mulps
%undef addps
%undef subps
%undef unpcklps
%undef unpckhps

%macro PREROTATER 5 ;-2*k, 2*k, input+n4, tcos+n8, tsin+n8
%if mmsize == 8 ; j*2+2-n4, n4-2-j*2, input+n4, tcos+n8, tsin+n8
    PSWAPD     m0, [%3+%2*4]
    movq       m2, [%3+%1*4-8]
    movq       m3, m0
    punpckldq  m0, m2
    punpckhdq  m2, m3
    movd       m1, [%4+%1*2-4] ; tcos[j]
    movd       m3, [%4+%2*2]   ; tcos[n4-j-1]
    punpckldq  m1, [%5+%1*2-4] ; tsin[j]
    punpckldq  m3, [%5+%2*2]   ; tsin[n4-j-1]

    mova       m4, m0
    PSWAPD     m5, m1
    pfmul      m0, m1
    pfmul      m4, m5
    mova       m6, m2
    PSWAPD     m5, m3
    pfmul      m2, m3
    pfmul      m6, m5
%if cpuflag(3dnowext)
    pfpnacc    m0, m4
    pfpnacc    m2, m6
%else
    SBUTTERFLY dq, 0, 4, 1
    SBUTTERFLY dq, 2, 6, 3
    pxor       m4, m7
    pxor       m6, m7
    pfadd      m0, m4
    pfadd      m2, m6
%endif
%else
    movaps   xmm0, [%3+%2*4]
    movaps   xmm1, [%3+%1*4-0x10]
    movaps   xmm2, xmm0
    shufps   xmm0, xmm1, 0x88
    shufps   xmm1, xmm2, 0x77
    movlps   xmm4, [%4+%2*2]
    movlps   xmm5, [%5+%2*2+0x0]
    movhps   xmm4, [%4+%1*2-0x8]
    movhps   xmm5, [%5+%1*2-0x8]
    movaps   xmm2, xmm0
    movaps   xmm3, xmm1
    mulps    xmm0, xmm5
    mulps    xmm1, xmm4
    mulps    xmm2, xmm4
    mulps    xmm3, xmm5
    subps    xmm1, xmm0
    addps    xmm2, xmm3
    movaps   xmm0, xmm1
    unpcklps xmm1, xmm2
    unpckhps xmm0, xmm2
%endif
%endmacro

%macro CMUL 6 ;j, xmm0, xmm1, 3, 4, 5
%if cpuflag(sse)
    mulps      m6, %3, [%5+%1]
    mulps      m7, %2, [%5+%1]
    mulps      %2, %2, [%6+%1]
    mulps      %3, %3, [%6+%1]
    subps      %2, %2, m6
    addps      %3, %3, m7
%elif cpuflag(3dnow)
    mova       m6, [%1+%2*2]
    mova       %3, [%1+%2*2+8]
    mova       %4, m6
    mova       m7, %3
    pfmul      m6, [%5+%2]
    pfmul      %3, [%6+%2]
    pfmul      %4, [%6+%2]
    pfmul      m7, [%5+%2]
    pfsub      %3, m6
    pfadd      %4, m7
%endif
%endmacro

%macro POSROTATESHUF 5 ;j, k, z+n8, tcos+n8, tsin+n8
.post:
%if cpuflag(avx)
    vmovaps      ymm1,   [%3+%1*2]
    vmovaps      ymm0,   [%3+%1*2+0x20]
    vmovaps      ymm3,   [%3+%2*2]
    vmovaps      ymm2,   [%3+%2*2+0x20]

    CMUL         %1, ymm0, ymm1, %3, %4, %5
    CMUL         %2, ymm2, ymm3, %3, %4, %5
    vshufps      ymm1, ymm1, ymm1, 0x1b
    vshufps      ymm3, ymm3, ymm3, 0x1b
    vperm2f128   ymm1, ymm1, ymm1, 0x01
    vperm2f128   ymm3, ymm3, ymm3, 0x01
    vunpcklps    ymm6, ymm2, ymm1
    vunpckhps    ymm4, ymm2, ymm1
    vunpcklps    ymm7, ymm0, ymm3
    vunpckhps    ymm5, ymm0, ymm3

    vextractf128 [%3+%1*2],      ymm7, 0
    vextractf128 [%3+%1*2+0x10], ymm5, 0
    vextractf128 [%3+%1*2+0x20], ymm7, 1
    vextractf128 [%3+%1*2+0x30], ymm5, 1

    vextractf128 [%3+%2*2],      ymm6, 0
    vextractf128 [%3+%2*2+0x10], ymm4, 0
    vextractf128 [%3+%2*2+0x20], ymm6, 1
    vextractf128 [%3+%2*2+0x30], ymm4, 1
    sub      %2,   0x20
    add      %1,   0x20
    jl       .post
%elif cpuflag(sse)
    movaps   xmm1, [%3+%1*2]
    movaps   xmm0, [%3+%1*2+0x10]
    CMUL     %1,   xmm0, xmm1, %3, %4, %5
    movaps   xmm5, [%3+%2*2]
    movaps   xmm4, [%3+%2*2+0x10]
    CMUL     %2,   xmm4, xmm5, %3, %4, %5
    shufps   xmm1, xmm1, 0x1b
    shufps   xmm5, xmm5, 0x1b
    movaps   xmm6, xmm4
    unpckhps xmm4, xmm1
    unpcklps xmm6, xmm1
    movaps   xmm2, xmm0
    unpcklps xmm0, xmm5
    unpckhps xmm2, xmm5
    movaps   [%3+%2*2],      xmm6
    movaps   [%3+%2*2+0x10], xmm4
    movaps   [%3+%1*2],      xmm0
    movaps   [%3+%1*2+0x10], xmm2
    sub      %2,   0x10
    add      %1,   0x10
    jl       .post
%elif cpuflag(3dnow)
    CMUL  %3, %1, m0, m1, %4, %5
    CMUL  %3, %2, m2, m3, %4, %5
    movd  [%3+%1*2+ 0], m0
    movd  [%3+%2*2+12], m1
    movd  [%3+%2*2+ 0], m2
    movd  [%3+%1*2+12], m3
    psrlq      m0, 32
    psrlq      m1, 32
    psrlq      m2, 32
    psrlq      m3, 32
    movd  [%3+%1*2+ 8], m0
    movd  [%3+%2*2+ 4], m1
    movd  [%3+%2*2+ 8], m2
    movd  [%3+%1*2+ 4], m3
    sub        %2, 8
    add        %1, 8
    jl         .post
%endif
%endmacro

%macro DECL_IMDCT 0
cglobal imdct_half, 3,12,8; FFTContext *s, FFTSample *output, const FFTSample *input
%if ARCH_X86_64
%define rrevtab r7
%define rtcos   r8
%define rtsin   r9
%else
%define rrevtab r6
%define rtsin   r6
%define rtcos   r5
%endif
    mov   r3d, [r0+FFTContext.mdctsize]
    add   r2, r3
    shr   r3, 1
    mov   rtcos, [r0+FFTContext.tcos]
    mov   rtsin, [r0+FFTContext.tsin]
    add   rtcos, r3
    add   rtsin, r3
%if ARCH_X86_64 == 0
    push  rtcos
    push  rtsin
%endif
    shr   r3, 1
    mov   rrevtab, [r0+FFTContext.revtab]
    add   rrevtab, r3
%if ARCH_X86_64 == 0
    push  rrevtab
%endif

%if mmsize == 8
    sub   r3, 2
%else
    sub   r3, 4
%endif
%if ARCH_X86_64 || mmsize == 8
    xor   r4, r4
    sub   r4, r3
%endif
%if notcpuflag(3dnowext) && mmsize == 8
    movd  m7, [ps_neg]
%endif
.pre:
%if ARCH_X86_64 == 0
;unspill
%if mmsize != 8
    xor   r4, r4
    sub   r4, r3
%endif
    mov   rtcos, [esp+8]
    mov   rtsin, [esp+4]
%endif

    PREROTATER r4, r3, r2, rtcos, rtsin
%if mmsize == 8
    mov    r6, [esp]                ; rrevtab = ptr+n8
    movzx  r5,  word [rrevtab+r4-2] ; rrevtab[j]
    movzx  r6,  word [rrevtab+r3]   ; rrevtab[n4-j-1]
    mova [r1+r5*8], m0
    mova [r1+r6*8], m2
    add    r4, 2
    sub    r3, 2
%else
%if ARCH_X86_64
    movzx  r5,  word [rrevtab+r4-4]
    movzx  r6,  word [rrevtab+r4-2]
    movzx  r10, word [rrevtab+r3]
    movzx  r11, word [rrevtab+r3+2]
    movlps [r1+r5 *8], xmm0
    movhps [r1+r6 *8], xmm0
    movlps [r1+r10*8], xmm1
    movhps [r1+r11*8], xmm1
    add    r4, 4
%else
    mov    r6, [esp]
    movzx  r5, word [r6+r4-4]
    movzx  r4, word [r6+r4-2]
    movlps [r1+r5*8], xmm0
    movhps [r1+r4*8], xmm0
    movzx  r5, word [r6+r3]
    movzx  r4, word [r6+r3+2]
    movlps [r1+r5*8], xmm1
    movhps [r1+r4*8], xmm1
%endif
    sub    r3, 4
%endif
    jns    .pre

    mov  r5, r0
    mov  r6, r1
    mov  r0, r1
    mov  r1d, [r5+FFTContext.nbits]

    FFT_DISPATCH SUFFIX, r1

    mov  r0d, [r5+FFTContext.mdctsize]
    add  r6, r0
    shr  r0, 1
%if ARCH_X86_64 == 0
%define rtcos r2
%define rtsin r3
    mov  rtcos, [esp+8]
    mov  rtsin, [esp+4]
%endif
    neg  r0
    mov  r1, -mmsize
    sub  r1, r0
    POSROTATESHUF r0, r1, r6, rtcos, rtsin
%if ARCH_X86_64 == 0
    add esp, 12
%endif
%if mmsize == 8
    femms
%endif
    RET
%endmacro

DECL_IMDCT

%if ARCH_X86_32
INIT_MMX 3dnow
DECL_IMDCT

INIT_MMX 3dnowext
DECL_IMDCT
%endif

INIT_YMM avx

%if HAVE_AVX_EXTERNAL
DECL_IMDCT
%endif