arithm.simd.hpp 59.4 KB

Edit Raw Blame History

// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html
#include "opencv2/core/hal/intrin.hpp"
//=========================================
// Declare & Define & Dispatch in one step
//=========================================
// ARITHM_DISPATCHING_ONLY defined by arithm dispatch file
#undef ARITHM_DECLARATIONS_ONLY
#ifdef CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY
    #define ARITHM_DECLARATIONS_ONLY
#endif
#undef ARITHM_DEFINITIONS_ONLY
#if !defined(CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY) && !defined(ARITHM_DISPATCHING_ONLY)
    #define ARITHM_DEFINITIONS_ONLY
#endif
#ifdef ARITHM_DECLARATIONS_ONLY
    #undef DEFINE_SIMD
    #define DEFINE_SIMD(fun_name, c_type, ...) \
        DECLARE_SIMD_FUN(fun_name, c_type)
#endif // ARITHM_DECLARATIONS_ONLY
#ifdef ARITHM_DEFINITIONS_ONLY
    #undef DEFINE_SIMD
    #define DEFINE_SIMD(fun_name, c_type, v_type, ...)          \
        DECLARE_SIMD_FUN(fun_name, c_type)                      \
        DEFINE_SIMD_FUN(fun_name, c_type, v_type, __VA_ARGS__)
#endif // ARITHM_DEFINITIONS_ONLY
#ifdef ARITHM_DISPATCHING_ONLY
    #undef DEFINE_SIMD
    #define DEFINE_SIMD(fun_name, c_type, v_type, ...)           \
        DISPATCH_SIMD_FUN(fun_name, c_type, v_type, __VA_ARGS__)
#endif // ARITHM_DISPATCHING_ONLY
// workaround when neon miss support of double precision
#undef DEFINE_NOSIMD
#ifdef ARITHM_DEFINITIONS_ONLY
    #define DEFINE_NOSIMD(fun_name, c_type, ...)          \
        DECLARE_SIMD_FUN(fun_name, c_type)                \
        DEFINE_NOSIMD_FUN(fun_name, c_type, __VA_ARGS__)
#else
    #define DEFINE_NOSIMD DEFINE_SIMD
#endif // ARITHM_DEFINITIONS_ONLY
#ifndef SIMD_GUARD
#define DEFINE_SIMD_U8(fun, ...) \
    DEFINE_SIMD(__CV_CAT(fun, 8u), uchar, v_uint8, __VA_ARGS__)
#define DEFINE_SIMD_S8(fun, ...) \
    DEFINE_SIMD(__CV_CAT(fun, 8s), schar, v_int8,  __VA_ARGS__)
#define DEFINE_SIMD_U16(fun, ...) \
    DEFINE_SIMD(__CV_CAT(fun, 16u), ushort, v_uint16, __VA_ARGS__)
#define DEFINE_SIMD_S16(fun, ...) \
    DEFINE_SIMD(__CV_CAT(fun, 16s), short, v_int16,  __VA_ARGS__)
#define DEFINE_SIMD_S32(fun, ...) \
    DEFINE_SIMD(__CV_CAT(fun, 32s), int, v_int32,  __VA_ARGS__)
#define DEFINE_SIMD_F32(fun, ...) \
    DEFINE_SIMD(__CV_CAT(fun, 32f), float, v_float32, __VA_ARGS__)
#if CV_SIMD_64F
    #define DEFINE_SIMD_F64(fun, ...) \
        DEFINE_SIMD(__CV_CAT(fun, 64f), double, v_float64, __VA_ARGS__)
#else
    #define DEFINE_SIMD_F64(fun, ...) \
        DEFINE_NOSIMD(__CV_CAT(fun, 64f), double, __VA_ARGS__)
#endif
#define DEFINE_SIMD_SAT(fun, ...)      \
    DEFINE_SIMD_U8(fun, __VA_ARGS__)   \
    DEFINE_SIMD_S8(fun, __VA_ARGS__)   \
    DEFINE_SIMD_U16(fun, __VA_ARGS__)  \
    DEFINE_SIMD_S16(fun, __VA_ARGS__)
#define DEFINE_SIMD_NSAT(fun, ...)     \
    DEFINE_SIMD_S32(fun, __VA_ARGS__)  \
    DEFINE_SIMD_F32(fun, __VA_ARGS__)  \
    DEFINE_SIMD_F64(fun, __VA_ARGS__)
#define DEFINE_SIMD_ALL(fun, ...)      \
    DEFINE_SIMD_SAT(fun, __VA_ARGS__)  \
    DEFINE_SIMD_NSAT(fun, __VA_ARGS__)
#endif // SIMD_GUARD
///////////////////////////////////////////////////////////////////////////
namespace cv { namespace hal {

#ifndef ARITHM_DISPATCHING_ONLY
    CV_CPU_OPTIMIZATION_NAMESPACE_BEGIN
#endif
#ifdef ARITHM_DEFINITIONS_ONLY
#if !CV_SIMD_64F
typedef int v_float64; // dummy
#endif
//=======================================
// Utility
//=======================================
/** add **/
template<typename T>
static inline T c_add(T a, T b)
{ return saturate_cast<T>(a + b); }
template<>
inline uchar c_add<uchar>(uchar a, uchar b)
{ return CV_FAST_CAST_8U(a + b); }
// scale
template<typename T1, typename T2>
static inline T1 c_add(T1 a, T1 b, T2 scalar)
{ return saturate_cast<T1>((T2)a * scalar + b); }
template<>
inline uchar c_add<uchar, float>(uchar a, uchar b, float scalar)
{ return saturate_cast<uchar>(CV_8TO32F(a) * scalar + b); }
// weight
template<typename T1, typename T2>
static inline T1 c_add(T1 a, T1 b, T2 alpha, T2 beta, T2 gamma)
{ return saturate_cast<T1>(a * alpha + b * beta + gamma); }
template<>
inline uchar c_add<uchar, float>(uchar a, uchar b, float alpha, float beta, float gamma)
{ return saturate_cast<uchar>(CV_8TO32F(a) * alpha + CV_8TO32F(b) * beta + gamma); }

/** sub **/
template<typename T>
static inline T c_sub(T a, T b)
{ return saturate_cast<T>(a - b); }
template<>
inline uchar c_sub<uchar>(uchar a, uchar b)
{ return CV_FAST_CAST_8U(a - b); }

/** max **/
template<typename T>
static inline T c_max(T a, T b)
{ return std::max(a, b); }
template<>
inline uchar c_max<uchar>(uchar a, uchar b)
{ return CV_MAX_8U(a, b); }

/** min **/
template<typename T>
static inline T c_min(T a, T b)
{ return std::min(a, b); }
template<>
inline uchar c_min<uchar>(uchar a, uchar b)
{ return CV_MIN_8U(a, b); }

/** absdiff **/
template<typename T>
static inline T c_absdiff(T a, T b)
{ return a > b ? a - b : b - a; }
template<>
inline schar c_absdiff(schar a, schar b)
{ return saturate_cast<schar>(std::abs(a - b)); }
template<>
inline short c_absdiff(short a, short b)
{ return saturate_cast<short>(std::abs(a - b)); }
// specializations to prevent "-0" results
template<>
inline float c_absdiff<float>(float a, float b)
{ return std::abs(a - b); }
template<>
inline double c_absdiff<double>(double a, double b)
{ return std::abs(a - b); }

/** multiply **/
template<typename T>
static inline T c_mul(T a, T b)
{ return saturate_cast<T>(a * b); }
template<>
inline uchar c_mul<uchar>(uchar a, uchar b)
{ return CV_FAST_CAST_8U(a * b); }
// scale
template<typename T1, typename T2>
static inline T1 c_mul(T1 a, T1 b, T2 scalar)
{ return saturate_cast<T1>(scalar * (T2)a * b); }
template<>
inline uchar c_mul<uchar, float>(uchar a, uchar b, float scalar)
{ return saturate_cast<uchar>(scalar * CV_8TO32F(a) * CV_8TO32F(b)); }

/** divide & reciprocal **/
template<typename T1, typename T2>
static inline T2 c_div(T1 a, T2 b)
{ return saturate_cast<T2>(a / b); }
// recip
template<>
inline uchar c_div<float, uchar>(float a, uchar b)
{ return saturate_cast<uchar>(a / CV_8TO32F(b)); }
// scale
template<typename T1, typename T2>
static inline T1 c_div(T1 a, T1 b, T2 scalar)
{ return saturate_cast<T1>(scalar * (T2)a / b); }
template<>
inline uchar c_div<uchar, float>(uchar a, uchar b, float scalar)
{ return saturate_cast<uchar>(scalar * CV_8TO32F(a) / CV_8TO32F(b)); }

//=======================================
// Arithmetic and logical operations
// +, -, *, /, &, |, ^, ~, abs ...
//=======================================
///////////////////////////// Operations //////////////////////////////////
// Add
template<typename T1, typename Tvec>
struct op_add
{
    static inline Tvec r(const Tvec& a, const Tvec& b)
    { return a + b; }
    static inline T1 r(T1 a, T1 b)
    { return c_add(a, b); }
};

// Subtract
template<typename T1, typename Tvec>
struct op_sub
{
    static inline Tvec r(const Tvec& a, const Tvec& b)
    { return a - b; }
    static inline T1 r(T1 a, T1 b)
    { return c_sub(a, b); }
};

// Max & Min
template<typename T1, typename Tvec>
struct op_max
{
    static inline Tvec r(const Tvec& a, const Tvec& b)
    { return v_max(a, b); }
    static inline T1 r(T1 a, T1 b)
    { return c_max(a, b); }
};

template<typename T1, typename Tvec>
struct op_min
{
    static inline Tvec r(const Tvec& a, const Tvec& b)
    { return v_min(a, b); }
    static inline T1 r(T1 a, T1 b)
    { return c_min(a, b); }
};

// Absolute difference
template<typename T1, typename Tvec>
struct op_absdiff
{
    static inline Tvec r(const Tvec& a, const Tvec& b)
    { return v_absdiff(a, b); }
    static inline T1 r(T1 a, T1 b)
    { return c_absdiff(a, b); }
};
// Signed absolute difference, 's'
template<>
struct op_absdiff<schar, v_int8>
{
    static inline v_int8 r(const v_int8& a, const v_int8& b)
    { return v_absdiffs(a, b); }
    static inline schar r(schar a, schar b)
    { return c_absdiff(a, b); }
};
template<>
struct op_absdiff<short, v_int16>
{
    static inline v_int16 r(const v_int16& a, const v_int16& b)
    { return v_absdiffs(a, b); }
    static inline short r(short a, short b)
    { return c_absdiff(a, b); }
};
template<>
struct op_absdiff<int, v_int32>
{
    static inline v_int32 r(const v_int32& a, const v_int32& b)
    { return v_reinterpret_as_s32(v_absdiff(a, b)); }
    static inline int r(int a, int b)
    { return c_absdiff(a, b); }
};

// Logical
template<typename T1, typename Tvec>
struct op_or
{
    static inline Tvec r(const Tvec& a, const Tvec& b)
    { return a | b; }
    static inline T1 r(T1 a, T1 b)
    { return a | b; }
};
template<typename T1, typename Tvec>
struct op_xor
{
    static inline Tvec r(const Tvec& a, const Tvec& b)
    { return a ^ b; }
    static inline T1 r(T1 a, T1 b)
    { return a ^ b; }
};
template<typename T1, typename Tvec>
struct op_and
{
    static inline Tvec r(const Tvec& a, const Tvec& b)
    { return a & b; }
    static inline T1 r(T1 a, T1 b)
    { return a & b; }
};
template<typename T1, typename Tvec>
struct op_not
{
    // ignored b from loader level
    static inline Tvec r(const Tvec& a)
    { return ~a; }
    static inline T1 r(T1 a, T1)
    { return ~a; }
};

//////////////////////////// Loaders /////////////////////////////////
#if CV_SIMD
template< template<typename T1, typename Tvec> class OP, typename T1, typename Tvec>
struct bin_loader
{
    typedef OP<T1, Tvec> op;

    static inline void l(const T1* src1, const T1* src2, T1* dst)
    {
        Tvec a = vx_load(src1);
        Tvec b = vx_load(src2);
        v_store(dst, op::r(a, b));
    }

    static inline void la(const T1* src1, const T1* src2, T1* dst)
    {
        Tvec a = vx_load_aligned(src1);
        Tvec b = vx_load_aligned(src2);
        v_store_aligned(dst, op::r(a, b)); // todo: try write without cache
    }

    static inline void l64(const T1* src1, const T1* src2, T1* dst)
    {
        Tvec a = vx_load_low(src1), b = vx_load_low(src2);
        v_store_low(dst, op::r(a, b));
    }
};

// void src2 for operation "not"
template<typename T1, typename Tvec>
struct bin_loader<op_not, T1, Tvec>
{
    typedef op_not<T1, Tvec> op;

    static inline void l(const T1* src1, const T1*, T1* dst)
    {
        Tvec a = vx_load(src1);
        v_store(dst, op::r(a));
    }

    static inline void la(const T1* src1, const T1*, T1* dst)
    {
        Tvec a = vx_load_aligned(src1);
        v_store_aligned(dst, op::r(a));
    }

    static inline void l64(const T1* src1, const T1*, T1* dst)
    {
        Tvec a = vx_load_low(src1);
        v_store_low(dst, op::r(a));
    }
};

#endif // CV_SIMD
//////////////////////////// Loops /////////////////////////////////
template<typename T1, typename T2>
static inline bool is_aligned(const T1* src1, const T1* src2, const T2* dst)
{ return (((size_t)src1|(size_t)src2|(size_t)dst) & (CV_SIMD_WIDTH - 1)) == 0; }

template<template<typename T1, typename Tvec> class OP, typename T1, typename Tvec>
static void bin_loop(const T1* src1, size_t step1, const T1* src2, size_t step2, T1* dst, size_t step, int width, int height)
{
    typedef OP<T1, Tvec> op;
#if CV_SIMD
    typedef bin_loader<OP, T1, Tvec> ldr;
    enum {wide_step = Tvec::nlanes};
    #if !CV_NEON && CV_SIMD_WIDTH == 16
        enum {wide_step_l = wide_step * 2};
    #else
        enum {wide_step_l = wide_step};
    #endif
#endif // CV_SIMD
    step1 /= sizeof(T1);
    step2 /= sizeof(T1);
    step  /= sizeof(T1);

    for (; height--; src1 += step1, src2 += step2, dst += step)
    {
        int x = 0;

    #if CV_SIMD
        #if !CV_NEON && !CV_MSA
        if (is_aligned(src1, src2, dst))
        {
            for (; x <= width - wide_step_l; x += wide_step_l)
            {
                ldr::la(src1 + x, src2 + x, dst + x);
                #if CV_SIMD_WIDTH == 16
                ldr::la(src1 + x + wide_step, src2 + x + wide_step, dst + x + wide_step);
                #endif
            }
        }
        else
        #endif
            for (; x <= width - wide_step_l; x += wide_step_l)
            {
                ldr::l(src1 + x, src2 + x, dst + x);
                #if !CV_NEON && CV_SIMD_WIDTH == 16
                ldr::l(src1 + x + wide_step, src2 + x + wide_step, dst + x + wide_step);
                #endif
            }

        #if CV_SIMD_WIDTH == 16
        for (; x <= width - 8/(int)sizeof(T1); x += 8/(int)sizeof(T1))
        {
            ldr::l64(src1 + x, src2 + x, dst + x);
        }
        #endif
    #endif // CV_SIMD
    #if CV_ENABLE_UNROLLED || CV_SIMD_WIDTH > 16
        for (; x <= width - 4; x += 4)
        {
            T1 t0 = op::r(src1[x], src2[x]);
            T1 t1 = op::r(src1[x + 1], src2[x + 1]);
            dst[x] = t0; dst[x + 1] = t1;

            t0 = op::r(src1[x + 2], src2[x + 2]);
            t1 = op::r(src1[x + 3], src2[x + 3]);
            dst[x + 2] = t0; dst[x + 3] = t1;
        }
    #endif
        for (; x < width; x++)
            dst[x] = op::r(src1[x], src2[x]);
    }

    vx_cleanup();
}

#if !CV_SIMD_64F
template<template<typename T1, typename Tvec> class OP, typename T1, typename Tvec>
static void bin_loop_nosimd(const T1* src1, size_t step1, const T1* src2, size_t step2, T1* dst, size_t step, int width, int height)
{
    typedef OP<T1, Tvec/*dummy*/> op;

    step1 /= sizeof(T1);
    step2 /= sizeof(T1);
    step  /= sizeof(T1);

    for (; height--; src1 += step1, src2 += step2, dst += step)
    {
        int x = 0;

        for (; x <= width - 4; x += 4)
        {
            T1 t0 = op::r(src1[x], src2[x]);
            T1 t1 = op::r(src1[x + 1], src2[x + 1]);
            dst[x] = t0; dst[x + 1] = t1;

            t0 = op::r(src1[x + 2], src2[x + 2]);
            t1 = op::r(src1[x + 3], src2[x + 3]);
            dst[x + 2] = t0; dst[x + 3] = t1;
        }

        for (; x < width; x++)
            dst[x] = op::r(src1[x], src2[x]);
    }
}
#define BIN_LOOP64F bin_loop_nosimd
#else
#define BIN_LOOP64F bin_loop
#endif //!CV_SIMD_64F
#endif // ARITHM_DEFINITIONS_ONLY
////////////////////////////////////////////////////////////////////////////////////
#ifndef SIMD_GUARD
#define BIN_ARGS(_T1) const _T1* src1, size_t step1, const _T1* src2, size_t step2, \
                      _T1* dst, size_t step, int width, int height
#define BIN_ARGS_PASS src1, step1, src2, step2, dst, step, width, height
#endif // SIMD_GUARD
#undef DECLARE_SIMD_FUN
#define DECLARE_SIMD_FUN(fun, _T1) void fun(BIN_ARGS(_T1));
#undef DISPATCH_SIMD_FUN
#define DISPATCH_SIMD_FUN(fun, _T1, _Tvec, _OP)                           \
    void fun(BIN_ARGS(_T1), void*)                                        \
    {                                                                     \
        CV_INSTRUMENT_REGION();                                           \
        CALL_HAL(fun, __CV_CAT(cv_hal_, fun), BIN_ARGS_PASS)              \
        ARITHM_CALL_IPP(__CV_CAT(arithm_ipp_, fun), BIN_ARGS_PASS)        \
        CV_CPU_DISPATCH(fun, (BIN_ARGS_PASS), CV_CPU_DISPATCH_MODES_ALL); \
    }
#undef DEFINE_SIMD_FUN
#define DEFINE_SIMD_FUN(fun, _T1, _Tvec, _OP)     \
    void fun(BIN_ARGS(_T1))                       \
    {                                             \
        CV_INSTRUMENT_REGION();                   \
        bin_loop<_OP, _T1, _Tvec>(BIN_ARGS_PASS); \
    }
#undef DEFINE_NOSIMD_FUN
#define DEFINE_NOSIMD_FUN(fun, _T1, _OP)                     \
    void fun(BIN_ARGS(_T1))                                  \
    {                                                        \
        CV_INSTRUMENT_REGION();                              \
        bin_loop_nosimd<_OP, _T1, v_float64>(BIN_ARGS_PASS); \
    }
DEFINE_SIMD_ALL(add, op_add)
DEFINE_SIMD_ALL(sub, op_sub)

DEFINE_SIMD_ALL(min, op_min)
DEFINE_SIMD_ALL(max, op_max)

DEFINE_SIMD_ALL(absdiff, op_absdiff)

DEFINE_SIMD_U8(or,  op_or)
DEFINE_SIMD_U8(xor, op_xor)
DEFINE_SIMD_U8(and, op_and)

// One source!, an exception for operation "not"
// we could use macros here but it's better to implement it
// with that way to give more clarification
// about how macroS "DEFINE_SIMD_*" are works
#if defined(ARITHM_DECLARATIONS_ONLY) || defined(ARITHM_DEFINITIONS_ONLY)
void not8u(const uchar* src1, size_t step1, const uchar* src2, size_t step2, uchar* dst, size_t step, int width, int height);
#endif
#ifdef ARITHM_DEFINITIONS_ONLY
void not8u(const uchar* src1, size_t step1, const uchar* src2, size_t step2, uchar* dst, size_t step, int width, int height)
{
    CV_INSTRUMENT_REGION();
    bin_loop<op_not, uchar, v_uint8>(src1, step1, src2, step2, dst, step, width, height);
}
#endif
#ifdef ARITHM_DISPATCHING_ONLY
void not8u(const uchar* src1, size_t step1, const uchar* src2, size_t step2, uchar* dst, size_t step, int width, int height, void*)
{
    CV_INSTRUMENT_REGION();
    CALL_HAL(not8u, cv_hal_not8u, src1, step1, dst, step, width, height)
    ARITHM_CALL_IPP(arithm_ipp_not8u, src1, step1, dst, step, width, height)
    CV_CPU_DISPATCH(not8u, (src1, step1, src2, step2, dst, step, width, height), CV_CPU_DISPATCH_MODES_ALL);
}
#endif
//=======================================
// Compare
//=======================================
#ifdef ARITHM_DEFINITIONS_ONLY
///////////////////////////// Operations //////////////////////////////////
template<typename T1, typename Tvec>
struct op_cmplt
{
    static inline Tvec r(const Tvec& a, const Tvec& b)
    { return a < b; }
    static inline uchar r(T1 a, T1 b)
    { return (uchar)-(int)(a < b); }
};

template<typename T1, typename Tvec>
struct op_cmple
{
    static inline Tvec r(const Tvec& a, const Tvec& b)
    { return a <= b; }
    static inline uchar r(T1 a, T1 b)
    { return (uchar)-(int)(a <= b); }
};

template<typename T1, typename Tvec>
struct op_cmpeq
{
    static inline Tvec r(const Tvec& a, const Tvec& b)
    { return a == b; }
    static inline uchar r(T1 a, T1 b)
    { return (uchar)-(int)(a == b); }
};

template<typename T1, typename Tvec>
struct op_cmpne
{
    static inline Tvec r(const Tvec& a, const Tvec& b)
    { return a != b; }
    static inline uchar r(T1 a, T1 b)
    { return (uchar)-(int)(a != b); }
};

//////////////////////////// Loaders /////////////////////////////////
#if CV_SIMD
// todo: add support for RW alignment & stream
template<int nload, template<typename T1, typename Tvec> class OP, typename T1, typename Tvec>
struct cmp_loader_n
{
    void l(const T1* src1, const T1* src2, uchar* dst);
};

template<template<typename T1, typename Tvec> class OP, typename T1, typename Tvec>
struct cmp_loader_n<sizeof(uchar), OP, T1, Tvec>
{
    typedef OP<T1, Tvec> op;

    static inline void l(const T1* src1, const T1* src2, uchar* dst)
    {
        Tvec a = vx_load(src1);
        Tvec b = vx_load(src2);
        v_store(dst, v_reinterpret_as_u8(op::r(a, b)));
    }
};

template<template<typename T1, typename Tvec> class OP, typename T1, typename Tvec>
struct cmp_loader_n<sizeof(ushort), OP, T1, Tvec>
{
    typedef OP<T1, Tvec> op;
    enum {step = Tvec::nlanes};

    static inline void l(const T1* src1, const T1* src2, uchar* dst)
    {
        Tvec c0 = op::r(vx_load(src1), vx_load(src2));
        Tvec c1 = op::r(vx_load(src1 + step), vx_load(src2 + step));
        v_store(dst, v_pack_b(v_reinterpret_as_u16(c0), v_reinterpret_as_u16(c1)));
    }
};

template<template<typename T1, typename Tvec> class OP, typename T1, typename Tvec>
struct cmp_loader_n<sizeof(unsigned), OP, T1, Tvec>
{
    typedef OP<T1, Tvec> op;
    enum {step = Tvec::nlanes};

    static inline void l(const T1* src1, const T1* src2, uchar* dst)
    {
        v_uint32 c0 = v_reinterpret_as_u32(op::r(vx_load(src1), vx_load(src2)));
        v_uint32 c1 = v_reinterpret_as_u32(op::r(vx_load(src1 + step), vx_load(src2 + step)));
        v_uint32 c2 = v_reinterpret_as_u32(op::r(vx_load(src1 + step * 2), vx_load(src2 + step * 2)));
        v_uint32 c3 = v_reinterpret_as_u32(op::r(vx_load(src1 + step * 3), vx_load(src2 + step * 3)));
        v_store(dst, v_pack_b(c0, c1, c2, c3));
    }
};

template<template<typename T1, typename Tvec> class OP, typename T1, typename Tvec>
struct cmp_loader_n<sizeof(double), OP, T1, Tvec>
{
    typedef OP<T1, Tvec> op;
    enum {step = Tvec::nlanes};

    static inline void l(const T1* src1, const T1* src2, uchar* dst)
    {
        v_uint64 c0 = v_reinterpret_as_u64(op::r(vx_load(src1), vx_load(src2)));
        v_uint64 c1 = v_reinterpret_as_u64(op::r(vx_load(src1 + step), vx_load(src2 + step)));
        v_uint64 c2 = v_reinterpret_as_u64(op::r(vx_load(src1 + step * 2), vx_load(src2 + step * 2)));
        v_uint64 c3 = v_reinterpret_as_u64(op::r(vx_load(src1 + step * 3), vx_load(src2 + step * 3)));

        v_uint64 c4 = v_reinterpret_as_u64(op::r(vx_load(src1 + step * 4), vx_load(src2 + step * 4)));
        v_uint64 c5 = v_reinterpret_as_u64(op::r(vx_load(src1 + step * 5), vx_load(src2 + step * 5)));
        v_uint64 c6 = v_reinterpret_as_u64(op::r(vx_load(src1 + step * 6), vx_load(src2 + step * 6)));
        v_uint64 c7 = v_reinterpret_as_u64(op::r(vx_load(src1 + step * 7), vx_load(src2 + step * 7)));
        v_store(dst, v_pack_b(c0, c1, c2, c3, c4, c5, c6, c7));
    }
};

#endif // CV_SIMD
//////////////////////////// Loops /////////////////////////////////
template<template<typename T1, typename Tvec> class OP, typename T1, typename Tvec>
static void cmp_loop(const T1* src1, size_t step1, const T1* src2, size_t step2, uchar* dst, size_t step, int width, int height)
{
    typedef OP<T1, Tvec> op;
#if CV_SIMD
    typedef cmp_loader_n<sizeof(T1), OP, T1, Tvec> ldr;
    enum {wide_step = Tvec::nlanes * sizeof(T1)};
#endif // CV_SIMD
    step1 /= sizeof(T1);
    step2 /= sizeof(T1);

    for (; height--; src1 += step1, src2 += step2, dst += step)
    {
        int x = 0;

    #if CV_SIMD
        for (; x <= width - wide_step; x += wide_step)
        {
            ldr::l(src1 + x, src2 + x, dst + x);
        }
    #endif // CV_SIMD
    #if CV_ENABLE_UNROLLED || CV_SIMD_WIDTH > 16
        for (; x <= width - 4; x += 4)
        {
            uchar t0 = op::r(src1[x], src2[x]);
            uchar t1 = op::r(src1[x + 1], src2[x + 1]);
            dst[x] = t0; dst[x + 1] = t1;

            t0 = op::r(src1[x + 2], src2[x + 2]);
            t1 = op::r(src1[x + 3], src2[x + 3]);
            dst[x + 2] = t0; dst[x + 3] = t1;
        }
    #endif
        for (; x < width; x++)
            dst[x] = op::r(src1[x], src2[x]);
    }

    vx_cleanup();
}

template<typename T1, typename Tvec>
static void cmp_loop(const T1* src1, size_t step1, const T1* src2, size_t step2,
                     uchar* dst, size_t step, int width, int height, int cmpop)
{
    switch(cmpop)
    {
    case CMP_LT:
        cmp_loop<op_cmplt, T1, Tvec>(src1, step1, src2, step2, dst, step, width, height);
        break;
    case CMP_GT:
        cmp_loop<op_cmplt, T1, Tvec>(src2, step2, src1, step1, dst, step, width, height);
        break;
    case CMP_LE:
        cmp_loop<op_cmple, T1, Tvec>(src1, step1, src2, step2, dst, step, width, height);
        break;
    case CMP_GE:
        cmp_loop<op_cmple, T1, Tvec>(src2, step2, src1, step1, dst, step, width, height);
        break;
    case CMP_EQ:
        cmp_loop<op_cmpeq, T1, Tvec>(src1, step1, src2, step2, dst, step, width, height);
        break;
    default:
        CV_Assert(cmpop == CMP_NE);
        cmp_loop<op_cmpne, T1, Tvec>(src1, step1, src2, step2, dst, step, width, height);
        break;
    }
}

#if !CV_SIMD_64F
template< template<typename T1, typename Tvec> class OP, typename T1>
static void cmp_loop_nosimd(const T1* src1, size_t step1, const T1* src2, size_t step2, uchar* dst, size_t step, int width, int height)
{
    typedef OP<T1, v_int32 /*dummy*/> op;

    step1 /= sizeof(T1);
    step2 /= sizeof(T1);

    for (; height--; src1 += step1, src2 += step2, dst += step)
    {
        int x = 0;

        for (; x <= width - 4; x += 4)
        {
            uchar t0 = op::r(src1[x], src2[x]);
            uchar t1 = op::r(src1[x + 1], src2[x + 1]);
            dst[x] = t0; dst[x + 1] = t1;

            t0 = op::r(src1[x + 2], src2[x + 2]);
            t1 = op::r(src1[x + 3], src2[x + 3]);
            dst[x + 2] = t0; dst[x + 3] = t1;
        }

        for (; x < width; x++)
            dst[x] = op::r(src1[x], src2[x]);
    }
}
static void cmp_loop_nosimd(const double* src1, size_t step1, const double* src2, size_t step2,
                            uchar* dst, size_t step, int width, int height, int cmpop)
{
    switch(cmpop)
    {
    case CMP_LT:
        cmp_loop_nosimd<op_cmplt, double>(src1, step1, src2, step2, dst, step, width, height);
        break;
    case CMP_GT:
        cmp_loop_nosimd<op_cmplt, double>(src2, step2, src1, step1, dst, step, width, height);
        break;
    case CMP_LE:
        cmp_loop_nosimd<op_cmple, double>(src1, step1, src2, step2, dst, step, width, height);
        break;
    case CMP_GE:
        cmp_loop_nosimd<op_cmple, double>(src2, step2, src1, step1, dst, step, width, height);
        break;
    case CMP_EQ:
        cmp_loop_nosimd<op_cmpeq, double>(src1, step1, src2, step2, dst, step, width, height);
        break;
    default:
        CV_Assert(cmpop == CMP_NE);
        cmp_loop_nosimd<op_cmpne, double>(src1, step1, src2, step2, dst, step, width, height);
        break;
    }
}
#endif // !CV_SIMD_64F
#endif // ARITHM_DEFINITIONS_ONLY
/////////////////////////////////////////////////////////////////////////////////////////////
#ifndef SIMD_GUARD
#define CMP_ARGS(_T1) const _T1* src1, size_t step1, const _T1* src2, size_t step2, \
                           uchar* dst, size_t step, int width, int height
#define CMP_ARGS_PASS src1, step1, src2, step2, dst, step, width, height
#endif // SIMD_GUARD
#undef DECLARE_SIMD_FUN
#define DECLARE_SIMD_FUN(fun, _T1) void fun(CMP_ARGS(_T1), int cmpop);
#undef DISPATCH_SIMD_FUN
#define DISPATCH_SIMD_FUN(fun, _T1, _Tvec, ...)                                          \
    void fun(CMP_ARGS(_T1), void* _cmpop)                                                \
    {                                                                                    \
        CV_INSTRUMENT_REGION();                                                          \
        CALL_HAL(fun, __CV_CAT(cv_hal_, fun), CMP_ARGS_PASS, *(int*)_cmpop)              \
        ARITHM_CALL_IPP(__CV_CAT(arithm_ipp_, fun), CMP_ARGS_PASS, *(int*)_cmpop)        \
        CV_CPU_DISPATCH(fun, (CMP_ARGS_PASS, *(int*)_cmpop), CV_CPU_DISPATCH_MODES_ALL); \
    }
#undef DEFINE_SIMD_FUN
#define DEFINE_SIMD_FUN(fun, _T1, _Tvec, ...)       \
    void fun(CMP_ARGS(_T1), int cmpop)              \
    {                                               \
        CV_INSTRUMENT_REGION();                     \
        cmp_loop<_T1, _Tvec>(CMP_ARGS_PASS, cmpop); \
    }
#undef DEFINE_NOSIMD_FUN
#define DEFINE_NOSIMD_FUN(fun, _T1, _Tvec, ...)     \
    void fun(CMP_ARGS(_T1), int cmpop)              \
    {                                               \
        CV_INSTRUMENT_REGION();                     \
        cmp_loop_nosimd(CMP_ARGS_PASS, cmpop);      \
    }
// todo: try to avoid define dispatcher functions using macros with these such cases
DEFINE_SIMD_ALL(cmp)

//=========================================================================
// scaling helpers for single and dual source
//
// Dual: Multiply, Div, AddWeighted
//
// Single: Reciprocal
//
//=========================================================================
#ifdef ARITHM_DEFINITIONS_ONLY
//////////////////////////// Loaders ///////////////////////////////
#if CV_SIMD
// todo: add support for RW alignment & stream
template<int nload, template<typename T1, typename T2, typename Tvec> class OP, typename T1, typename T2, typename Tvec>
struct scalar_loader_n
{
    void l(const T1* src1, const T1* src2, const T2* scalar, T1* dst);
    // single source
    void l(const T1* src1, const T2* scalar, T1* dst);
};

template<template<typename T1, typename T2, typename Tvec> class OP, typename T1, typename T2, typename Tvec>
struct scalar_loader_n<sizeof(uchar), OP, T1, T2, Tvec>
{
    typedef OP<T1, T2, v_int16> op;

    static inline void l(const T1* src1, const T1* src2, const T2* scalar, T1* dst)
    {
        v_int16 v_src1 = v_reinterpret_as_s16(vx_load_expand(src1));
        v_int16 v_src2 = v_reinterpret_as_s16(vx_load_expand(src2));

        v_int32 t0, t1, t2, t3;
        v_expand(v_src1, t0, t2);
        v_expand(v_src2, t1, t3);

        v_float32 f0, f1, f2, f3;
        f0 = v_cvt_f32(t0);
        f1 = v_cvt_f32(t1);
        f2 = v_cvt_f32(t2);
        f3 = v_cvt_f32(t3);

        f0 = op::r(f0, f1, scalar);
        f2 = op::r(f2, f3, scalar);

        v_int32 r0 = v_round(f0);
        v_int32 r1 = v_round(f2);

        store(dst, v_src2, r0, r1);
    }

    static inline void l(const T1* src1, const T2* scalar, T1* dst)
    {
        v_int16 v_src1 = v_reinterpret_as_s16(vx_load_expand(src1));

        v_int32 t0, t1;
        v_expand(v_src1, t0, t1);

        v_float32 f0, f1;
        f0 = v_cvt_f32(t0);
        f1 = v_cvt_f32(t1);

        f0 = op::r(f0, scalar);
        f1 = op::r(f1, scalar);

        v_int32 r0 = v_round(f0);
        v_int32 r1 = v_round(f1);

        store(dst, v_src1, r0, r1);
    }

    static inline void store(uchar* dst, const v_int16& src, const v_int32& a, const v_int32& b)
    {
        v_pack_u_store(dst, op::pre(src, v_pack(a, b)));
    }
    static inline void store(schar* dst, const v_int16& src, const v_int32& a, const v_int32& b)
    {
        v_pack_store(dst, op::pre(src, v_pack(a, b)));
    }
};

template<template<typename T1, typename T2, typename Tvec> class OP, typename T1, typename T2, typename Tvec>
struct scalar_loader_n<sizeof(ushort), OP, T1, T2, Tvec>
{
    typedef typename V_RegTraits<Tvec>::w_reg Twvec;
    typedef OP<T1, T2, Tvec> op;

    static inline void l(const T1* src1, const T1* src2, const T2* scalar, T1* dst)
    {
        Tvec v_src1 = vx_load(src1);
        Tvec v_src2 = vx_load(src2);

        Twvec t0, t1, t2, t3;
        v_expand(v_src1, t0, t2);
        v_expand(v_src2, t1, t3);

        v_float32 f0, f1, f2, f3;
        f0 = v_cvt_f32(v_reinterpret_as_s32(t0));
        f1 = v_cvt_f32(v_reinterpret_as_s32(t1));
        f2 = v_cvt_f32(v_reinterpret_as_s32(t2));
        f3 = v_cvt_f32(v_reinterpret_as_s32(t3));

        f0 = op::r(f0, f1, scalar);
        f2 = op::r(f2, f3, scalar);

        v_int32 r0 = v_round(f0);
        v_int32 r1 = v_round(f2);

        store(dst, v_src2, r0, r1);
    }

    static inline void l(const T1* src1, const T2* scalar, T1* dst)
    {
        Tvec v_src1 = vx_load(src1);

        Twvec t0, t1;
        v_expand(v_src1, t0, t1);

        v_float32 f0, f1;
        f0 = v_cvt_f32(v_reinterpret_as_s32(t0));
        f1 = v_cvt_f32(v_reinterpret_as_s32(t1));

        f0 = op::r(f0, scalar);
        f1 = op::r(f1, scalar);

        v_int32 r0 = v_round(f0);
        v_int32 r1 = v_round(f1);

        store(dst, v_src1, r0, r1);
    }

    static inline void store(ushort* dst, const Tvec& src, const v_int32& a, const v_int32& b)
    {
        v_store(dst, op::pre(src, v_pack_u(a, b)));
    }
    static inline void store(short* dst, const Tvec& src, const v_int32& a, const v_int32& b)
    {
        v_store(dst, op::pre(src, v_pack(a, b)));
    }
};

template<template<typename T1, typename T2, typename Tvec> class OP, typename T2>
struct scalar_loader_n<sizeof(int), OP, int, T2, v_int32>
{
    typedef OP<int, T2, v_int32> op;
    enum {step = v_int32::nlanes};

    static inline void l(const int* src1, const int* src2, const T2* scalar, int* dst)
    {
        v_int32 v_src1 = vx_load(src1);
        v_int32 v_src2 = vx_load(src2);
        v_int32 v_src1s = vx_load(src1 + step);
        v_int32 v_src2s = vx_load(src2 + step);

        v_float32 f0, f1, f2, f3;
        f0 = v_cvt_f32(v_reinterpret_as_s32(v_src1));
        f1 = v_cvt_f32(v_reinterpret_as_s32(v_src2));
        f2 = v_cvt_f32(v_reinterpret_as_s32(v_src1s));
        f3 = v_cvt_f32(v_reinterpret_as_s32(v_src2s));

        f0 = op::r(f0, f1, scalar);
        f2 = op::r(f2, f3, scalar);

        v_int32 r0 = v_round(f0);
        v_int32 r1 = v_round(f2);

        r0 = op::pre(v_src2, r0);
        r1 = op::pre(v_src2s, r1);

        v_store(dst, r0);
        v_store(dst + step, r1);
    }

    static inline void l(const int* src1, const T2* scalar, int* dst)
    {
        v_int32 v_src1 = vx_load(src1);
        v_int32 v_src1s = vx_load(src1 + step);

        v_float32 f0, f1;
        f0 = v_cvt_f32(v_src1);
        f1 = v_cvt_f32(v_src1s);

        f0 = op::r(f0, scalar);
        f1 = op::r(f1, scalar);

        v_int32 r0 = v_round(f0);
        v_int32 r1 = v_round(f1);

        r0 = op::pre(v_src1, r0);
        r1 = op::pre(v_src1s, r1);

        v_store(dst, r0);
        v_store(dst + step, r1);
    }
};

template<template<typename T1, typename T2, typename Tvec> class OP, typename T2>
struct scalar_loader_n<sizeof(float), OP, float, T2, v_float32>
{
    typedef OP<float, T2, v_float32> op;
    enum {step = v_float32::nlanes};

    static inline void l(const float* src1, const float* src2, const T2* scalar, float* dst)
    {
        v_float32 v_src1 = vx_load(src1);
        v_float32 v_src2 = vx_load(src2);
        v_float32 v_src1s = vx_load(src1 + step);
        v_float32 v_src2s = vx_load(src2 + step);

        v_float32 r0 = op::r(v_src1, v_src2, scalar);
        v_float32 r1 = op::r(v_src1s, v_src2s, scalar);

        v_store(dst, r0);
        v_store(dst + step, r1);
    }

    static inline void l(const float* src1, const T2* scalar, float* dst)
    {
        v_float32 v_src1 = vx_load(src1);
        v_float32 v_src1s = vx_load(src1 + step);

        v_float32 r0 = op::r(v_src1, scalar);
        v_float32 r1 = op::r(v_src1s, scalar);

        v_store(dst, r0);
        v_store(dst + step, r1);
    }
};
#endif // CV_SIMD
#if CV_SIMD_64F
template<template<typename T1, typename T2, typename Tvec> class OP>
struct scalar_loader_n<sizeof(int), OP, int, double, v_int32>
{
    typedef OP<int, float, v_int32> op;
    typedef OP<double, double, v_float64> op64;
    enum {step = v_int32::nlanes};

    static inline void l(const int* src1, const int* src2, const double* scalar, int* dst)
    {
        v_int32 v_src1 = vx_load(src1);
        v_int32 v_src2 = vx_load(src2);
        v_int32 v_src1s = vx_load(src1 + step);
        v_int32 v_src2s = vx_load(src2 + step);

        v_int32 r0 = r(v_src1, v_src2, scalar);
        v_int32 r1 = r(v_src1s, v_src2s, scalar);

        r0 = op::pre(v_src2, r0);
        r1 = op::pre(v_src2s, r1);

        v_store(dst, r0);
        v_store(dst + step, r1);
    }
    static inline void l(const int* src1, const double* scalar, int* dst)
    {
        v_int32 v_src1 = vx_load(src1);
        v_int32 v_src1s = vx_load(src1 + step);

        v_int32 r0 = r(v_src1, scalar);
        v_int32 r1 = r(v_src1s, scalar);

        r0 = op::pre(v_src1, r0);
        r1 = op::pre(v_src1s, r1);

        v_store(dst, r0);
        v_store(dst + step, r1);
    }

    static inline v_int32 r(const v_int32& a, const v_int32& b, const double* scalar)
    {
        v_float64 f0, f1, f2, f3;
        f0 = v_cvt_f64(a);
        f1 = v_cvt_f64_high(a);
        f2 = v_cvt_f64(b);
        f3 = v_cvt_f64_high(b);

        v_float64 r0 = op64::r(f0, f2, scalar);
        v_float64 r1 = op64::r(f1, f3, scalar);

        return v_round(r0, r1);
    }
    static inline v_int32 r(const v_int32& a, const double* scalar)
    {
        v_float64 f0, f1;
        f0 = v_cvt_f64(a);
        f1 = v_cvt_f64_high(a);

        v_float64 r0 = op64::r(f0, scalar);
        v_float64 r1 = op64::r(f1, scalar);

        return v_round(r0, r1);
    }
};

template<template<typename T1, typename T2, typename Tvec> class OP>
struct scalar_loader_n<sizeof(float), OP, float, double, v_float32>
{
    typedef OP<float, float, v_float32> op;
    typedef OP<double, double, v_float64> op64;
    enum {step = v_float32::nlanes};

    static inline void l(const float* src1, const float* src2, const double* scalar, float* dst)
    {
        v_float32 v_src1 = vx_load(src1);
        v_float32 v_src2 = vx_load(src2);
        v_float32 v_src1s = vx_load(src1 + step);
        v_float32 v_src2s = vx_load(src2 + step);

        v_float32 r0 = r(v_src1, v_src2, scalar);
        v_float32 r1 = r(v_src1s, v_src2s, scalar);

        v_store(dst, r0);
        v_store(dst + step, r1);
    }
    static inline void l(const float* src1, const double* scalar, float* dst)
    {
        v_float32 v_src1 = vx_load(src1);
        v_float32 v_src1s = vx_load(src1 + step);

        v_float32 r0 = r(v_src1, scalar);
        v_float32 r1 = r(v_src1s, scalar);

        v_store(dst, r0);
        v_store(dst + step, r1);
    }

    static inline v_float32 r(const v_float32& a, const v_float32& b, const double* scalar)
    {
        v_float64 f0, f1, f2, f3;
        f0 = v_cvt_f64(a);
        f1 = v_cvt_f64_high(a);
        f2 = v_cvt_f64(b);
        f3 = v_cvt_f64_high(b);

        v_float64 r0 = op64::r(f0, f2, scalar);
        v_float64 r1 = op64::r(f1, f3, scalar);

        return v_cvt_f32(r0, r1);
    }
    static inline v_float32 r(const v_float32& a, const double* scalar)
    {
        v_float64 f0, f1;
        f0 = v_cvt_f64(a);
        f1 = v_cvt_f64_high(a);

        v_float64 r0 = op64::r(f0, scalar);
        v_float64 r1 = op64::r(f1, scalar);

        return v_cvt_f32(r0, r1);
    }
};

template<template<typename T1, typename T2, typename Tvec> class OP>
struct scalar_loader_n<sizeof(double), OP, double, double, v_float64>
{
    typedef OP<double, double, v_float64> op;
    enum {step = v_float64::nlanes};

    static inline void l(const double* src1, const double* src2, const double* scalar, double* dst)
    {
        v_float64 v_src1 = vx_load(src1);
        v_float64 v_src2 = vx_load(src2);
        v_float64 v_src1s = vx_load(src1 + step);
        v_float64 v_src2s = vx_load(src2 + step);

        v_float64 r0 = op::r(v_src1, v_src2, scalar);
        v_float64 r1 = op::r(v_src1s, v_src2s, scalar);

        v_store(dst, r0);
        v_store(dst + step, r1);
    }
    static inline void l(const double* src1, const double* scalar, double* dst)
    {
        v_float64 v_src1 = vx_load(src1);
        v_float64 v_src1s = vx_load(src1 + step);

        v_float64 r0 = op::r(v_src1, scalar);
        v_float64 r1 = op::r(v_src1s, scalar);

        v_store(dst, r0);
        v_store(dst + step, r1);
    }
};
#endif // CV_SIMD_64F
//////////////////////////// Loops /////////////////////////////////
// dual source
template<template<typename T1, typename T2, typename Tvec> class OP, typename T1, typename T2, typename Tvec>
static void scalar_loop(const T1* src1, size_t step1, const T1* src2, size_t step2,
                 T1* dst, size_t step, int width, int height, const T2* scalar)
{
    typedef OP<T1, T2, Tvec> op;
#if CV_SIMD
    typedef scalar_loader_n<sizeof(T1), OP, T1, T2, Tvec> ldr;
    const int wide_step = sizeof(T1) > sizeof(ushort) ? Tvec::nlanes * 2 :
                          sizeof(T1) == sizeof(uchar) ? Tvec::nlanes / 2 : Tvec::nlanes;
#endif // CV_SIMD
    step1 /= sizeof(T1);
    step2 /= sizeof(T1);
    step  /= sizeof(T1);

    for (; height--; src1 += step1, src2 += step2, dst += step)
    {
        int x = 0;

    #if CV_SIMD
        for (; x <= width - wide_step; x += wide_step)
        {
            ldr::l(src1 + x, src2 + x, scalar, dst + x);
        }
    #endif // CV_SIMD
    #if CV_ENABLE_UNROLLED || CV_SIMD_WIDTH > 16
        for (; x <= width - 4; x += 4)
        {
            T1 t0 = op::r(src1[x], src2[x], scalar);
            T1 t1 = op::r(src1[x + 1], src2[x + 1], scalar);
            dst[x] = t0; dst[x + 1] = t1;

            t0 = op::r(src1[x + 2], src2[x + 2], scalar);
            t1 = op::r(src1[x + 3], src2[x + 3], scalar);
            dst[x + 2] = t0; dst[x + 3] = t1;
        }
    #endif
        for (; x < width; ++x)
            dst[x] = op::r(src1[x], src2[x], scalar);
    }

    vx_cleanup();
}

// single source
template<template<typename T1, typename T2, typename Tvec> class OP, typename T1, typename T2, typename Tvec>
static void scalar_loop(const T1* src1, size_t step1, T1* dst, size_t step, int width, int height, const T2* scalar)
{
    typedef OP<T1, T2, Tvec> op;
#if CV_SIMD
    typedef scalar_loader_n<sizeof(T1), OP, T1, T2, Tvec> ldr;
    const int wide_step = sizeof(T1) > sizeof(ushort) ? Tvec::nlanes * 2 :
                          sizeof(T1) == sizeof(uchar) ? Tvec::nlanes / 2 : Tvec::nlanes;
#endif // CV_SIMD
    step1 /= sizeof(T1);
    step  /= sizeof(T1);

    for (; height--; src1 += step1, dst += step)
    {
        int x = 0;

    #if CV_SIMD
        for (; x <= width - wide_step; x += wide_step)
        {
            ldr::l(src1 + x, scalar, dst + x);
        }
    #endif // CV_SIMD
    #if CV_ENABLE_UNROLLED || CV_SIMD_WIDTH > 16
        for (; x <= width - 4; x += 4)
        {
            T1 t0 = op::r(src1[x], scalar);
            T1 t1 = op::r(src1[x + 1], scalar);
            dst[x] = t0; dst[x + 1] = t1;

            t0 = op::r(src1[x + 2], scalar);
            t1 = op::r(src1[x + 3], scalar);
            dst[x + 2] = t0; dst[x + 3] = t1;
        }
    #endif
        for (; x < width; ++x)
            dst[x] = op::r(src1[x], scalar);
    }

    vx_cleanup();
}

#if !CV_SIMD_64F
// dual source
template<template<typename T1, typename T2, typename Tvec> class OP, typename T1, typename T2, typename Tvec>
static void scalar_loop_nosimd(const T1* src1, size_t step1, const T1* src2, size_t step2,
                 T1* dst, size_t step, int width, int height, const T2* scalar)
{
    typedef OP<T1, T2, Tvec> op;

    step1 /= sizeof(T1);
    step2 /= sizeof(T1);
    step  /= sizeof(T1);

    for (; height--; src1 += step1, src2 += step2, dst += step)
    {
        int x = 0;

        for (; x <= width - 4; x += 4)
        {
            T1 t0 = op::r(src1[x], src2[x], scalar);
            T1 t1 = op::r(src1[x + 1], src2[x + 1], scalar);
            dst[x] = t0; dst[x + 1] = t1;

            t0 = op::r(src1[x + 2], src2[x + 2], scalar);
            t1 = op::r(src1[x + 3], src2[x + 3], scalar);
            dst[x + 2] = t0; dst[x + 3] = t1;
        }

        for (; x < width; ++x)
            dst[x] = op::r(src1[x], src2[x], scalar);
    }
}

// single source
template<template<typename T1, typename T2, typename Tvec> class OP, typename T1, typename T2, typename Tvec>
static void scalar_loop_nosimd(const T1* src1, size_t step1, T1* dst, size_t step, int width, int height, const T2* scalar)
{
    typedef OP<T1, T2, Tvec> op;

    step1 /= sizeof(T1);
    step  /= sizeof(T1);

    for (; height--; src1 += step1, dst += step)
    {
        int x = 0;

        for (; x <= width - 4; x += 4)
        {
            T1 t0 = op::r(src1[x], scalar);
            T1 t1 = op::r(src1[x + 1], scalar);
            dst[x] = t0; dst[x + 1] = t1;

            t0 = op::r(src1[x + 2], scalar);
            t1 = op::r(src1[x + 3], scalar);
            dst[x + 2] = t0; dst[x + 3] = t1;
        }

        for (; x < width; ++x)
            dst[x] = op::r(src1[x], scalar);
    }
}

#define SCALAR_LOOP64F scalar_loop_nosimd
#else
#define SCALAR_LOOP64F scalar_loop
#endif // !CV_SIMD_64F
#endif // ARITHM_DEFINITIONS_ONLY
//=========================================================================
// Multiply
//=========================================================================
#ifdef ARITHM_DEFINITIONS_ONLY
///////////////////////////// Operations //////////////////////////////////
template<typename T1, typename Tvec>
struct op_mul
{
    static inline Tvec r(const Tvec& a, const Tvec& b)
    { return a * b; }
    static inline T1 r(T1 a, T1 b)
    { return saturate_cast<T1>(a * b); }
};

template<typename T1, typename T2, typename Tvec>
struct op_mul_scale
{
    static inline v_float32 r(const v_float32& a, const v_float32& b, const T2* scalar)
    {
        const v_float32 v_scalar = vx_setall_f32(*scalar);
        return v_scalar * a * b;
    }
    static inline T1 r(T1 a, T1 b, const T2* scalar)
    { return c_mul(a, b, *scalar); }
    static inline Tvec pre(const Tvec&, const Tvec& res)
    { return res; }
};

template<>
struct op_mul_scale<double, double, v_float64>
{
#if CV_SIMD_64F
    static inline v_float64 r(const v_float64& a, const v_float64& b, const double* scalar)
    {
        const v_float64 v_scalar = vx_setall_f64(*scalar);
        return v_scalar * a * b;
    }
#endif
    static inline double r(double a, double b, const double* scalar)
    { return c_mul(a, b, *scalar); }
    static inline v_float64 pre(const v_float64&, const v_float64& res)
    { return res; }
};

//////////////////////////// Loops /////////////////////////////////
template<typename T1, typename Tvec>
static void mul_loop(const T1* src1, size_t step1, const T1* src2, size_t step2,
              T1* dst, size_t step, int width, int height, const double* scalar)
{
    float fscalar = (float)*scalar;
    if (std::fabs(fscalar - 1.0f) <= FLT_EPSILON)
    {
        bin_loop<op_mul, T1, Tvec>(src1, step1, src2, step2, dst, step, width, height);
    }
    else
    {
        scalar_loop<op_mul_scale, T1, float, Tvec>(src1, step1, src2, step2,
            dst, step, width, height, &fscalar);
    }
}

template<typename T1, typename Tvec>
static void mul_loop_d(const T1* src1, size_t step1, const T1* src2, size_t step2,
                T1* dst, size_t step, int width, int height, const double* scalar)
{
    if (std::fabs(*scalar - 1.0) <= FLT_EPSILON)
    {
        bin_loop<op_mul, T1, Tvec>(src1, step1, src2, step2, dst, step, width, height);
    }
    else
    {
        SCALAR_LOOP64F<op_mul_scale, T1, double, Tvec>(src1, step1, src2, step2,
            dst, step, width, height, scalar);
    }
}

template<>
void mul_loop_d<double, v_float64>(const double* src1, size_t step1, const double* src2, size_t step2,
                                  double* dst, size_t step, int width, int height, const double* scalar)
{
    if (*scalar == 1.0)
    {
        BIN_LOOP64F<op_mul, double, v_float64>(src1, step1, src2, step2, dst, step, width, height);
    }
    else
    {
        SCALAR_LOOP64F<op_mul_scale, double, double, v_float64>(src1, step1, src2, step2,
            dst, step, width, height, scalar);
    }
}

#endif // ARITHM_DEFINITIONS_ONLY
//////////////////////////////////////////////////////////////////////////
#undef SCALAR_ARGS
#define SCALAR_ARGS(_T1) const _T1* src1, size_t step1, const _T1* src2, size_t step2, \
                          _T1* dst, size_t step, int width, int height
#undef SCALAR_ARGS_PASS
#define SCALAR_ARGS_PASS src1, step1, src2, step2, dst, step, width, height
#undef DECLARE_SIMD_FUN
#define DECLARE_SIMD_FUN(fun, _T1) void fun(SCALAR_ARGS(_T1), const double* scalar);
#undef DISPATCH_SIMD_FUN
#define DISPATCH_SIMD_FUN(fun, _T1, _Tvec, ...)                          \
    void fun(SCALAR_ARGS(_T1), void* scalar)                             \
    {                                                                    \
        CV_INSTRUMENT_REGION();                                          \
        CALL_HAL(fun, __CV_CAT(cv_hal_, fun),                            \
            SCALAR_ARGS_PASS, *(const double*)scalar)                    \
        ARITHM_CALL_IPP(__CV_CAT(arithm_ipp_, fun),                      \
            SCALAR_ARGS_PASS, *(const double*)scalar)                    \
        CV_CPU_DISPATCH(fun, (SCALAR_ARGS_PASS, (const double*)scalar),  \
            CV_CPU_DISPATCH_MODES_ALL);                                  \
    }
#undef DEFINE_SIMD_FUN
#define DEFINE_SIMD_FUN(fun, _T1, _Tvec, op)           \
    void fun(SCALAR_ARGS(_T1), const double* scalar)   \
    {                                                  \
        CV_INSTRUMENT_REGION();                        \
        op<_T1, _Tvec>(SCALAR_ARGS_PASS, scalar);      \
    }
#undef DEFINE_NOSIMD_FUN
#define DEFINE_NOSIMD_FUN(fun, _T1, _OP) \
    DEFINE_SIMD_FUN(fun, _T1, v_float64, _OP)
DEFINE_SIMD_SAT(mul, mul_loop)
DEFINE_SIMD_F32(mul, mul_loop_d)
DEFINE_SIMD_S32(mul, mul_loop_d)
DEFINE_SIMD_F64(mul, mul_loop_d)

//=========================================================================
// Div
//=========================================================================
#ifdef ARITHM_DEFINITIONS_ONLY
///////////////////////////// Operations //////////////////////////////////
template<typename T1, typename Tvec>
struct op_div_f
{
    static inline Tvec r(const Tvec& a, const Tvec& b)
    { return a / b; }
    static inline T1 r(T1 a, T1 b)
    { return a / b; }
};

template<typename T1, typename T2, typename Tvec>
struct op_div_scale
{
    static inline v_float32 r(const v_float32& a, const v_float32& b, const T2* scalar)
    {
        const v_float32 v_scalar = vx_setall_f32(*scalar);
        return a * v_scalar / b;
    }
    static inline Tvec pre(const Tvec& denom, const Tvec& res)
    {
        const Tvec v_zero = vx_setall<typename Tvec::lane_type>(0);
        return v_select(denom == v_zero, v_zero, res);
    }
    static inline T1 r(T1 a, T1 denom, const T2* scalar)
    {
        CV_StaticAssert(std::numeric_limits<T1>::is_integer, "");
        return denom != (T1)0 ? c_div(a, denom, *scalar) : (T1)0;
    }
};

template<>
struct op_div_scale<float, float, v_float32>
{
    static inline v_float32 r(const v_float32& a, const v_float32& b, const float* scalar)
    {
        const v_float32 v_scalar = vx_setall_f32(*scalar);
        return a * v_scalar / b;
    }
    static inline float r(float a, float denom, const float* scalar)
    { return c_div(a, denom, *scalar); }
};

template<>
struct op_div_scale<double, double, v_float64>
{
#if CV_SIMD_64F
    static inline v_float64 r(const v_float64& a, const v_float64& b, const double* scalar)
    {
        const v_float64 v_scalar = vx_setall_f64(*scalar);
        return a * v_scalar / b;
    }
#endif
    static inline double r(double a, double denom, const double* scalar)
    { return c_div(a, denom, *scalar); }
};

//////////////////////////// Loops /////////////////////////////////
template<typename T1, typename Tvec>
static void div_loop(const T1* src1, size_t step1, const T1* src2, size_t step2,
              T1* dst, size_t step, int width, int height, const double* scalar)
{
    float fscalar = (float)*scalar;
    // todo: add new intrinsics for integer divide
    scalar_loop<op_div_scale, T1, float, Tvec>(src1, step1, src2, step2,
        dst, step, width, height, &fscalar);
}

template<>
void div_loop<float, v_float32>(const float* src1, size_t step1, const float* src2, size_t step2,
                                float* dst, size_t step, int width, int height, const double* scalar)
{
    float fscalar = (float)*scalar;
    if (std::fabs(fscalar - 1.0f) <= FLT_EPSILON)
    {
        bin_loop<op_div_f, float, v_float32>(src1, step1, src2, step2, dst, step, width, height);
    }
    else
    {
        SCALAR_LOOP64F<op_div_scale, float, float, v_float32>(src1, step1, src2, step2,
            dst, step, width, height, &fscalar);
    }
}

template<>
void div_loop<double, v_float64>(const double* src1, size_t step1, const double* src2, size_t step2,
                                double* dst, size_t step, int width, int height, const double* scalar)
{
    if (*scalar == 1.0)
    {
        BIN_LOOP64F<op_div_f, double, v_float64>(src1, step1, src2, step2, dst, step, width, height);
    }
    else
    {
        SCALAR_LOOP64F<op_div_scale, double, double, v_float64>(src1, step1, src2, step2,
            dst, step, width, height, scalar);
    }
}

#endif // ARITHM_DEFINITIONS_ONLY
//////////////////////////////////////////////////////////////////////////
DEFINE_SIMD_ALL(div, div_loop)

//=========================================================================
// AddWeighted
//=========================================================================
#ifdef ARITHM_DEFINITIONS_ONLY
///////////////////////////// Operations //////////////////////////////////
///// Add scale
template<typename T1, typename T2, typename Tvec>
struct op_add_scale
{
    static inline v_float32 r(const v_float32& a, const v_float32& b, const T2* scalar)
    {
        const v_float32 v_alpha = vx_setall_f32(*scalar);
        return v_fma(a, v_alpha, b);
    }
    static inline T1 r(T1 a, T1 b, const T2* scalar)
    { return c_add(a, b, *scalar); }
    static inline Tvec pre(const Tvec&, const Tvec& res)
    { return res; }
};

template<>
struct op_add_scale<double, double, v_float64>
{
#if CV_SIMD_64F
    static inline v_float64 r(const v_float64& a, const v_float64& b, const double* scalar)
    {
        const v_float64 v_alpha = vx_setall_f64(*scalar);
        return v_fma(a, v_alpha, b);
    }
#endif
    static inline double r(double a, double b, const double* scalar)
    { return c_add(a, b, *scalar); }
    static inline v_float64 pre(const v_float64&, const v_float64& res)
    { return res; }
};

///// Weighted sum
template<typename T1, typename T2, typename Tvec>
struct op_add_weighted
{
    static inline v_float32 r(const v_float32& a, const v_float32& b, const T2* scalars)
    {
        const v_float32 v_alpha = vx_setall_f32(scalars[0]);
        const v_float32 v_beta  = vx_setall_f32(scalars[1]);
        const v_float32 v_gamma = vx_setall_f32(scalars[2]);
        return v_fma(a, v_alpha, v_fma(b, v_beta, v_gamma));
    }
    static inline T1 r(T1 a, T1 b, const T2* scalars)
    { return c_add(a, b, scalars[0], scalars[1], scalars[2]); }
    static inline Tvec pre(const Tvec&, const Tvec& res)
    { return res; }
};

template<>
struct op_add_weighted<double, double, v_float64>
{
#if CV_SIMD_64F
    static inline v_float64 r(const v_float64& a, const v_float64& b, const double* scalars)
    {
        const v_float64 v_alpha = vx_setall_f64(scalars[0]);
        const v_float64 v_beta  = vx_setall_f64(scalars[1]);
        const v_float64 v_gamma = vx_setall_f64(scalars[2]);
        return v_fma(a, v_alpha, v_fma(b, v_beta, v_gamma));
    }
#endif
    static inline double r(double a, double b, const double* scalars)
    { return c_add(a, b, scalars[0], scalars[1], scalars[2]); }
    static inline v_float64 pre(const v_float64&, const v_float64& res)
    { return res; }
};

//////////////////////////// Loops /////////////////////////////////
template<typename T1, typename Tvec>
static void add_weighted_loop(const T1* src1, size_t step1, const T1* src2, size_t step2,
                       T1* dst, size_t step, int width, int height, const double* scalars)
{
    float fscalars[] = {(float)scalars[0], (float)scalars[1], (float)scalars[2]};
    if (fscalars[1] == 1.0f && fscalars[2] == 0.0f)
    {
        scalar_loop<op_add_scale, T1, float, Tvec>(src1, step1, src2, step2,
            dst, step, width, height, fscalars);
    }
    else
    {
        scalar_loop<op_add_weighted, T1, float, Tvec>(src1, step1, src2, step2,
            dst, step, width, height, fscalars);
    }
}

template<typename T1, typename Tvec>
static void add_weighted_loop_d(const T1* src1, size_t step1, const T1* src2, size_t step2,
                         T1* dst, size_t step, int width, int height, const double* scalars)
{
    if (scalars[1] == 1.0 && scalars[2] == 0.0)
    {
        SCALAR_LOOP64F<op_add_scale, T1, double, Tvec>(src1, step1, src2, step2,
            dst, step, width, height, scalars);
    }
    else
    {
        SCALAR_LOOP64F<op_add_weighted, T1, double, Tvec>(src1, step1, src2, step2,
            dst, step, width, height, scalars);
    }
}

template<>
void add_weighted_loop_d<double, v_float64>(const double* src1, size_t step1, const double* src2, size_t step2,
                                            double* dst, size_t step, int width, int height, const double* scalars)
{
    if (scalars[1] == 1.0 && scalars[2] == 0.0)
    {
        SCALAR_LOOP64F<op_add_scale, double, double, v_float64>(src1, step1, src2, step2,
            dst, step, width, height, scalars);
    }
    else
    {
        SCALAR_LOOP64F<op_add_weighted, double, double, v_float64>(src1, step1, src2, step2,
            dst, step, width, height, scalars);
    }
}

#endif // ARITHM_DEFINITIONS_ONLY
//////////////////////////////////////////////////////////////////////////
#undef DISPATCH_SIMD_FUN
#define DISPATCH_SIMD_FUN(fun, _T1, _Tvec, ...)                          \
    void fun(SCALAR_ARGS(_T1), void* scalar)                             \
    {                                                                    \
        CV_INSTRUMENT_REGION();                                          \
        CALL_HAL(fun, __CV_CAT(cv_hal_, fun),                            \
            SCALAR_ARGS_PASS, (const double*)scalar)                     \
        ARITHM_CALL_IPP(__CV_CAT(arithm_ipp_, fun),                      \
            SCALAR_ARGS_PASS, (const double*)scalar)                     \
        CV_CPU_DISPATCH(fun, (SCALAR_ARGS_PASS, (const double*)scalar),  \
            CV_CPU_DISPATCH_MODES_ALL);                                  \
    }
DEFINE_SIMD_SAT(addWeighted, add_weighted_loop)
DEFINE_SIMD_S32(addWeighted, add_weighted_loop_d)
DEFINE_SIMD_F32(addWeighted, add_weighted_loop_d)
DEFINE_SIMD_F64(addWeighted, add_weighted_loop_d)

//=======================================
// Reciprocal
//=======================================
#ifdef ARITHM_DEFINITIONS_ONLY
///////////////////////////// Operations //////////////////////////////////
template<typename T1, typename T2, typename Tvec>
struct op_recip
{
    static inline v_float32 r(const v_float32& a, const T2* scalar)
    {
        const v_float32 v_scalar = vx_setall_f32(*scalar);
        return v_scalar / a;
    }
    static inline Tvec pre(const Tvec& denom, const Tvec& res)
    {
        const Tvec v_zero = vx_setall<typename Tvec::lane_type>(0);
        return v_select(denom == v_zero, v_zero, res);
    }
    static inline T1 r(T1 denom, const T2* scalar)
    {
        CV_StaticAssert(std::numeric_limits<T1>::is_integer, "");
        return denom != (T1)0 ? c_div(*scalar, denom) : (T1)0;
    }
};

template<>
struct op_recip<float, float, v_float32>
{
    static inline v_float32 r(const v_float32& a, const float* scalar)
    {
        const v_float32 v_scalar = vx_setall_f32(*scalar);
        return v_scalar / a;
    }
    static inline float r(float denom, const float* scalar)
    { return c_div(*scalar, denom); }
};

template<>
struct op_recip<double, double, v_float64>
{
#if CV_SIMD_64F
    static inline v_float64 r(const v_float64& a, const double* scalar)
    {
        const v_float64 v_scalar = vx_setall_f64(*scalar);
        return v_scalar / a;
    }
#endif
    static inline double r(double denom, const double* scalar)
    { return c_div(*scalar, denom); }
};

//////////////////////////// Loops /////////////////////////////////
template<typename T1, typename Tvec>
static void recip_loop(const T1* src1, size_t step1, T1* dst, size_t step, int width, int height, const double* scalar)
{
    float fscalar = (float)*scalar;
    scalar_loop<op_recip, T1, float, Tvec>(src1, step1, dst, step, width, height, &fscalar);
}

template<>
void recip_loop<double, v_float64>(const double* src1, size_t step1, double* dst, size_t step, int width, int height, const double* scalar)
{
    SCALAR_LOOP64F<op_recip, double, double, v_float64>(src1, step1, dst, step, width, height, scalar);
}

#endif // ARITHM_DEFINITIONS_ONLY
//////////////////////////////////////////////////////////////////////////
#undef SCALAR_ARGS
#define SCALAR_ARGS(_T1) const _T1* src1, size_t step1, _T1* dst, size_t step, int width, int height
#undef SCALAR_ARGS_PASS
#define SCALAR_ARGS_PASS src1, step1, dst, step, width, height
#undef DISPATCH_SIMD_FUN
#define DISPATCH_SIMD_FUN(fun, _T1, _Tvec, ...)                          \
    void fun(const _T1*, size_t, SCALAR_ARGS(_T1), void* scalar)         \
    {                                                                    \
        CV_INSTRUMENT_REGION();                                          \
        CALL_HAL(fun, __CV_CAT(cv_hal_, fun),                            \
            SCALAR_ARGS_PASS, *(const double*)scalar)                    \
        ARITHM_CALL_IPP(__CV_CAT(arithm_ipp_, fun),                      \
            SCALAR_ARGS_PASS, *(const double*)scalar)                    \
        CV_CPU_DISPATCH(fun, (SCALAR_ARGS_PASS, (const double*)scalar),  \
            CV_CPU_DISPATCH_MODES_ALL);                                  \
    }
DEFINE_SIMD_ALL(recip, recip_loop)

#ifndef ARITHM_DISPATCHING_ONLY
    CV_CPU_OPTIMIZATION_NAMESPACE_END
#endif
#ifndef SIMD_GUARD
    #define SIMD_GUARD
#endif
}} // cv::hal::