audio: Added ARM NEON versions of audio converters.
authorRyan C. Gordon <icculus@icculus.org>
Wed, 16 May 2018 02:03:06 -0400
changeset 1199208c415f14810
parent 11991 2a487acdb306
child 11993 fdf104726ced
audio: Added ARM NEON versions of audio converters.

These are _much_ faster than the scalar equivalents on the Raspberry Pi that
I tested on. Often 3x to 4x as fast!
docs/README-raspberrypi.md
include/SDL_cpuinfo.h
src/audio/SDL_audiotypecvt.c
     1.1 --- a/docs/README-raspberrypi.md	Tue May 15 02:29:35 2018 -0400
     1.2 +++ b/docs/README-raspberrypi.md	Wed May 16 02:03:06 2018 -0400
     1.3 @@ -27,6 +27,16 @@
     1.4      
     1.5  sudo apt-get install libraspberrypi0 libraspberrypi-bin libraspberrypi-dev
     1.6  
     1.7 +
     1.8 +================================================================================
     1.9 + NEON
    1.10 +================================================================================
    1.11 +
    1.12 +If your Pi has NEON support, make sure you add -mfpu=neon to your CFLAGS so
    1.13 +that SDL will select some otherwise-disabled highly-optimized code. The
    1.14 +original Pi units don't have NEON, the Pi2 probably does, and the Pi3
    1.15 +definitely does.
    1.16 +
    1.17  ================================================================================
    1.18   Cross compiling from x86 Linux
    1.19  ================================================================================
     2.1 --- a/include/SDL_cpuinfo.h	Tue May 15 02:29:35 2018 -0400
     2.2 +++ b/include/SDL_cpuinfo.h	Wed May 16 02:03:06 2018 -0400
     2.3 @@ -57,6 +57,9 @@
     2.4  #undef bool
     2.5  #endif
     2.6  #endif
     2.7 +#if defined(__ARM_NEON__) && !defined(SDL_DISABLE_ARM_NEON_H)
     2.8 +#include <arm_neon.h>
     2.9 +#endif
    2.10  #if defined(__3dNOW__) && !defined(SDL_DISABLE_MM3DNOW_H)
    2.11  #include <mm3dnow.h>
    2.12  #endif
     3.1 --- a/src/audio/SDL_audiotypecvt.c	Tue May 15 02:29:35 2018 -0400
     3.2 +++ b/src/audio/SDL_audiotypecvt.c	Wed May 16 02:03:06 2018 -0400
     3.3 @@ -25,8 +25,9 @@
     3.4  #include "SDL_cpuinfo.h"
     3.5  #include "SDL_assert.h"
     3.6  
     3.7 -/* !!! FIXME: write NEON code. */
     3.8 -#define HAVE_NEON_INTRINSICS 0
     3.9 +#ifdef __ARM_NEON__
    3.10 +#define HAVE_NEON_INTRINSICS 1
    3.11 +#endif
    3.12  
    3.13  #ifdef __SSE2__
    3.14  #define HAVE_SSE2_INTRINSICS 1
    3.15 @@ -850,6 +851,538 @@
    3.16  #endif
    3.17  
    3.18  
    3.19 +#if HAVE_NEON_INTRINSICS
    3.20 +static void SDLCALL
    3.21 +SDL_Convert_S8_to_F32_NEON(SDL_AudioCVT *cvt, SDL_AudioFormat format)
    3.22 +{
    3.23 +    const Sint8 *src = ((const Sint8 *) (cvt->buf + cvt->len_cvt)) - 1;
    3.24 +    float *dst = ((float *) (cvt->buf + cvt->len_cvt * 4)) - 1;
    3.25 +    int i;
    3.26 +
    3.27 +    LOG_DEBUG_CONVERT("AUDIO_S8", "AUDIO_F32 (using NEON)");
    3.28 +
    3.29 +    /* Get dst aligned to 16 bytes (since buffer is growing, we don't have to worry about overreading from src) */
    3.30 +    for (i = cvt->len_cvt; i && (((size_t) (dst-15)) & 15); --i, --src, --dst) {
    3.31 +        *dst = ((float) *src) * DIVBY128;
    3.32 +    }
    3.33 +
    3.34 +    src -= 15; dst -= 15;  /* adjust to read NEON blocks from the start. */
    3.35 +    SDL_assert(!i || ((((size_t) dst) & 15) == 0));
    3.36 +
    3.37 +    /* Make sure src is aligned too. */
    3.38 +    if ((((size_t) src) & 15) == 0) {
    3.39 +        /* Aligned! Do NEON blocks as long as we have 16 bytes available. */
    3.40 +        const int8_t *mmsrc = (const int8_t *) src;
    3.41 +        const float32x4_t divby128 = vdupq_n_f32(DIVBY128);
    3.42 +        while (i >= 16) {   /* 16 * 8-bit */
    3.43 +            const int8x16_t bytes = vld1q_s8(mmsrc);  /* get 16 sint8 into a NEON register. */
    3.44 +            const int16x8_t int16hi = vmovl_s8(vget_high_s8(bytes));  /* convert top 8 bytes to 8 int16 */
    3.45 +            const int16x8_t int16lo = vmovl_s8(vget_low_s8(bytes));   /* convert bottom 8 bytes to 8 int16 */
    3.46 +            /* split int16 to two int32, then convert to float, then multiply to normalize, store. */
    3.47 +            vst1q_f32(dst, vmulq_f32(vcvtq_f32_s32(vmovl_s16(vget_high_s16(int16hi))), divby128));
    3.48 +            vst1q_f32(dst+4, vmulq_f32(vcvtq_f32_s32(vmovl_s16(vget_low_s16(int16hi))), divby128));
    3.49 +            vst1q_f32(dst+8, vmulq_f32(vcvtq_f32_s32(vmovl_s16(vget_high_s16(int16lo))), divby128));
    3.50 +            vst1q_f32(dst+12, vmulq_f32(vcvtq_f32_s32(vmovl_s16(vget_low_s16(int16lo))), divby128));
    3.51 +            i -= 16; mmsrc -= 16; dst -= 16;
    3.52 +        }
    3.53 +
    3.54 +        src = (const Sint8 *) mmsrc;
    3.55 +    }
    3.56 +
    3.57 +    src += 15; dst += 15;  /* adjust for any scalar finishing. */
    3.58 +
    3.59 +    /* Finish off any leftovers with scalar operations. */
    3.60 +    while (i) {
    3.61 +        *dst = ((float) *src) * DIVBY128;
    3.62 +        i--; src--; dst--;
    3.63 +    }
    3.64 +
    3.65 +    cvt->len_cvt *= 4;
    3.66 +    if (cvt->filters[++cvt->filter_index]) {
    3.67 +        cvt->filters[cvt->filter_index](cvt, AUDIO_F32SYS);
    3.68 +    }
    3.69 +}
    3.70 +
    3.71 +static void SDLCALL
    3.72 +SDL_Convert_U8_to_F32_NEON(SDL_AudioCVT *cvt, SDL_AudioFormat format)
    3.73 +{
    3.74 +    const Uint8 *src = ((const Uint8 *) (cvt->buf + cvt->len_cvt)) - 1;
    3.75 +    float *dst = ((float *) (cvt->buf + cvt->len_cvt * 4)) - 1;
    3.76 +    int i;
    3.77 +
    3.78 +    LOG_DEBUG_CONVERT("AUDIO_U8", "AUDIO_F32 (using NEON)");
    3.79 +
    3.80 +    /* Get dst aligned to 16 bytes (since buffer is growing, we don't have to worry about overreading from src) */
    3.81 +    for (i = cvt->len_cvt; i && (((size_t) (dst-15)) & 15); --i, --src, --dst) {
    3.82 +        *dst = (((float) *src) * DIVBY128) - 1.0f;
    3.83 +    }
    3.84 +
    3.85 +    src -= 15; dst -= 15;  /* adjust to read NEON blocks from the start. */
    3.86 +    SDL_assert(!i || ((((size_t) dst) & 15) == 0));
    3.87 +
    3.88 +    /* Make sure src is aligned too. */
    3.89 +    if ((((size_t) src) & 15) == 0) {
    3.90 +        /* Aligned! Do NEON blocks as long as we have 16 bytes available. */
    3.91 +        const uint8_t *mmsrc = (const uint8_t *) src;
    3.92 +        const float32x4_t divby128 = vdupq_n_f32(DIVBY128);
    3.93 +        const float32x4_t one = vdupq_n_f32(1.0f);
    3.94 +        while (i >= 16) {   /* 16 * 8-bit */
    3.95 +            const uint8x16_t bytes = vld1q_u8(mmsrc);  /* get 16 uint8 into a NEON register. */
    3.96 +            const uint16x8_t uint16hi = vmovl_u8(vget_high_u8(bytes));  /* convert top 8 bytes to 8 uint16 */
    3.97 +            const uint16x8_t uint16lo = vmovl_u8(vget_low_u8(bytes));   /* convert bottom 8 bytes to 8 uint16 */
    3.98 +            /* split uint16 to two uint32, then convert to float, then multiply to normalize, subtract to adjust for sign, store. */
    3.99 +            vst1q_f32(dst, vmlsq_f32(vcvtq_f32_u32(vmovl_u16(vget_high_u16(uint16hi))), divby128, one));
   3.100 +            vst1q_f32(dst+4, vmlsq_f32(vcvtq_f32_u32(vmovl_u16(vget_low_u16(uint16hi))), divby128, one));
   3.101 +            vst1q_f32(dst+8, vmlsq_f32(vcvtq_f32_u32(vmovl_u16(vget_high_u16(uint16lo))), divby128, one));
   3.102 +            vst1q_f32(dst+12, vmlsq_f32(vcvtq_f32_u32(vmovl_u16(vget_low_u16(uint16lo))), divby128, one));
   3.103 +            i -= 16; mmsrc -= 16; dst -= 16;
   3.104 +        }
   3.105 +
   3.106 +        src = (const Uint8 *) mmsrc;
   3.107 +    }
   3.108 +
   3.109 +    src += 15; dst += 15;  /* adjust for any scalar finishing. */
   3.110 +
   3.111 +    /* Finish off any leftovers with scalar operations. */
   3.112 +    while (i) {
   3.113 +        *dst = (((float) *src) * DIVBY128) - 1.0f;
   3.114 +        i--; src--; dst--;
   3.115 +    }
   3.116 +
   3.117 +    cvt->len_cvt *= 4;
   3.118 +    if (cvt->filters[++cvt->filter_index]) {
   3.119 +        cvt->filters[cvt->filter_index](cvt, AUDIO_F32SYS);
   3.120 +    }
   3.121 +}
   3.122 +
   3.123 +static void SDLCALL
   3.124 +SDL_Convert_S16_to_F32_NEON(SDL_AudioCVT *cvt, SDL_AudioFormat format)
   3.125 +{
   3.126 +    const Sint16 *src = ((const Sint16 *) (cvt->buf + cvt->len_cvt)) - 1;
   3.127 +    float *dst = ((float *) (cvt->buf + cvt->len_cvt * 2)) - 1;
   3.128 +    int i;
   3.129 +
   3.130 +    LOG_DEBUG_CONVERT("AUDIO_S16", "AUDIO_F32 (using NEON)");
   3.131 +
   3.132 +    /* Get dst aligned to 16 bytes (since buffer is growing, we don't have to worry about overreading from src) */
   3.133 +    for (i = cvt->len_cvt / sizeof (Sint16); i && (((size_t) (dst-7)) & 15); --i, --src, --dst) {
   3.134 +        *dst = ((float) *src) * DIVBY32768;
   3.135 +    }
   3.136 +
   3.137 +    src -= 7; dst -= 7;  /* adjust to read NEON blocks from the start. */
   3.138 +    SDL_assert(!i || ((((size_t) dst) & 15) == 0));
   3.139 +
   3.140 +    /* Make sure src is aligned too. */
   3.141 +    if ((((size_t) src) & 15) == 0) {
   3.142 +        /* Aligned! Do NEON blocks as long as we have 16 bytes available. */
   3.143 +        const float32x4_t divby32768 = vdupq_n_f32(DIVBY32768);
   3.144 +        while (i >= 8) {   /* 8 * 16-bit */
   3.145 +            const int16x8_t ints = vld1q_s16((int16_t const *) src);  /* get 8 sint16 into a NEON register. */
   3.146 +            /* split int16 to two int32, then convert to float, then multiply to normalize, store. */
   3.147 +            vst1q_f32(dst, vmulq_f32(vcvtq_f32_s32(vmovl_s16(vget_low_s16(ints))), divby32768));
   3.148 +            vst1q_f32(dst+4, vmulq_f32(vcvtq_f32_s32(vmovl_s16(vget_high_s16(ints))), divby32768));
   3.149 +            i -= 8; src -= 8; dst -= 8;
   3.150 +        }
   3.151 +    }
   3.152 +
   3.153 +    src += 7; dst += 7;  /* adjust for any scalar finishing. */
   3.154 +
   3.155 +    /* Finish off any leftovers with scalar operations. */
   3.156 +    while (i) {
   3.157 +        *dst = ((float) *src) * DIVBY32768;
   3.158 +        i--; src--; dst--;
   3.159 +    }
   3.160 +
   3.161 +    cvt->len_cvt *= 2;
   3.162 +    if (cvt->filters[++cvt->filter_index]) {
   3.163 +        cvt->filters[cvt->filter_index](cvt, AUDIO_F32SYS);
   3.164 +    }
   3.165 +}
   3.166 +
   3.167 +static void SDLCALL
   3.168 +SDL_Convert_U16_to_F32_NEON(SDL_AudioCVT *cvt, SDL_AudioFormat format)
   3.169 +{
   3.170 +    const Uint16 *src = ((const Uint16 *) (cvt->buf + cvt->len_cvt)) - 1;
   3.171 +    float *dst = ((float *) (cvt->buf + cvt->len_cvt * 2)) - 1;
   3.172 +    int i;
   3.173 +
   3.174 +    LOG_DEBUG_CONVERT("AUDIO_U16", "AUDIO_F32 (using NEON)");
   3.175 +
   3.176 +    /* Get dst aligned to 16 bytes (since buffer is growing, we don't have to worry about overreading from src) */
   3.177 +    for (i = cvt->len_cvt / sizeof (Sint16); i && (((size_t) (dst-7)) & 15); --i, --src, --dst) {
   3.178 +        *dst = (((float) *src) * DIVBY32768) - 1.0f;
   3.179 +    }
   3.180 +
   3.181 +    src -= 7; dst -= 7;  /* adjust to read NEON blocks from the start. */
   3.182 +    SDL_assert(!i || ((((size_t) dst) & 15) == 0));
   3.183 +
   3.184 +    /* Make sure src is aligned too. */
   3.185 +    if ((((size_t) src) & 15) == 0) {
   3.186 +        /* Aligned! Do NEON blocks as long as we have 16 bytes available. */
   3.187 +        const float32x4_t divby32768 = vdupq_n_f32(DIVBY32768);
   3.188 +        const float32x4_t one = vdupq_n_f32(1.0f);
   3.189 +        while (i >= 8) {   /* 8 * 16-bit */
   3.190 +            const uint16x8_t uints = vld1q_u16((uint16_t const *) src);  /* get 8 uint16 into a NEON register. */
   3.191 +            /* split uint16 to two int32, then convert to float, then multiply to normalize, subtract for sign, store. */
   3.192 +            vst1q_f32(dst, vmlsq_f32(one, vcvtq_f32_u32(vmovl_u16(vget_low_u16(uints))), divby32768));
   3.193 +            vst1q_f32(dst+4, vmlsq_f32(one, vcvtq_f32_u32(vmovl_u16(vget_high_u16(uints))), divby32768));
   3.194 +            i -= 8; src -= 8; dst -= 8;
   3.195 +        }
   3.196 +    }
   3.197 +
   3.198 +    src += 7; dst += 7;  /* adjust for any scalar finishing. */
   3.199 +
   3.200 +    /* Finish off any leftovers with scalar operations. */
   3.201 +    while (i) {
   3.202 +        *dst = (((float) *src) * DIVBY32768) - 1.0f;
   3.203 +        i--; src--; dst--;
   3.204 +    }
   3.205 +
   3.206 +    cvt->len_cvt *= 2;
   3.207 +    if (cvt->filters[++cvt->filter_index]) {
   3.208 +        cvt->filters[cvt->filter_index](cvt, AUDIO_F32SYS);
   3.209 +    }
   3.210 +}
   3.211 +
   3.212 +static void SDLCALL
   3.213 +SDL_Convert_S32_to_F32_NEON(SDL_AudioCVT *cvt, SDL_AudioFormat format)
   3.214 +{
   3.215 +    const Sint32 *src = (const Sint32 *) cvt->buf;
   3.216 +    float *dst = (float *) cvt->buf;
   3.217 +    int i;
   3.218 +
   3.219 +    LOG_DEBUG_CONVERT("AUDIO_S32", "AUDIO_F32 (using NEON)");
   3.220 +
   3.221 +    /* Get dst aligned to 16 bytes */
   3.222 +    for (i = cvt->len_cvt / sizeof (Sint32); i && (((size_t) dst) & 15); --i, ++src, ++dst) {
   3.223 +        *dst = ((float) (*src>>8)) * DIVBY8388607;
   3.224 +    }
   3.225 +
   3.226 +    SDL_assert(!i || ((((size_t) dst) & 15) == 0));
   3.227 +    SDL_assert(!i || ((((size_t) src) & 15) == 0));
   3.228 +
   3.229 +    {
   3.230 +        /* Aligned! Do NEON blocks as long as we have 16 bytes available. */
   3.231 +        const float32x4_t divby8388607 = vdupq_n_f32(DIVBY8388607);
   3.232 +        const int32_t *mmsrc = (const int32_t *) src;
   3.233 +        while (i >= 4) {   /* 4 * sint32 */
   3.234 +            /* shift out lowest bits so int fits in a float32. Small precision loss, but much faster. */
   3.235 +            vst1q_f32(dst, vmulq_f32(vcvtq_f32_s32(vshrq_n_s32(vld1q_s32(mmsrc), 8)), divby8388607));
   3.236 +            i -= 4; mmsrc += 4; dst += 4;
   3.237 +        }
   3.238 +        src = (const Sint32 *) mmsrc;
   3.239 +    }
   3.240 +
   3.241 +    /* Finish off any leftovers with scalar operations. */
   3.242 +    while (i) {
   3.243 +        *dst = ((float) (*src>>8)) * DIVBY8388607;
   3.244 +        i--; src++; dst++;
   3.245 +    }
   3.246 +
   3.247 +    if (cvt->filters[++cvt->filter_index]) {
   3.248 +        cvt->filters[cvt->filter_index](cvt, AUDIO_F32SYS);
   3.249 +    }
   3.250 +}
   3.251 +
   3.252 +static void SDLCALL
   3.253 +SDL_Convert_F32_to_S8_NEON(SDL_AudioCVT *cvt, SDL_AudioFormat format)
   3.254 +{
   3.255 +    const float *src = (const float *) cvt->buf;
   3.256 +    Sint8 *dst = (Sint8 *) cvt->buf;
   3.257 +    int i;
   3.258 +
   3.259 +    LOG_DEBUG_CONVERT("AUDIO_F32", "AUDIO_S8 (using NEON)");
   3.260 +
   3.261 +    /* Get dst aligned to 16 bytes */
   3.262 +    for (i = cvt->len_cvt / sizeof (float); i && (((size_t) dst) & 15); --i, ++src, ++dst) {
   3.263 +        const float sample = *src;
   3.264 +        if (sample >= 1.0f) {
   3.265 +            *dst = 127;
   3.266 +        } else if (sample <= -1.0f) {
   3.267 +            *dst = -128;
   3.268 +        } else {
   3.269 +            *dst = (Sint8)(sample * 127.0f);
   3.270 +        }
   3.271 +    }
   3.272 +
   3.273 +    SDL_assert(!i || ((((size_t) dst) & 15) == 0));
   3.274 +
   3.275 +    /* Make sure src is aligned too. */
   3.276 +    if ((((size_t) src) & 15) == 0) {
   3.277 +        /* Aligned! Do NEON blocks as long as we have 16 bytes available. */
   3.278 +        const float32x4_t one = vdupq_n_f32(1.0f);
   3.279 +        const float32x4_t negone = vdupq_n_f32(-1.0f);
   3.280 +        const float32x4_t mulby127 = vdupq_n_f32(127.0f);
   3.281 +        int8_t *mmdst = (int8_t *) dst;
   3.282 +        while (i >= 16) {   /* 16 * float32 */
   3.283 +            const int32x4_t ints1 = vcvtq_s32_f32(vmulq_f32(vminq_f32(vmaxq_f32(negone, vld1q_f32(src)), one), mulby127));  /* load 4 floats, clamp, convert to sint32 */
   3.284 +            const int32x4_t ints2 = vcvtq_s32_f32(vmulq_f32(vminq_f32(vmaxq_f32(negone, vld1q_f32(src+4)), one), mulby127));  /* load 4 floats, clamp, convert to sint32 */
   3.285 +            const int32x4_t ints3 = vcvtq_s32_f32(vmulq_f32(vminq_f32(vmaxq_f32(negone, vld1q_f32(src+8)), one), mulby127));  /* load 4 floats, clamp, convert to sint32 */
   3.286 +            const int32x4_t ints4 = vcvtq_s32_f32(vmulq_f32(vminq_f32(vmaxq_f32(negone, vld1q_f32(src+12)), one), mulby127));  /* load 4 floats, clamp, convert to sint32 */
   3.287 +            const int8x8_t i8lo = vmovn_s16(vcombine_s16(vmovn_s32(ints1), vmovn_s32(ints2))); /* narrow to sint16, combine, narrow to sint8 */
   3.288 +            const int8x8_t i8hi = vmovn_s16(vcombine_s16(vmovn_s32(ints3), vmovn_s32(ints4))); /* narrow to sint16, combine, narrow to sint8 */
   3.289 +            vst1q_s8(mmdst, vcombine_s8(i8lo, i8hi));  /* combine to int8x16_t, store out */
   3.290 +            i -= 16; src += 16; mmdst += 16;
   3.291 +        }
   3.292 +        dst = (Sint8 *) mmdst;
   3.293 +    }
   3.294 +
   3.295 +    /* Finish off any leftovers with scalar operations. */
   3.296 +    while (i) {
   3.297 +        const float sample = *src;
   3.298 +        if (sample >= 1.0f) {
   3.299 +            *dst = 127;
   3.300 +        } else if (sample <= -1.0f) {
   3.301 +            *dst = -128;
   3.302 +        } else {
   3.303 +            *dst = (Sint8)(sample * 127.0f);
   3.304 +        }
   3.305 +        i--; src++; dst++;
   3.306 +    }
   3.307 +
   3.308 +    cvt->len_cvt /= 4;
   3.309 +    if (cvt->filters[++cvt->filter_index]) {
   3.310 +        cvt->filters[cvt->filter_index](cvt, AUDIO_S8);
   3.311 +    }
   3.312 +}
   3.313 +
   3.314 +static void SDLCALL
   3.315 +SDL_Convert_F32_to_U8_NEON(SDL_AudioCVT *cvt, SDL_AudioFormat format)
   3.316 +{
   3.317 +    const float *src = (const float *) cvt->buf;
   3.318 +    Uint8 *dst = (Uint8 *) cvt->buf;
   3.319 +    int i;
   3.320 +
   3.321 +    LOG_DEBUG_CONVERT("AUDIO_F32", "AUDIO_U8 (using NEON)");
   3.322 +
   3.323 +    /* Get dst aligned to 16 bytes */
   3.324 +    for (i = cvt->len_cvt / sizeof (float); i && (((size_t) dst) & 15); --i, ++src, ++dst) {
   3.325 +        const float sample = *src;
   3.326 +        if (sample >= 1.0f) {
   3.327 +            *dst = 255;
   3.328 +        } else if (sample <= -1.0f) {
   3.329 +            *dst = 0;
   3.330 +        } else {
   3.331 +            *dst = (Uint8)((sample + 1.0f) * 127.0f);
   3.332 +        }
   3.333 +    }
   3.334 +
   3.335 +    SDL_assert(!i || ((((size_t) dst) & 15) == 0));
   3.336 +
   3.337 +    /* Make sure src is aligned too. */
   3.338 +    if ((((size_t) src) & 15) == 0) {
   3.339 +        /* Aligned! Do NEON blocks as long as we have 16 bytes available. */
   3.340 +        const float32x4_t one = vdupq_n_f32(1.0f);
   3.341 +        const float32x4_t negone = vdupq_n_f32(-1.0f);
   3.342 +        const float32x4_t mulby127 = vdupq_n_f32(127.0f);
   3.343 +        uint8_t *mmdst = (uint8_t *) dst;
   3.344 +        while (i >= 16) {   /* 16 * float32 */
   3.345 +            const uint32x4_t uints1 = vcvtq_u32_f32(vmulq_f32(vaddq_f32(vminq_f32(vmaxq_f32(negone, vld1q_f32(src)), one), one), mulby127));  /* load 4 floats, clamp, convert to uint32 */
   3.346 +            const uint32x4_t uints2 = vcvtq_u32_f32(vmulq_f32(vaddq_f32(vminq_f32(vmaxq_f32(negone, vld1q_f32(src+4)), one), one), mulby127));  /* load 4 floats, clamp, convert to uint32 */
   3.347 +            const uint32x4_t uints3 = vcvtq_u32_f32(vmulq_f32(vaddq_f32(vminq_f32(vmaxq_f32(negone, vld1q_f32(src+8)), one), one), mulby127));  /* load 4 floats, clamp, convert to uint32 */
   3.348 +            const uint32x4_t uints4 = vcvtq_u32_f32(vmulq_f32(vaddq_f32(vminq_f32(vmaxq_f32(negone, vld1q_f32(src+12)), one), one), mulby127));  /* load 4 floats, clamp, convert to uint32 */
   3.349 +            const uint8x8_t ui8lo = vmovn_u16(vcombine_u16(vmovn_u32(uints1), vmovn_u32(uints2))); /* narrow to uint16, combine, narrow to uint8 */
   3.350 +            const uint8x8_t ui8hi = vmovn_u16(vcombine_u16(vmovn_u32(uints3), vmovn_u32(uints4))); /* narrow to uint16, combine, narrow to uint8 */
   3.351 +            vst1q_u8(mmdst, vcombine_u8(ui8lo, ui8hi));  /* combine to uint8x16_t, store out */
   3.352 +            i -= 16; src += 16; mmdst += 16;
   3.353 +        }
   3.354 +
   3.355 +        dst = (Uint8 *) mmdst;
   3.356 +    }
   3.357 +
   3.358 +    /* Finish off any leftovers with scalar operations. */
   3.359 +    while (i) {
   3.360 +        const float sample = *src;
   3.361 +        if (sample >= 1.0f) {
   3.362 +            *dst = 255;
   3.363 +        } else if (sample <= -1.0f) {
   3.364 +            *dst = 0;
   3.365 +        } else {
   3.366 +            *dst = (Uint8)((sample + 1.0f) * 127.0f);
   3.367 +        }
   3.368 +        i--; src++; dst++;
   3.369 +    }
   3.370 +
   3.371 +    cvt->len_cvt /= 4;
   3.372 +    if (cvt->filters[++cvt->filter_index]) {
   3.373 +        cvt->filters[cvt->filter_index](cvt, AUDIO_U8);
   3.374 +    }
   3.375 +}
   3.376 +
   3.377 +static void SDLCALL
   3.378 +SDL_Convert_F32_to_S16_NEON(SDL_AudioCVT *cvt, SDL_AudioFormat format)
   3.379 +{
   3.380 +    const float *src = (const float *) cvt->buf;
   3.381 +    Sint16 *dst = (Sint16 *) cvt->buf;
   3.382 +    int i;
   3.383 +
   3.384 +    LOG_DEBUG_CONVERT("AUDIO_F32", "AUDIO_S16 (using NEON)");
   3.385 +
   3.386 +    /* Get dst aligned to 16 bytes */
   3.387 +    for (i = cvt->len_cvt / sizeof (float); i && (((size_t) dst) & 15); --i, ++src, ++dst) {
   3.388 +        const float sample = *src;
   3.389 +        if (sample >= 1.0f) {
   3.390 +            *dst = 32767;
   3.391 +        } else if (sample <= -1.0f) {
   3.392 +            *dst = -32768;
   3.393 +        } else {
   3.394 +            *dst = (Sint16)(sample * 32767.0f);
   3.395 +        }
   3.396 +    }
   3.397 +
   3.398 +    SDL_assert(!i || ((((size_t) dst) & 15) == 0));
   3.399 +
   3.400 +    /* Make sure src is aligned too. */
   3.401 +    if ((((size_t) src) & 15) == 0) {
   3.402 +        /* Aligned! Do NEON blocks as long as we have 16 bytes available. */
   3.403 +        const float32x4_t one = vdupq_n_f32(1.0f);
   3.404 +        const float32x4_t negone = vdupq_n_f32(-1.0f);
   3.405 +        const float32x4_t mulby32767 = vdupq_n_f32(32767.0f);
   3.406 +        int16_t *mmdst = (int16_t *) dst;
   3.407 +        while (i >= 8) {   /* 8 * float32 */
   3.408 +            const int32x4_t ints1 = vcvtq_s32_f32(vmulq_f32(vminq_f32(vmaxq_f32(negone, vld1q_f32(src)), one), mulby32767));  /* load 4 floats, clamp, convert to sint32 */
   3.409 +            const int32x4_t ints2 = vcvtq_s32_f32(vmulq_f32(vminq_f32(vmaxq_f32(negone, vld1q_f32(src+4)), one), mulby32767));  /* load 4 floats, clamp, convert to sint32 */
   3.410 +            vst1q_s16(mmdst, vcombine_s16(vmovn_s32(ints1), vmovn_s32(ints2)));  /* narrow to sint16, combine, store out. */
   3.411 +            i -= 8; src += 8; mmdst += 8;
   3.412 +        }
   3.413 +        dst = (Sint16 *) mmdst;
   3.414 +    }
   3.415 +
   3.416 +    /* Finish off any leftovers with scalar operations. */
   3.417 +    while (i) {
   3.418 +        const float sample = *src;
   3.419 +        if (sample >= 1.0f) {
   3.420 +            *dst = 32767;
   3.421 +        } else if (sample <= -1.0f) {
   3.422 +            *dst = -32768;
   3.423 +        } else {
   3.424 +            *dst = (Sint16)(sample * 32767.0f);
   3.425 +        }
   3.426 +        i--; src++; dst++;
   3.427 +    }
   3.428 +
   3.429 +    cvt->len_cvt /= 2;
   3.430 +    if (cvt->filters[++cvt->filter_index]) {
   3.431 +        cvt->filters[cvt->filter_index](cvt, AUDIO_S16SYS);
   3.432 +    }
   3.433 +}
   3.434 +
   3.435 +static void SDLCALL
   3.436 +SDL_Convert_F32_to_U16_NEON(SDL_AudioCVT *cvt, SDL_AudioFormat format)
   3.437 +{
   3.438 +    const float *src = (const float *) cvt->buf;
   3.439 +    Uint16 *dst = (Uint16 *) cvt->buf;
   3.440 +    int i;
   3.441 +
   3.442 +    LOG_DEBUG_CONVERT("AUDIO_F32", "AUDIO_U16 (using NEON)");
   3.443 +
   3.444 +    /* Get dst aligned to 16 bytes */
   3.445 +    for (i = cvt->len_cvt / sizeof (float); i && (((size_t) dst) & 15); --i, ++src, ++dst) {
   3.446 +        const float sample = *src;
   3.447 +        if (sample >= 1.0f) {
   3.448 +            *dst = 65535;
   3.449 +        } else if (sample <= -1.0f) {
   3.450 +            *dst = 0;
   3.451 +        } else {
   3.452 +            *dst = (Uint16)((sample + 1.0f) * 32767.0f);
   3.453 +        }
   3.454 +    }
   3.455 +
   3.456 +    SDL_assert(!i || ((((size_t) dst) & 15) == 0));
   3.457 +
   3.458 +    /* Make sure src is aligned too. */
   3.459 +    if ((((size_t) src) & 15) == 0) {
   3.460 +        /* Aligned! Do NEON blocks as long as we have 16 bytes available. */
   3.461 +        const float32x4_t one = vdupq_n_f32(1.0f);
   3.462 +        const float32x4_t negone = vdupq_n_f32(-1.0f);
   3.463 +        const float32x4_t mulby32767 = vdupq_n_f32(32767.0f);
   3.464 +        uint16_t *mmdst = (uint16_t *) dst;
   3.465 +        while (i >= 8) {   /* 8 * float32 */
   3.466 +            const uint32x4_t uints1 = vcvtq_u32_f32(vmulq_f32(vaddq_f32(vminq_f32(vmaxq_f32(negone, vld1q_f32(src)), one), one), mulby32767));  /* load 4 floats, clamp, convert to uint32 */
   3.467 +            const uint32x4_t uints2 = vcvtq_u32_f32(vmulq_f32(vaddq_f32(vminq_f32(vmaxq_f32(negone, vld1q_f32(src+4)), one), one), mulby32767));  /* load 4 floats, clamp, convert to uint32 */
   3.468 +            vst1q_u16(mmdst, vcombine_u16(vmovn_u32(uints1), vmovn_u32(uints2)));  /* narrow to uint16, combine, store out. */
   3.469 +            i -= 8; src += 8; mmdst += 8;
   3.470 +        }
   3.471 +        dst = (Uint16 *) mmdst;
   3.472 +    }
   3.473 +
   3.474 +    /* Finish off any leftovers with scalar operations. */
   3.475 +    while (i) {
   3.476 +        const float sample = *src;
   3.477 +        if (sample >= 1.0f) {
   3.478 +            *dst = 65535;
   3.479 +        } else if (sample <= -1.0f) {
   3.480 +            *dst = 0;
   3.481 +        } else {
   3.482 +            *dst = (Uint16)((sample + 1.0f) * 32767.0f);
   3.483 +        }
   3.484 +        i--; src++; dst++;
   3.485 +    }
   3.486 +
   3.487 +    cvt->len_cvt /= 2;
   3.488 +    if (cvt->filters[++cvt->filter_index]) {
   3.489 +        cvt->filters[cvt->filter_index](cvt, AUDIO_U16SYS);
   3.490 +    }
   3.491 +}
   3.492 +
   3.493 +static void SDLCALL
   3.494 +SDL_Convert_F32_to_S32_NEON(SDL_AudioCVT *cvt, SDL_AudioFormat format)
   3.495 +{
   3.496 +    const float *src = (const float *) cvt->buf;
   3.497 +    Sint32 *dst = (Sint32 *) cvt->buf;
   3.498 +    int i;
   3.499 +
   3.500 +    LOG_DEBUG_CONVERT("AUDIO_F32", "AUDIO_S32 (using NEON)");
   3.501 +
   3.502 +    /* Get dst aligned to 16 bytes */
   3.503 +    for (i = cvt->len_cvt / sizeof (float); i && (((size_t) dst) & 15); --i, ++src, ++dst) {
   3.504 +        const float sample = *src;
   3.505 +        if (sample >= 1.0f) {
   3.506 +            *dst = 2147483647;
   3.507 +        } else if (sample <= -1.0f) {
   3.508 +            *dst = -2147483648;
   3.509 +        } else {
   3.510 +            *dst = ((Sint32)(sample * 8388607.0f)) << 8;
   3.511 +        }
   3.512 +    }
   3.513 +
   3.514 +    SDL_assert(!i || ((((size_t) dst) & 15) == 0));
   3.515 +    SDL_assert(!i || ((((size_t) src) & 15) == 0));
   3.516 +
   3.517 +    {
   3.518 +        /* Aligned! Do NEON blocks as long as we have 16 bytes available. */
   3.519 +        const float32x4_t one = vdupq_n_f32(1.0f);
   3.520 +        const float32x4_t negone = vdupq_n_f32(-1.0f);
   3.521 +        const float32x4_t mulby8388607 = vdupq_n_f32(8388607.0f);
   3.522 +        int32_t *mmdst = (int32_t *) dst;
   3.523 +        while (i >= 4) {   /* 4 * float32 */
   3.524 +            vst1q_s32(mmdst, vshlq_n_s32(vcvtq_s32_f32(vmulq_f32(vminq_f32(vmaxq_f32(negone, vld1q_f32(src)), one), mulby8388607)), 8));
   3.525 +            i -= 4; src += 4; mmdst += 4;
   3.526 +        }
   3.527 +        dst = (Sint32 *) mmdst;
   3.528 +    }
   3.529 +
   3.530 +    /* Finish off any leftovers with scalar operations. */
   3.531 +    while (i) {
   3.532 +        const float sample = *src;
   3.533 +        if (sample >= 1.0f) {
   3.534 +            *dst = 2147483647;
   3.535 +        } else if (sample <= -1.0f) {
   3.536 +            *dst = -2147483648;
   3.537 +        } else {
   3.538 +            *dst = ((Sint32)(sample * 8388607.0f)) << 8;
   3.539 +        }
   3.540 +        i--; src++; dst++;
   3.541 +    }
   3.542 +
   3.543 +    if (cvt->filters[++cvt->filter_index]) {
   3.544 +        cvt->filters[cvt->filter_index](cvt, AUDIO_S32SYS);
   3.545 +    }
   3.546 +}
   3.547 +#endif
   3.548 +
   3.549 +
   3.550 +
   3.551  void SDL_ChooseAudioConverters(void)
   3.552  {
   3.553      static SDL_bool converters_chosen = SDL_FALSE;
   3.554 @@ -878,6 +1411,13 @@
   3.555      }
   3.556  #endif
   3.557  
   3.558 +#if HAVE_NEON_INTRINSICS
   3.559 +    if (SDL_HasNEON()) {
   3.560 +        SET_CONVERTER_FUNCS(NEON);
   3.561 +        return;
   3.562 +    }
   3.563 +#endif
   3.564 +
   3.565  #if NEED_SCALAR_CONVERTER_FALLBACKS
   3.566      SET_CONVERTER_FUNCS(Scalar);
   3.567  #endif