From 5718293092100420a077ba10a86c1ff7c2b8fcfb Mon Sep 17 00:00:00 2001 From: "Ryan C. Gordon" Date: Mon, 16 Jan 2017 00:58:28 -0500 Subject: [PATCH] audio: Implemented SIMD support for audio data type converters. This currently adds an SSE2 implementation (but it's #ifdef'd out for now, until it's hooked up to the configure script and such). --- src/audio/SDL_audio.c | 4 + src/audio/SDL_audio_c.h | 21 +- src/audio/SDL_audiotypecvt.c | 608 +++++++++++++++++++++++++++++++++-- 3 files changed, 603 insertions(+), 30 deletions(-) diff --git a/src/audio/SDL_audio.c b/src/audio/SDL_audio.c index fa2a2b8e10581..a62ef4234e3ba 100644 --- a/src/audio/SDL_audio.c +++ b/src/audio/SDL_audio.c @@ -843,6 +843,8 @@ SDL_GetAudioDriver(int index) return NULL; } +extern void SDL_ChooseAudioConverters(void); + int SDL_AudioInit(const char *driver_name) { @@ -857,6 +859,8 @@ SDL_AudioInit(const char *driver_name) SDL_zero(current_audio); SDL_zero(open_devices); + SDL_ChooseAudioConverters(); + /* Select the proper audio driver */ if (driver_name == NULL) { driver_name = SDL_getenv("SDL_AUDIODRIVER"); diff --git a/src/audio/SDL_audio_c.h b/src/audio/SDL_audio_c.h index 62fd6bcc8e2b7..8a8af137ac548 100644 --- a/src/audio/SDL_audio_c.h +++ b/src/audio/SDL_audio_c.h @@ -53,16 +53,17 @@ extern SDL_AudioFormat SDL_NextAudioFormat(void); /* Function to calculate the size and silence for a SDL_AudioSpec */ extern void SDL_CalculateAudioSpec(SDL_AudioSpec * spec); -void SDLCALL SDL_Convert_S8_to_F32(SDL_AudioCVT *cvt, SDL_AudioFormat format); -void SDLCALL SDL_Convert_U8_to_F32(SDL_AudioCVT *cvt, SDL_AudioFormat format); -void SDLCALL SDL_Convert_S16_to_F32(SDL_AudioCVT *cvt, SDL_AudioFormat format); -void SDLCALL SDL_Convert_U16_to_F32(SDL_AudioCVT *cvt, SDL_AudioFormat format); -void SDLCALL SDL_Convert_S32_to_F32(SDL_AudioCVT *cvt, SDL_AudioFormat format); -void SDLCALL SDL_Convert_F32_to_S8(SDL_AudioCVT *cvt, SDL_AudioFormat format); -void SDLCALL SDL_Convert_F32_to_U8(SDL_AudioCVT *cvt, SDL_AudioFormat format); -void SDLCALL SDL_Convert_F32_to_S16(SDL_AudioCVT *cvt, SDL_AudioFormat format); -void SDLCALL SDL_Convert_F32_to_U16(SDL_AudioCVT *cvt, SDL_AudioFormat format); -void SDLCALL SDL_Convert_F32_to_S32(SDL_AudioCVT *cvt, SDL_AudioFormat format); +/* These pointers get set during init to various SIMD implementations. */ +extern SDL_AudioFilter SDL_Convert_S8_to_F32; +extern SDL_AudioFilter SDL_Convert_U8_to_F32; +extern SDL_AudioFilter SDL_Convert_S16_to_F32; +extern SDL_AudioFilter SDL_Convert_U16_to_F32; +extern SDL_AudioFilter SDL_Convert_S32_to_F32; +extern SDL_AudioFilter SDL_Convert_F32_to_S8; +extern SDL_AudioFilter SDL_Convert_F32_to_U8; +extern SDL_AudioFilter SDL_Convert_F32_to_S16; +extern SDL_AudioFilter SDL_Convert_F32_to_U16; +extern SDL_AudioFilter SDL_Convert_F32_to_S32; /* SDL_AudioStream is a new audio conversion interface. It diff --git a/src/audio/SDL_audiotypecvt.c b/src/audio/SDL_audiotypecvt.c index cebfca6e5cdbb..505e25a007573 100644 --- a/src/audio/SDL_audiotypecvt.c +++ b/src/audio/SDL_audiotypecvt.c @@ -22,14 +22,55 @@ #include "../SDL_internal.h" #include "SDL_audio.h" #include "SDL_audio_c.h" +#include "SDL_cpuinfo.h" #include "SDL_assert.h" +/* !!! FIXME: write NEON code. */ +#define HAVE_NEON_INTRINSICS 0 + +/* !!! FIXME: wire this up to the configure script, etc. */ +#define HAVE_SSE2_INTRINSICS 0 + +#if HAVE_SSE2_INTRINSICS +#include +#endif + +#if defined(__x86_64__) && HAVE_SSE2_INTRINSICS +#define NEED_SCALAR_CONVERTER_FALLBACKS 0 /* x86_64 guarantees SSE2. */ +#elif __MACOSX__ && HAVE_SSE2_INTRINSICS +#define NEED_SCALAR_CONVERTER_FALLBACKS 0 /* Mac OS X/Intel guarantees SSE2. */ +#elif defined(__ARM_ARCH) && (__ARM_ARCH >= 8) && HAVE_NEON_INTRINSICS +#define NEED_SCALAR_CONVERTER_FALLBACKS 0 /* ARMv8+ promise NEON. */ +#elif defined(__APPLE__) && defined(__ARM_ARCH) && (__ARM_ARCH >= 7) && HAVE_NEON_INTRINSICS +#define NEED_SCALAR_CONVERTER_FALLBACKS 0 /* All Apple ARMv7 chips promise NEON support. */ +#endif + +/* Set to zero if platform is guaranteed to use a SIMD codepath here. */ +#ifndef NEED_SCALAR_CONVERTER_FALLBACKS +#define NEED_SCALAR_CONVERTER_FALLBACKS 1 +#endif + +/* Function pointers set to a CPU-specific implementation. */ +SDL_AudioFilter SDL_Convert_S8_to_F32 = NULL; +SDL_AudioFilter SDL_Convert_U8_to_F32 = NULL; +SDL_AudioFilter SDL_Convert_S16_to_F32 = NULL; +SDL_AudioFilter SDL_Convert_U16_to_F32 = NULL; +SDL_AudioFilter SDL_Convert_S32_to_F32 = NULL; +SDL_AudioFilter SDL_Convert_F32_to_S8 = NULL; +SDL_AudioFilter SDL_Convert_F32_to_U8 = NULL; +SDL_AudioFilter SDL_Convert_F32_to_S16 = NULL; +SDL_AudioFilter SDL_Convert_F32_to_U16 = NULL; +SDL_AudioFilter SDL_Convert_F32_to_S32 = NULL; + + #define DIVBY127 0.0078740157480315f #define DIVBY32767 3.05185094759972e-05f #define DIVBY2147483647 4.6566128752458e-10f -void SDLCALL -SDL_Convert_S8_to_F32(SDL_AudioCVT *cvt, SDL_AudioFormat format) + +#if NEED_SCALAR_CONVERTER_FALLBACKS +static void SDLCALL +SDL_Convert_S8_to_F32_Scalar(SDL_AudioCVT *cvt, SDL_AudioFormat format) { const Sint8 *src = ((const Sint8 *) (cvt->buf + cvt->len_cvt)) - 1; float *dst = ((float *) (cvt->buf + cvt->len_cvt * 4)) - 1; @@ -47,8 +88,8 @@ SDL_Convert_S8_to_F32(SDL_AudioCVT *cvt, SDL_AudioFormat format) } } -void SDLCALL -SDL_Convert_U8_to_F32(SDL_AudioCVT *cvt, SDL_AudioFormat format) +static void SDLCALL +SDL_Convert_U8_to_F32_Scalar(SDL_AudioCVT *cvt, SDL_AudioFormat format) { const Uint8 *src = ((const Uint8 *) (cvt->buf + cvt->len_cvt)) - 1; float *dst = ((float *) (cvt->buf + cvt->len_cvt * 4)) - 1; @@ -66,8 +107,8 @@ SDL_Convert_U8_to_F32(SDL_AudioCVT *cvt, SDL_AudioFormat format) } } -void SDLCALL -SDL_Convert_S16_to_F32(SDL_AudioCVT *cvt, SDL_AudioFormat format) +static void SDLCALL +SDL_Convert_S16_to_F32_Scalar(SDL_AudioCVT *cvt, SDL_AudioFormat format) { const Sint16 *src = ((const Sint16 *) (cvt->buf + cvt->len_cvt)) - 1; float *dst = ((float *) (cvt->buf + cvt->len_cvt * 2)) - 1; @@ -85,8 +126,8 @@ SDL_Convert_S16_to_F32(SDL_AudioCVT *cvt, SDL_AudioFormat format) } } -void SDLCALL -SDL_Convert_U16_to_F32(SDL_AudioCVT *cvt, SDL_AudioFormat format) +static void SDLCALL +SDL_Convert_U16_to_F32_Scalar(SDL_AudioCVT *cvt, SDL_AudioFormat format) { const Uint16 *src = ((const Uint16 *) (cvt->buf + cvt->len_cvt)) - 1; float *dst = ((float *) (cvt->buf + cvt->len_cvt * 2)) - 1; @@ -104,8 +145,8 @@ SDL_Convert_U16_to_F32(SDL_AudioCVT *cvt, SDL_AudioFormat format) } } -void SDLCALL -SDL_Convert_S32_to_F32(SDL_AudioCVT *cvt, SDL_AudioFormat format) +static void SDLCALL +SDL_Convert_S32_to_F32_Scalar(SDL_AudioCVT *cvt, SDL_AudioFormat format) { const Sint32 *src = (const Sint32 *) cvt->buf; float *dst = (float *) cvt->buf; @@ -122,8 +163,8 @@ SDL_Convert_S32_to_F32(SDL_AudioCVT *cvt, SDL_AudioFormat format) } } -void SDLCALL -SDL_Convert_F32_to_S8(SDL_AudioCVT *cvt, SDL_AudioFormat format) +static void SDLCALL +SDL_Convert_F32_to_S8_Scalar(SDL_AudioCVT *cvt, SDL_AudioFormat format) { const float *src = (const float *) cvt->buf; Sint8 *dst = (Sint8 *) cvt->buf; @@ -141,8 +182,8 @@ SDL_Convert_F32_to_S8(SDL_AudioCVT *cvt, SDL_AudioFormat format) } } -void SDLCALL -SDL_Convert_F32_to_U8(SDL_AudioCVT *cvt, SDL_AudioFormat format) +static void SDLCALL +SDL_Convert_F32_to_U8_Scalar(SDL_AudioCVT *cvt, SDL_AudioFormat format) { const float *src = (const float *) cvt->buf; Uint8 *dst = (Uint8 *) cvt->buf; @@ -160,8 +201,8 @@ SDL_Convert_F32_to_U8(SDL_AudioCVT *cvt, SDL_AudioFormat format) } } -void SDLCALL -SDL_Convert_F32_to_S16(SDL_AudioCVT *cvt, SDL_AudioFormat format) +static void SDLCALL +SDL_Convert_F32_to_S16_Scalar(SDL_AudioCVT *cvt, SDL_AudioFormat format) { const float *src = (const float *) cvt->buf; Sint16 *dst = (Sint16 *) cvt->buf; @@ -179,8 +220,8 @@ SDL_Convert_F32_to_S16(SDL_AudioCVT *cvt, SDL_AudioFormat format) } } -void SDLCALL -SDL_Convert_F32_to_U16(SDL_AudioCVT *cvt, SDL_AudioFormat format) +static void SDLCALL +SDL_Convert_F32_to_U16_Scalar(SDL_AudioCVT *cvt, SDL_AudioFormat format) { const float *src = (const float *) cvt->buf; Uint16 *dst = (Uint16 *) cvt->buf; @@ -198,8 +239,8 @@ SDL_Convert_F32_to_U16(SDL_AudioCVT *cvt, SDL_AudioFormat format) } } -void SDLCALL -SDL_Convert_F32_to_S32(SDL_AudioCVT *cvt, SDL_AudioFormat format) +static void SDLCALL +SDL_Convert_F32_to_S32_Scalar(SDL_AudioCVT *cvt, SDL_AudioFormat format) { const float *src = (const float *) cvt->buf; Sint32 *dst = (Sint32 *) cvt->buf; @@ -215,5 +256,532 @@ SDL_Convert_F32_to_S32(SDL_AudioCVT *cvt, SDL_AudioFormat format) cvt->filters[cvt->filter_index](cvt, AUDIO_S32SYS); } } +#endif + + +#if HAVE_SSE2_INTRINSICS +static void SDLCALL +SDL_Convert_S8_to_F32_SSE2(SDL_AudioCVT *cvt, SDL_AudioFormat format) +{ + const Sint8 *src = ((const Sint8 *) (cvt->buf + cvt->len_cvt)) - 1; + float *dst = ((float *) (cvt->buf + cvt->len_cvt * 4)) - 1; + int i; + + LOG_DEBUG_CONVERT("AUDIO_S8", "AUDIO_F32 (using SSE2)"); + + /* Get dst aligned to 16 bytes (since buffer is growing, we don't have to worry about overreading from src) */ + for (i = cvt->len_cvt; i && (((size_t) (dst-15)) & 15); --i, --src, --dst) { + *dst = (((float) *src) * DIVBY127); + } + + src -= 15; dst -= 15; /* adjust to read SSE blocks from the start. */ + SDL_assert(!i || ((((size_t) dst) & 15) == 0)); + + /* Make sure src is aligned too. */ + if ((((size_t) src) & 15) == 0) { + /* Aligned! Do SSE blocks as long as we have 16 bytes available. */ + const __m128i *mmsrc = (const __m128i *) src; + const __m128i zero = _mm_setzero_si128(); + const __m128 divby127 = _mm_set1_ps(DIVBY127); + while (i >= 16) { /* 16 * 8-bit */ + const __m128i bytes = _mm_load_si128(mmsrc); /* get 16 sint8 into an XMM register. */ + /* treat as int16, shift left to clear every other sint16, then back right with sign-extend. Now sint16. */ + const __m128i shorts1 = _mm_srai_epi16(_mm_slli_epi16(bytes, 8), 8); + /* right-shift-sign-extend gets us sint16 with the other set of values. */ + const __m128i shorts2 = _mm_srai_epi16(bytes, 8); + /* unpack against zero to make these int32, shift to make them sign-extend, convert to float, multiply. Whew! */ + const __m128 floats1 = _mm_mul_ps(_mm_cvtepi32_ps(_mm_srai_epi32(_mm_slli_epi32(_mm_unpacklo_epi16(shorts1, zero), 16), 16)), divby127); + const __m128 floats2 = _mm_mul_ps(_mm_cvtepi32_ps(_mm_srai_epi32(_mm_slli_epi32(_mm_unpacklo_epi16(shorts2, zero), 16), 16)), divby127); + const __m128 floats3 = _mm_mul_ps(_mm_cvtepi32_ps(_mm_srai_epi32(_mm_slli_epi32(_mm_unpackhi_epi16(shorts1, zero), 16), 16)), divby127); + const __m128 floats4 = _mm_mul_ps(_mm_cvtepi32_ps(_mm_srai_epi32(_mm_slli_epi32(_mm_unpackhi_epi16(shorts2, zero), 16), 16)), divby127); + /* Interleave back into correct order, store. */ + _mm_store_ps(dst, _mm_unpacklo_ps(floats1, floats2)); + _mm_store_ps(dst+4, _mm_unpackhi_ps(floats1, floats2)); + _mm_store_ps(dst+8, _mm_unpacklo_ps(floats3, floats4)); + _mm_store_ps(dst+12, _mm_unpackhi_ps(floats3, floats4)); + i -= 16; mmsrc--; dst -= 16; + } + + src = (const Sint8 *) mmsrc; + } + + src += 15; dst += 15; /* adjust for any scalar finishing. */ + + /* Finish off any leftovers with scalar operations. */ + while (i) { + *dst = (((float) *src) * DIVBY127); + i--; src--; dst--; + } + + cvt->len_cvt *= 4; + if (cvt->filters[++cvt->filter_index]) { + cvt->filters[cvt->filter_index](cvt, AUDIO_F32SYS); + } +} + +static void SDLCALL +SDL_Convert_U8_to_F32_SSE2(SDL_AudioCVT *cvt, SDL_AudioFormat format) +{ + const Uint8 *src = ((const Uint8 *) (cvt->buf + cvt->len_cvt)) - 1; + float *dst = ((float *) (cvt->buf + cvt->len_cvt * 4)) - 1; + int i; + + LOG_DEBUG_CONVERT("AUDIO_U8", "AUDIO_F32 (using SSE2)"); + + /* Get dst aligned to 16 bytes (since buffer is growing, we don't have to worry about overreading from src) */ + for (i = cvt->len_cvt; i && (((size_t) (dst-15)) & 15); --i, --src, --dst) { + *dst = ((((float) *src) * DIVBY127) - 1.0f); + } + + src -= 15; dst -= 15; /* adjust to read SSE blocks from the start. */ + SDL_assert(!i || ((((size_t) dst) & 15) == 0)); + + /* Make sure src is aligned too. */ + if ((((size_t) src) & 15) == 0) { + /* Aligned! Do SSE blocks as long as we have 16 bytes available. */ + const __m128i *mmsrc = (const __m128i *) src; + const __m128i zero = _mm_setzero_si128(); + const __m128 divby127 = _mm_set1_ps(DIVBY127); + const __m128 minus1 = _mm_set1_ps(-1.0f); + while (i >= 16) { /* 16 * 8-bit */ + const __m128i bytes = _mm_load_si128(mmsrc); /* get 16 uint8 into an XMM register. */ + /* treat as int16, shift left to clear every other sint16, then back right with zero-extend. Now uint16. */ + const __m128i shorts1 = _mm_srli_epi16(_mm_slli_epi16(bytes, 8), 8); + /* right-shift-zero-extend gets us uint16 with the other set of values. */ + const __m128i shorts2 = _mm_srli_epi16(bytes, 8); + /* unpack against zero to make these int32, convert to float, multiply, add. Whew! */ + /* Note that AVX2 can do floating point multiply+add in one instruction, fwiw. SSE2 cannot. */ + const __m128 floats1 = _mm_add_ps(_mm_mul_ps(_mm_cvtepi32_ps(_mm_unpacklo_epi16(shorts1, zero)), divby127), minus1); + const __m128 floats2 = _mm_add_ps(_mm_mul_ps(_mm_cvtepi32_ps(_mm_unpacklo_epi16(shorts2, zero)), divby127), minus1); + const __m128 floats3 = _mm_add_ps(_mm_mul_ps(_mm_cvtepi32_ps(_mm_unpackhi_epi16(shorts1, zero)), divby127), minus1); + const __m128 floats4 = _mm_add_ps(_mm_mul_ps(_mm_cvtepi32_ps(_mm_unpackhi_epi16(shorts2, zero)), divby127), minus1); + /* Interleave back into correct order, store. */ + _mm_store_ps(dst, _mm_unpacklo_ps(floats1, floats2)); + _mm_store_ps(dst+4, _mm_unpackhi_ps(floats1, floats2)); + _mm_store_ps(dst+8, _mm_unpacklo_ps(floats3, floats4)); + _mm_store_ps(dst+12, _mm_unpackhi_ps(floats3, floats4)); + i -= 16; mmsrc--; dst -= 16; + } + + src = (const Uint8 *) mmsrc; + } + + src += 15; dst += 15; /* adjust for any scalar finishing. */ + + /* Finish off any leftovers with scalar operations. */ + while (i) { + *dst = ((((float) *src) * DIVBY127) - 1.0f); + i--; src--; dst--; + } + + cvt->len_cvt *= 4; + if (cvt->filters[++cvt->filter_index]) { + cvt->filters[cvt->filter_index](cvt, AUDIO_F32SYS); + } +} + +static void SDLCALL +SDL_Convert_S16_to_F32_SSE2(SDL_AudioCVT *cvt, SDL_AudioFormat format) +{ + const Sint16 *src = ((const Sint16 *) (cvt->buf + cvt->len_cvt)) - 1; + float *dst = ((float *) (cvt->buf + cvt->len_cvt * 2)) - 1; + int i; + + LOG_DEBUG_CONVERT("AUDIO_S16", "AUDIO_F32 (using SSE2)"); + + /* Get dst aligned to 16 bytes (since buffer is growing, we don't have to worry about overreading from src) */ + for (i = cvt->len_cvt / sizeof (Sint16); i && (((size_t) (dst-7)) & 15); --i, --src, --dst) { + *dst = (((float) *src) * DIVBY32767); + } + + src -= 7; dst -= 7; /* adjust to read SSE blocks from the start. */ + SDL_assert(!i || ((((size_t) dst) & 15) == 0)); + + /* Make sure src is aligned too. */ + if ((((size_t) src) & 15) == 0) { + /* Aligned! Do SSE blocks as long as we have 16 bytes available. */ + const __m128 divby32767 = _mm_set1_ps(DIVBY32767); + while (i >= 8) { /* 8 * 16-bit */ + const __m128i ints = _mm_load_si128((__m128i const *) src); /* get 8 sint16 into an XMM register. */ + /* treat as int32, shift left to clear every other sint16, then back right with sign-extend. Now sint32. */ + const __m128i a = _mm_srai_epi32(_mm_slli_epi32(ints, 16), 16); + /* right-shift-sign-extend gets us sint32 with the other set of values. */ + const __m128i b = _mm_srai_epi32(ints, 16); + /* Interleave these back into the right order, convert to float, multiply, store. */ + _mm_store_ps(dst, _mm_mul_ps(_mm_cvtepi32_ps(_mm_unpacklo_epi32(a, b)), divby32767)); + _mm_store_ps(dst+4, _mm_mul_ps(_mm_cvtepi32_ps(_mm_unpackhi_epi32(a, b)), divby32767)); + i -= 8; src -= 8; dst -= 8; + } + } + + src += 7; dst += 7; /* adjust for any scalar finishing. */ + + /* Finish off any leftovers with scalar operations. */ + while (i) { + *dst = (((float) *src) * DIVBY32767); + i--; src--; dst--; + } + + cvt->len_cvt *= 2; + if (cvt->filters[++cvt->filter_index]) { + cvt->filters[cvt->filter_index](cvt, AUDIO_F32SYS); + } +} + +static void SDLCALL +SDL_Convert_U16_to_F32_SSE2(SDL_AudioCVT *cvt, SDL_AudioFormat format) +{ + const Uint16 *src = ((const Uint16 *) (cvt->buf + cvt->len_cvt)) - 1; + float *dst = ((float *) (cvt->buf + cvt->len_cvt * 2)) - 1; + int i; + + LOG_DEBUG_CONVERT("AUDIO_U16", "AUDIO_F32 (using SSE2)"); + + /* Get dst aligned to 16 bytes (since buffer is growing, we don't have to worry about overreading from src) */ + for (i = cvt->len_cvt / sizeof (Sint16); i && (((size_t) (dst-7)) & 15); --i, --src, --dst) { + *dst = ((((float) *src) * DIVBY32767) - 1.0f); + } + + src -= 7; dst -= 7; /* adjust to read SSE blocks from the start. */ + SDL_assert(!i || ((((size_t) dst) & 15) == 0)); + + /* Make sure src is aligned too. */ + if ((((size_t) src) & 15) == 0) { + /* Aligned! Do SSE blocks as long as we have 16 bytes available. */ + const __m128 divby32767 = _mm_set1_ps(DIVBY32767); + const __m128 minus1 = _mm_set1_ps(1.0f); + while (i >= 8) { /* 8 * 16-bit */ + const __m128i ints = _mm_load_si128((__m128i const *) src); /* get 8 sint16 into an XMM register. */ + /* treat as int32, shift left to clear every other sint16, then back right with zero-extend. Now sint32. */ + const __m128i a = _mm_srli_epi32(_mm_slli_epi32(ints, 16), 16); + /* right-shift-sign-extend gets us sint32 with the other set of values. */ + const __m128i b = _mm_srli_epi32(ints, 16); + /* Interleave these back into the right order, convert to float, multiply, store. */ + _mm_store_ps(dst, _mm_add_ps(_mm_mul_ps(_mm_cvtepi32_ps(_mm_unpacklo_epi32(a, b)), divby32767), minus1)); + _mm_store_ps(dst+4, _mm_add_ps(_mm_mul_ps(_mm_cvtepi32_ps(_mm_unpackhi_epi32(a, b)), divby32767), minus1)); + i -= 8; src -= 8; dst -= 8; + } + } + + src += 7; dst += 7; /* adjust for any scalar finishing. */ + + /* Finish off any leftovers with scalar operations. */ + while (i) { + *dst = ((((float) *src) * DIVBY32767) - 1.0f); + i--; src--; dst--; + } + + cvt->len_cvt *= 2; + if (cvt->filters[++cvt->filter_index]) { + cvt->filters[cvt->filter_index](cvt, AUDIO_F32SYS); + } +} + +static void SDLCALL +SDL_Convert_S32_to_F32_SSE2(SDL_AudioCVT *cvt, SDL_AudioFormat format) +{ + const Sint32 *src = (const Sint32 *) cvt->buf; + float *dst = (float *) cvt->buf; + int i; + + LOG_DEBUG_CONVERT("AUDIO_S32", "AUDIO_F32 (using SSE2)"); + + /* Get dst aligned to 16 bytes */ + for (i = cvt->len_cvt / sizeof (Sint32); i && (((size_t) dst) & 15); --i, ++src, ++dst) { + *dst = (float) (((double) *src) * DIVBY2147483647); + } + + SDL_assert(!i || ((((size_t) dst) & 15) == 0)); + SDL_assert(!i || ((((size_t) src) & 15) == 0)); + + { + /* Aligned! Do SSE blocks as long as we have 16 bytes available. */ + const __m128d divby2147483647 = _mm_set1_pd(DIVBY2147483647); + const __m128i *mmsrc = (const __m128i *) src; + while (i >= 4) { /* 4 * sint32 */ + const __m128i ints = _mm_load_si128(mmsrc); + /* bitshift the whole register over, so _mm_cvtepi32_pd can read the top ints in the bottom of the vector. */ + const __m128d doubles1 = _mm_mul_pd(_mm_cvtepi32_pd(_mm_bsrli_si128(ints, 8)), divby2147483647); + const __m128d doubles2 = _mm_mul_pd(_mm_cvtepi32_pd(ints), divby2147483647); + /* convert to float32, bitshift/or to get these into a vector to store. */ + _mm_store_ps(dst, _mm_castsi128_ps(_mm_or_si128(_mm_bslli_si128(_mm_castps_si128(_mm_cvtpd_ps(doubles1)), 8), _mm_castps_si128(_mm_cvtpd_ps(doubles2))))); + i -= 4; mmsrc++; dst += 4; + } + src = (const Sint32 *) mmsrc; + } + + /* Finish off any leftovers with scalar operations. */ + while (i) { + *dst = (float) (((double) *src) * DIVBY2147483647); + i--; src++; dst++; + } + + if (cvt->filters[++cvt->filter_index]) { + cvt->filters[cvt->filter_index](cvt, AUDIO_F32SYS); + } +} + +static void SDLCALL +SDL_Convert_F32_to_S8_SSE2(SDL_AudioCVT *cvt, SDL_AudioFormat format) +{ + const float *src = (const float *) cvt->buf; + Sint8 *dst = (Sint8 *) cvt->buf; + int i; + + LOG_DEBUG_CONVERT("AUDIO_F32", "AUDIO_S8 (using SSE2)"); + + /* Get dst aligned to 16 bytes */ + for (i = cvt->len_cvt / sizeof (float); i && (((size_t) dst) & 15); --i, ++src, ++dst) { + *dst = (Sint8) (*src * 127.0f); + } + + SDL_assert(!i || ((((size_t) dst) & 15) == 0)); + + /* Make sure src is aligned too. */ + if ((((size_t) src) & 15) == 0) { + /* Aligned! Do SSE blocks as long as we have 16 bytes available. */ + const __m128 mulby127 = _mm_set1_ps(127.0f); + __m128i *mmdst = (__m128i *) dst; + while (i >= 16) { /* 16 * float32 */ + const __m128i ints1 = _mm_cvtps_epi32(_mm_mul_ps(_mm_load_ps(src), mulby127)); /* load 4 floats, convert to sint32 */ + const __m128i ints2 = _mm_cvtps_epi32(_mm_mul_ps(_mm_load_ps(src+4), mulby127)); /* load 4 floats, convert to sint32 */ + const __m128i ints3 = _mm_cvtps_epi32(_mm_mul_ps(_mm_load_ps(src+8), mulby127)); /* load 4 floats, convert to sint32 */ + const __m128i ints4 = _mm_cvtps_epi32(_mm_mul_ps(_mm_load_ps(src+12), mulby127)); /* load 4 floats, convert to sint32 */ + _mm_store_si128(mmdst, _mm_packs_epi16(_mm_packs_epi32(ints1, ints2), _mm_packs_epi32(ints3, ints4))); /* pack down, store out. */ + i -= 16; src += 16; mmdst++; + } + dst = (Sint8 *) mmdst; + } + + /* Finish off any leftovers with scalar operations. */ + while (i) { + *dst = (Sint8) (*src * 127.0f); + i--; src++; dst++; + } + + cvt->len_cvt /= 4; + if (cvt->filters[++cvt->filter_index]) { + cvt->filters[cvt->filter_index](cvt, AUDIO_S8); + } +} + +static void SDLCALL +SDL_Convert_F32_to_U8_SSE2(SDL_AudioCVT *cvt, SDL_AudioFormat format) +{ + const float *src = (const float *) cvt->buf; + Uint8 *dst = (Uint8 *) cvt->buf; + int i; + + LOG_DEBUG_CONVERT("AUDIO_F32", "AUDIO_U8 (using SSE2)"); + + /* Get dst aligned to 16 bytes */ + for (i = cvt->len_cvt / sizeof (float); i && (((size_t) dst) & 15); --i, ++src, ++dst) { + *dst = (Uint8) ((*src + 1.0f) * 127.0f); + } + + SDL_assert(!i || ((((size_t) dst) & 15) == 0)); + + /* Make sure src is aligned too. */ + if ((((size_t) src) & 15) == 0) { + /* Aligned! Do SSE blocks as long as we have 16 bytes available. */ + const __m128 add1 = _mm_set1_ps(1.0f); + const __m128 mulby127 = _mm_set1_ps(127.0f); + __m128i *mmdst = (__m128i *) dst; + while (i >= 16) { /* 16 * float32 */ + const __m128i ints1 = _mm_cvtps_epi32(_mm_mul_ps(_mm_add_ps(_mm_load_ps(src), add1), mulby127)); /* load 4 floats, convert to sint32 */ + const __m128i ints2 = _mm_cvtps_epi32(_mm_mul_ps(_mm_add_ps(_mm_load_ps(src+4), add1), mulby127)); /* load 4 floats, convert to sint32 */ + const __m128i ints3 = _mm_cvtps_epi32(_mm_mul_ps(_mm_add_ps(_mm_load_ps(src+8), add1), mulby127)); /* load 4 floats, convert to sint32 */ + const __m128i ints4 = _mm_cvtps_epi32(_mm_mul_ps(_mm_add_ps(_mm_load_ps(src+12), add1), mulby127)); /* load 4 floats, convert to sint32 */ + _mm_store_si128(mmdst, _mm_packus_epi16(_mm_packs_epi32(ints1, ints2), _mm_packs_epi32(ints3, ints4))); /* pack down, store out. */ + i -= 16; src += 16; mmdst++; + } + dst = (Uint8 *) mmdst; + } + + /* Finish off any leftovers with scalar operations. */ + while (i) { + *dst = (Uint8) ((*src + 1.0f) * 127.0f); + i--; src++; dst++; + } + + cvt->len_cvt /= 4; + if (cvt->filters[++cvt->filter_index]) { + cvt->filters[cvt->filter_index](cvt, AUDIO_U8); + } +} + +static void SDLCALL +SDL_Convert_F32_to_S16_SSE2(SDL_AudioCVT *cvt, SDL_AudioFormat format) +{ + const float *src = (const float *) cvt->buf; + Sint16 *dst = (Sint16 *) cvt->buf; + int i; + + LOG_DEBUG_CONVERT("AUDIO_F32", "AUDIO_S16 (using SSE2)"); + + /* Get dst aligned to 16 bytes */ + for (i = cvt->len_cvt / sizeof (float); i && (((size_t) dst) & 15); --i, ++src, ++dst) { + *dst = (Sint16) (*src * 32767.0f); + } + + SDL_assert(!i || ((((size_t) dst) & 15) == 0)); + + /* Make sure src is aligned too. */ + if ((((size_t) src) & 15) == 0) { + /* Aligned! Do SSE blocks as long as we have 16 bytes available. */ + const __m128 mulby32767 = _mm_set1_ps(32767.0f); + __m128i *mmdst = (__m128i *) dst; + while (i >= 8) { /* 8 * float32 */ + const __m128i ints1 = _mm_cvtps_epi32(_mm_mul_ps(_mm_load_ps(src), mulby32767)); /* load 4 floats, convert to sint32 */ + const __m128i ints2 = _mm_cvtps_epi32(_mm_mul_ps(_mm_load_ps(src+4), mulby32767)); /* load 4 floats, convert to sint32 */ + _mm_store_si128(mmdst, _mm_packs_epi32(ints1, ints2)); /* pack to sint16, store out. */ + i -= 8; src += 8; mmdst++; + } + dst = (Sint16 *) mmdst; + } + + /* Finish off any leftovers with scalar operations. */ + while (i) { + *dst = (((float) *src) * DIVBY32767); + i--; src++; dst++; + } + + cvt->len_cvt /= 2; + if (cvt->filters[++cvt->filter_index]) { + cvt->filters[cvt->filter_index](cvt, AUDIO_S16SYS); + } +} + +static void SDLCALL +SDL_Convert_F32_to_U16_SSE2(SDL_AudioCVT *cvt, SDL_AudioFormat format) +{ + const float *src = (const float *) cvt->buf; + Uint16 *dst = (Uint16 *) cvt->buf; + int i; + + LOG_DEBUG_CONVERT("AUDIO_F32", "AUDIO_U16 (using SSE2)"); + + /* Get dst aligned to 16 bytes */ + for (i = cvt->len_cvt / sizeof (float); i && (((size_t) dst) & 15); --i, ++src, ++dst) { + *dst = (Uint16) ((*src + 1.0f) * 32767.0f); + } + + SDL_assert(!i || ((((size_t) dst) & 15) == 0)); + + /* Make sure src is aligned too. */ + if ((((size_t) src) & 15) == 0) { + /* Aligned! Do SSE blocks as long as we have 16 bytes available. */ + /* This calculates differently than the scalar path because SSE2 can't + pack int32 data down to unsigned int16. _mm_packs_epi32 does signed + saturation, so that would corrupt our data. _mm_packus_epi32 exists, + but not before SSE 4.1. So we convert from float to sint16, packing + that down with legit signed saturation, and then xor the top bit + against 1. This results in the correct unsigned 16-bit value, even + though it looks like dark magic. */ + const __m128 mulby32767 = _mm_set1_ps(32767.0f); + const __m128i topbit = _mm_set1_epi16(-32768); + __m128i *mmdst = (__m128i *) dst; + while (i >= 8) { /* 8 * float32 */ + const __m128i ints1 = _mm_cvtps_epi32(_mm_mul_ps(_mm_load_ps(src), mulby32767)); /* load 4 floats, convert to sint32 */ + const __m128i ints2 = _mm_cvtps_epi32(_mm_mul_ps(_mm_load_ps(src+4), mulby32767)); /* load 4 floats, convert to sint32 */ + _mm_store_si128(mmdst, _mm_xor_si128(_mm_packs_epi32(ints1, ints2), topbit)); /* pack to sint16, xor top bit, store out. */ + i -= 8; src += 8; mmdst++; + } + dst = (Uint16 *) mmdst; + } + + /* Finish off any leftovers with scalar operations. */ + while (i) { + *dst = (Uint16) ((*src + 1.0f) * 32767.0f); + i--; src++; dst++; + } + + cvt->len_cvt /= 2; + if (cvt->filters[++cvt->filter_index]) { + cvt->filters[cvt->filter_index](cvt, AUDIO_U16SYS); + } +} + +static void SDLCALL +SDL_Convert_F32_to_S32_SSE2(SDL_AudioCVT *cvt, SDL_AudioFormat format) +{ + const float *src = (const float *) cvt->buf; + Sint32 *dst = (Sint32 *) cvt->buf; + int i; + + LOG_DEBUG_CONVERT("AUDIO_F32", "AUDIO_S32 (using SSE2)"); + + /* Get dst aligned to 16 bytes */ + for (i = cvt->len_cvt / sizeof (float); i && (((size_t) dst) & 15); --i, ++src, ++dst) { + *dst = (Sint32) (((double) *src) * 2147483647.0); + } + + SDL_assert(!i || ((((size_t) dst) & 15) == 0)); + SDL_assert(!i || ((((size_t) src) & 15) == 0)); + + { + /* Aligned! Do SSE blocks as long as we have 16 bytes available. */ + const __m128d mulby2147483647 = _mm_set1_pd(2147483647.0); + __m128i *mmdst = (__m128i *) dst; + while (i >= 4) { /* 4 * float32 */ + const __m128 floats = _mm_load_ps(src); + /* bitshift the whole register over, so _mm_cvtps_pd can read the top floats in the bottom of the vector. */ + const __m128d doubles1 = _mm_mul_pd(_mm_cvtps_pd(_mm_castsi128_ps(_mm_bsrli_si128(_mm_castps_si128(floats), 8))), mulby2147483647); + const __m128d doubles2 = _mm_mul_pd(_mm_cvtps_pd(floats), mulby2147483647); + _mm_store_si128(mmdst, _mm_or_si128(_mm_bslli_si128(_mm_cvtpd_epi32(doubles1), 8), _mm_cvtpd_epi32(doubles2))); + i -= 4; src += 4; mmdst++; + } + dst = (Sint32 *) mmdst; + } + + /* Finish off any leftovers with scalar operations. */ + while (i) { + *dst = (Sint32) (((double) *src) * 2147483647.0); + i--; src++; dst++; + } + + if (cvt->filters[++cvt->filter_index]) { + cvt->filters[cvt->filter_index](cvt, AUDIO_S32SYS); + } +} +#endif + + +void SDL_ChooseAudioConverters(void) +{ + static SDL_bool converters_chosen = SDL_FALSE; + + if (converters_chosen) { + return; + } + + #define SET_CONVERTER_FUNCS(fntype) \ + SDL_Convert_S8_to_F32 = SDL_Convert_S8_to_F32_##fntype; \ + SDL_Convert_U8_to_F32 = SDL_Convert_U8_to_F32_##fntype; \ + SDL_Convert_S16_to_F32 = SDL_Convert_S16_to_F32_##fntype; \ + SDL_Convert_U16_to_F32 = SDL_Convert_U16_to_F32_##fntype; \ + SDL_Convert_S32_to_F32 = SDL_Convert_S32_to_F32_##fntype; \ + SDL_Convert_F32_to_S8 = SDL_Convert_F32_to_S8_##fntype; \ + SDL_Convert_F32_to_U8 = SDL_Convert_F32_to_U8_##fntype; \ + SDL_Convert_F32_to_S16 = SDL_Convert_F32_to_S16_##fntype; \ + SDL_Convert_F32_to_U16 = SDL_Convert_F32_to_U16_##fntype; \ + SDL_Convert_F32_to_S32 = SDL_Convert_F32_to_S32_##fntype; \ + converters_chosen = SDL_TRUE + + #if HAVE_SSE2_INTRINSICS + if (SDL_HasSSE2()) { + SET_CONVERTER_FUNCS(SSE2); + return; + } + #endif + + #if NEED_SCALAR_CONVERTER_FALLBACKS + SET_CONVERTER_FUNCS(Scalar); + #endif + + #undef SET_CONVERTER_FUNCS + + SDL_assert(converters_chosen == SDL_TRUE); +} /* vi: set ts=4 sw=4 expandtab: */