emscripten audio: Added audio capture support.
authorRyan C. Gordon <icculus@icculus.org>
Tue, 09 Aug 2016 16:58:32 -0400
changeset 10274cc6461b9c5bc
parent 10273 77a266c9c786
child 10275 757db914bde0
emscripten audio: Added audio capture support.
src/audio/emscripten/SDL_emscriptenaudio.c
     1.1 --- a/src/audio/emscripten/SDL_emscriptenaudio.c	Tue Aug 09 16:58:06 2016 -0400
     1.2 +++ b/src/audio/emscripten/SDL_emscriptenaudio.c	Tue Aug 09 16:58:32 2016 -0400
     1.3 @@ -61,7 +61,6 @@
     1.4      Uint8 *buf = NULL;
     1.5      int byte_len = 0;
     1.6      int bytes = SDL_AUDIO_BITSIZE(this->spec.format) / 8;
     1.7 -    int bytes_in = SDL_AUDIO_BITSIZE(this->convert.src_format) / 8;
     1.8  
     1.9      /* Only do something if audio is enabled */
    1.10      if (!SDL_AtomicGet(&this->enabled) || SDL_AtomicGet(&this->paused)) {
    1.11 @@ -69,6 +68,8 @@
    1.12      }
    1.13  
    1.14      if (this->convert.needed) {
    1.15 +        const int bytes_in = SDL_AUDIO_BITSIZE(this->convert.src_format) / 8;
    1.16 +
    1.17          if (this->hidden->conv_in_len != 0) {
    1.18              this->convert.len = this->hidden->conv_in_len * bytes_in * this->spec.channels;
    1.19          }
    1.20 @@ -134,8 +135,99 @@
    1.21  }
    1.22  
    1.23  static void
    1.24 +HandleCaptureProcess(_THIS)
    1.25 +{
    1.26 +    Uint8 *buf;
    1.27 +    int buflen;
    1.28 +
    1.29 +    /* Only do something if audio is enabled */
    1.30 +    if (!SDL_AtomicGet(&this->enabled) || SDL_AtomicGet(&this->paused)) {
    1.31 +        return;
    1.32 +    }
    1.33 +
    1.34 +    if (this->convert.needed) {
    1.35 +        buf = this->convert.buf;
    1.36 +        buflen = this->convert.len_cvt;
    1.37 +    } else {
    1.38 +        if (!this->hidden->mixbuf) {
    1.39 +            this->hidden->mixbuf = (Uint8 *) SDL_malloc(this->spec.size);
    1.40 +            if (!this->hidden->mixbuf) {
    1.41 +                return;  /* oh well. */
    1.42 +            }
    1.43 +        }
    1.44 +        buf = this->hidden->mixbuf;
    1.45 +        buflen = this->spec.size;
    1.46 +    }
    1.47 +
    1.48 +    EM_ASM_ARGS({
    1.49 +        var numChannels = SDL2.capture.currentCaptureBuffer.numberOfChannels;
    1.50 +        if (numChannels == 1) {  /* fastpath this a little for the common (mono) case. */
    1.51 +            var channelData = SDL2.capture.currentCaptureBuffer.getChannelData(0);
    1.52 +            if (channelData.length != $1) {
    1.53 +                throw 'Web Audio capture buffer length mismatch! Destination size: ' + channelData.length + ' samples vs expected ' + $1 + ' samples!';
    1.54 +            }
    1.55 +            for (var j = 0; j < $1; ++j) {
    1.56 +                setValue($0 + (j * 4), channelData[j], 'float');
    1.57 +            }
    1.58 +        } else {
    1.59 +            for (var c = 0; c < numChannels; ++c) {
    1.60 +                var channelData = SDL2.capture.currentCaptureBuffer.getChannelData(c);
    1.61 +                if (channelData.length != $1) {
    1.62 +                    throw 'Web Audio capture buffer length mismatch! Destination size: ' + channelData.length + ' samples vs expected ' + $1 + ' samples!';
    1.63 +                }
    1.64 +
    1.65 +                for (var j = 0; j < $1; ++j) {
    1.66 +                    setValue($0 + (((j * numChannels) + c) * 4), channelData[j], 'float');
    1.67 +                }
    1.68 +            }
    1.69 +        }
    1.70 +    }, buf, (this->spec.size / sizeof (float)) / this->spec.channels);
    1.71 +
    1.72 +    /* okay, we've got an interleaved float32 array in C now. */
    1.73 +
    1.74 +    if (this->convert.needed) {
    1.75 +        SDL_ConvertAudio(&this->convert);
    1.76 +    }
    1.77 +
    1.78 +    /* Send it to the app. */
    1.79 +    (*this->spec.callback) (this->spec.userdata, buf, buflen);
    1.80 +}
    1.81 +
    1.82 +
    1.83 +
    1.84 +static void
    1.85  Emscripten_CloseDevice(_THIS)
    1.86  {
    1.87 +    EM_ASM_({
    1.88 +        if ($0) {
    1.89 +            if (SDL2.capture.silenceTimer !== undefined) {
    1.90 +                clearTimeout(SDL2.capture.silenceTimer);
    1.91 +            }
    1.92 +            if (SDL2.capture.scriptProcessorNode !== undefined) {
    1.93 +                SDL2.capture.scriptProcessorNode.disconnect();
    1.94 +                SDL2.capture.scriptProcessorNode = undefined;
    1.95 +            }
    1.96 +            if (SDL2.capture.mediaStreamNode !== undefined) {
    1.97 +                SDL2.capture.mediaStreamNode.disconnect();
    1.98 +                SDL2.capture.mediaStreamNode = undefined;
    1.99 +            }
   1.100 +            if (SDL2.capture.silenceBuffer !== undefined) {
   1.101 +                SDL2.capture.silenceBuffer = undefined
   1.102 +            }
   1.103 +            SDL2.capture = undefined;
   1.104 +        } else {
   1.105 +            if (SDL2.audio.scriptProcessorNode != undefined) {
   1.106 +                SDL2.audio.scriptProcessorNode.disconnect();
   1.107 +                SDL2.audio.scriptProcessorNode = undefined;
   1.108 +            }
   1.109 +            SDL2.audio = undefined;
   1.110 +        }
   1.111 +        if ((SDL2.audioContext !== undefined) && (SDL2.audio === undefined) && (SDL2.capture === undefined)) {
   1.112 +            SDL2.audioContext.close();
   1.113 +            SDL2.audioContext = undefined;
   1.114 +        }
   1.115 +    }, this->iscapture);
   1.116 +
   1.117      SDL_free(this->hidden->mixbuf);
   1.118      SDL_free(this->hidden);
   1.119  }
   1.120 @@ -144,11 +236,38 @@
   1.121  Emscripten_OpenDevice(_THIS, void *handle, const char *devname, int iscapture)
   1.122  {
   1.123      SDL_bool valid_format = SDL_FALSE;
   1.124 -    SDL_AudioFormat test_format = SDL_FirstAudioFormat(this->spec.format);
   1.125 +    SDL_AudioFormat test_format;
   1.126      int i;
   1.127      float f;
   1.128      int result;
   1.129  
   1.130 +    /* based on parts of library_sdl.js */
   1.131 +
   1.132 +    /* create context (TODO: this puts stuff in the global namespace...)*/
   1.133 +    result = EM_ASM_INT({
   1.134 +        if(typeof(SDL2) === 'undefined') {
   1.135 +            SDL2 = {};
   1.136 +        }
   1.137 +        if (!$0) {
   1.138 +            SDL2.audio = {};
   1.139 +        } else {
   1.140 +            SDL2.capture = {};
   1.141 +        }
   1.142 +
   1.143 +        if (!SDL2.audioContext) {
   1.144 +            if (typeof(AudioContext) !== 'undefined') {
   1.145 +                SDL2.audioContext = new AudioContext();
   1.146 +            } else if (typeof(webkitAudioContext) !== 'undefined') {
   1.147 +                SDL2.audioContext = new webkitAudioContext();
   1.148 +            }
   1.149 +        }
   1.150 +        return SDL2.audioContext === undefined ? -1 : 0;
   1.151 +    }, iscapture);
   1.152 +    if (result < 0) {
   1.153 +        return SDL_SetError("Web Audio API is not available!");
   1.154 +    }
   1.155 +
   1.156 +    test_format = SDL_FirstAudioFormat(this->spec.format);
   1.157      while ((!valid_format) && (test_format)) {
   1.158          switch (test_format) {
   1.159          case AUDIO_F32: /* web audio only supports floats */
   1.160 @@ -173,34 +292,9 @@
   1.161      }
   1.162      SDL_zerop(this->hidden);
   1.163  
   1.164 -    /* based on parts of library_sdl.js */
   1.165 -
   1.166 -    /* create context (TODO: this puts stuff in the global namespace...)*/
   1.167 -    result = EM_ASM_INT_V({
   1.168 -        if(typeof(SDL2) === 'undefined')
   1.169 -            SDL2 = {};
   1.170 -
   1.171 -        if(typeof(SDL2.audio) === 'undefined')
   1.172 -            SDL2.audio = {};
   1.173 -
   1.174 -        if (!SDL2.audioContext) {
   1.175 -            if (typeof(AudioContext) !== 'undefined') {
   1.176 -                SDL2.audioContext = new AudioContext();
   1.177 -            } else if (typeof(webkitAudioContext) !== 'undefined') {
   1.178 -                SDL2.audioContext = new webkitAudioContext();
   1.179 -            } else {
   1.180 -                return -1;
   1.181 -            }
   1.182 -        }
   1.183 -        return 0;
   1.184 -    });
   1.185 -    if (result < 0) {
   1.186 -        return SDL_SetError("Web Audio API is not available!");
   1.187 -    }
   1.188 -
   1.189      /* limit to native freq */
   1.190 -    int sampleRate = EM_ASM_INT_V({
   1.191 -        return SDL2.audioContext['sampleRate'];
   1.192 +    const int sampleRate = EM_ASM_INT_V({
   1.193 +        return SDL2.audioContext.sampleRate;
   1.194      });
   1.195  
   1.196      if(this->spec.freq != sampleRate) {
   1.197 @@ -217,15 +311,71 @@
   1.198  
   1.199      SDL_CalculateAudioSpec(&this->spec);
   1.200  
   1.201 -    /* setup a ScriptProcessorNode */
   1.202 -    EM_ASM_ARGS({
   1.203 -        SDL2.audio.scriptProcessorNode = SDL2.audioContext['createScriptProcessor']($1, 0, $0);
   1.204 -        SDL2.audio.scriptProcessorNode['onaudioprocess'] = function (e) {
   1.205 -            SDL2.audio.currentOutputBuffer = e['outputBuffer'];
   1.206 -            Runtime.dynCall('vi', $2, [$3]);
   1.207 -        };
   1.208 -        SDL2.audio.scriptProcessorNode['connect'](SDL2.audioContext['destination']);
   1.209 -    }, this->spec.channels, this->spec.samples, HandleAudioProcess, this);
   1.210 +    if (iscapture) {
   1.211 +        /* The idea is to take the capture media stream, hook it up to an
   1.212 +           audio graph where we can pass it through a ScriptProcessorNode
   1.213 +           to access the raw PCM samples and push them to the SDL app's
   1.214 +           callback. From there, we "process" the audio data into silence
   1.215 +           and forget about it. */
   1.216 +
   1.217 +        /* This should, strictly speaking, use MediaRecorder for capture, but
   1.218 +           this API is cleaner to use and better supported, and fires a
   1.219 +           callback whenever there's enough data to fire down into the app.
   1.220 +           The downside is that we are spending CPU time silencing a buffer
   1.221 +           that the audiocontext uselessly mixes into any output. On the
   1.222 +           upside, both of those things are not only run in native code in
   1.223 +           the browser, they're probably SIMD code, too. MediaRecorder
   1.224 +           feels like it's a pretty inefficient tapdance in similar ways,
   1.225 +           to be honest. */
   1.226 +
   1.227 +        EM_ASM_({
   1.228 +            var have_microphone = function(stream) {
   1.229 +                clearTimeout(SDL2.capture.silenceTimer);
   1.230 +                SDL2.capture.silenceTimer = undefined;
   1.231 +                SDL2.capture.mediaStreamNode = SDL2.audioContext.createMediaStreamSource(stream);
   1.232 +                SDL2.capture.scriptProcessorNode = SDL2.audioContext.createScriptProcessor($1, $0, 1);
   1.233 +                SDL2.capture.scriptProcessorNode.onaudioprocess = function(audioProcessingEvent) {
   1.234 +                    audioProcessingEvent.outputBuffer.getChannelData(0).fill(0.0);
   1.235 +                    SDL2.capture.currentCaptureBuffer = audioProcessingEvent.inputBuffer;
   1.236 +                    Runtime.dynCall('vi', $2, [$3]);
   1.237 +                };
   1.238 +                SDL2.capture.mediaStreamNode.connect(SDL2.capture.scriptProcessorNode);
   1.239 +                SDL2.capture.scriptProcessorNode.connect(SDL2.audioContext.destination);
   1.240 +            };
   1.241 +
   1.242 +            var no_microphone = function(error) {
   1.243 +                console.log('we DO NOT have a microphone! (' + error.name + ')...leaving silence callback running.');
   1.244 +            };
   1.245 +
   1.246 +            /* we write silence to the audio callback until the microphone is available (user approves use, etc). */
   1.247 +            SDL2.capture.silenceBuffer = SDL2.audioContext.createBuffer($0, $1, SDL2.audioContext.sampleRate);
   1.248 +            SDL2.capture.silenceBuffer.getChannelData(0).fill(0.0);
   1.249 +
   1.250 +            var silence_callback = function() {
   1.251 +                SDL2.capture.currentCaptureBuffer = SDL2.capture.silenceBuffer;
   1.252 +                Runtime.dynCall('vi', $2, [$3]);
   1.253 +            };
   1.254 +
   1.255 +            SDL2.capture.silenceTimer = setTimeout(silence_callback, $1 / SDL2.audioContext.sampleRate);
   1.256 +
   1.257 +            if ((navigator.mediaDevices !== undefined) && (navigator.mediaDevices.getUserMedia !== undefined)) {
   1.258 +                navigator.mediaDevices.getUserMedia({ audio: true, video: false }).then(have_microphone).catch(no_microphone);
   1.259 +            } else if (navigator.webkitGetUserMedia !== undefined) {
   1.260 +                navigator.webkitGetUserMedia({ audio: true, video: false }, have_microphone, no_microphone);
   1.261 +            }
   1.262 +        }, this->spec.channels, this->spec.samples, HandleCaptureProcess, this);
   1.263 +    } else {
   1.264 +        /* setup a ScriptProcessorNode */
   1.265 +        EM_ASM_ARGS({
   1.266 +            SDL2.audio.scriptProcessorNode = SDL2.audioContext['createScriptProcessor']($1, 0, $0);
   1.267 +            SDL2.audio.scriptProcessorNode['onaudioprocess'] = function (e) {
   1.268 +                SDL2.audio.currentOutputBuffer = e['outputBuffer'];
   1.269 +                Runtime.dynCall('vi', $2, [$3]);
   1.270 +            };
   1.271 +            SDL2.audio.scriptProcessorNode['connect'](SDL2.audioContext['destination']);
   1.272 +        }, this->spec.channels, this->spec.samples, HandleAudioProcess, this);
   1.273 +    }
   1.274 +
   1.275      return 0;
   1.276  }
   1.277  
   1.278 @@ -236,7 +386,6 @@
   1.279      impl->OpenDevice = Emscripten_OpenDevice;
   1.280      impl->CloseDevice = Emscripten_CloseDevice;
   1.281  
   1.282 -    /* only one output */
   1.283      impl->OnlyHasDefaultOutputDevice = 1;
   1.284  
   1.285      /* no threads here */
   1.286 @@ -244,7 +393,7 @@
   1.287      impl->ProvidesOwnCallbackThread = 1;
   1.288  
   1.289      /* check availability */
   1.290 -    int available = EM_ASM_INT_V({
   1.291 +    const int available = EM_ASM_INT_V({
   1.292          if (typeof(AudioContext) !== 'undefined') {
   1.293              return 1;
   1.294          } else if (typeof(webkitAudioContext) !== 'undefined') {
   1.295 @@ -257,6 +406,18 @@
   1.296          SDL_SetError("No audio context available");
   1.297      }
   1.298  
   1.299 +    const int capture_available = available && EM_ASM_INT_V({
   1.300 +        if ((typeof(navigator.mediaDevices) !== 'undefined') && (typeof(navigator.mediaDevices.getUserMedia) !== 'undefined')) {
   1.301 +            return 1;
   1.302 +        } else if (typeof(navigator.webkitGetUserMedia) !== 'undefined') {
   1.303 +            return 1;
   1.304 +        }
   1.305 +        return 0;
   1.306 +    });
   1.307 +
   1.308 +    impl->HasCaptureSupport = capture_available ? SDL_TRUE : SDL_FALSE;
   1.309 +    impl->OnlyHasDefaultCaptureDevice = capture_available ? SDL_TRUE : SDL_FALSE;
   1.310 +
   1.311      return available;
   1.312  }
   1.313