Vuo  2.0.0
VuoAudio.cc
Go to the documentation of this file.
1 
10 #include "VuoAudio.h"
11 #include "VuoPool.hh"
12 #include "VuoTriggerSet.hh"
13 #include "VuoApp.h"
14 #include "VuoEventLoop.h"
15 #include "VuoOsStatus.h"
16 
17 #pragma clang diagnostic push
18 #pragma clang diagnostic ignored "-Wdocumentation"
19 #include <RtAudio/RtAudio.h>
20 #pragma clang diagnostic pop
21 
22 #include <queue>
23 #include <CoreAudio/CoreAudio.h>
24 #include <objc/objc-runtime.h>
25 
26 
27 extern "C"
28 {
29 
30 #ifdef VUO_COMPILER
32  "title" : "VuoAudio",
33  "dependencies" : [
34  "VuoAudioSamples",
35  "VuoAudioInputDevice",
36  "VuoAudioOutputDevice",
37  "VuoList_VuoAudioSamples",
38  "VuoList_VuoAudioInputDevice",
39  "VuoList_VuoAudioOutputDevice",
40  "VuoOsStatus",
41  "rtaudio",
42  "CoreAudio.framework"
43  ]
44  });
45 #endif
46 }
47 
49 
52 unsigned int VuoAudio_useCount = 0;
53 
59 static void __attribute__((constructor)) VuoAudio_init()
60 {
61  // Some audio drivers (such as Jack) assume that they're being initialized on the main thread,
62  // so our first audio-related call (which initializes the drivers) should be on the main thread.
63  // https://b33p.net/kosada/node/12798
64  dispatch_async(dispatch_get_main_queue(), ^{
65  CFRunLoopRef theRunLoop = NULL;
66  AudioObjectPropertyAddress theAddress = { kAudioHardwarePropertyRunLoop, kAudioObjectPropertyScopeGlobal, kAudioObjectPropertyElementMaster };
67  AudioObjectSetPropertyData(kAudioObjectSystemObject, &theAddress, 0, NULL, sizeof(CFRunLoopRef), &theRunLoop);
68  });
69 }
70 
75 {
76  RtAudio rta;
77  unsigned int deviceCount = rta.getDeviceCount();
79  for (unsigned int i = 0; i < deviceCount; ++i)
80  {
81  RtAudio::DeviceInfo info = rta.getDeviceInfo(i);
82  if (info.inputChannels)
83  VuoListAppendValue_VuoAudioInputDevice(inputDevices, VuoAudioInputDevice_make(i, VuoText_make(info.modelUid.c_str()), VuoText_make(info.name.c_str()), info.inputChannels));
84  }
85  return inputDevices;
86 }
87 
92 {
93  RtAudio rta;
94  unsigned int deviceCount = rta.getDeviceCount();
96  for (unsigned int i = 0; i < deviceCount; ++i)
97  {
98  RtAudio::DeviceInfo info = rta.getDeviceInfo(i);
99  if (info.outputChannels)
100  VuoListAppendValue_VuoAudioOutputDevice(outputDevices, VuoAudioOutputDevice_make(i, VuoText_make(info.modelUid.c_str()), VuoText_make(info.name.c_str()), info.outputChannels));
101  }
102  return outputDevices;
103 }
104 
108 static OSStatus VuoAudio_reconfigurationCallback(AudioObjectID inObjectID, UInt32 inNumberAddresses, const AudioObjectPropertyAddress *inAddresses, void *inClientData)
109 {
112  return noErr;
113 }
114 
121 void VuoAudio_use(void)
122 {
123  if (__sync_add_and_fetch(&VuoAudio_useCount, 1) == 1)
124  {
125  AudioObjectPropertyAddress address;
126  address.mSelector = kAudioHardwarePropertyDevices;
127  address.mScope = kAudioObjectPropertyScopeGlobal;
128  address.mElement = kAudioObjectPropertyElementMaster;
129  OSStatus ret = AudioObjectAddPropertyListener(kAudioObjectSystemObject, &address, &VuoAudio_reconfigurationCallback, NULL);
130  if (ret)
131  {
132  char *description = VuoOsStatus_getText(ret);
133  VUserLog("Error: Couldn't register device change listener: %s", description);
134  free(description);
135  }
136  }
137 }
138 
145 void VuoAudio_disuse(void)
146 {
147  if (VuoAudio_useCount <= 0)
148  {
149  VUserLog("Error: Unbalanced VuoAudio_use() / _disuse() calls.");
150  return;
151  }
152 
153  if (__sync_sub_and_fetch(&VuoAudio_useCount, 1) == 0)
154  {
155  AudioObjectPropertyAddress address;
156  address.mSelector = kAudioHardwarePropertyDevices;
157  address.mScope = kAudioObjectPropertyScopeGlobal;
158  address.mElement = kAudioObjectPropertyElementMaster;
159  OSStatus ret = AudioObjectRemovePropertyListener(kAudioObjectSystemObject, &address, &VuoAudio_reconfigurationCallback, NULL);
160  if (ret)
161  {
162  char *description = VuoOsStatus_getText(ret);
163  VUserLog("Error: Couldn't unregister device change listener: %s", description);
164  free(description);
165  }
166  }
167 }
168 
178 {
179  VuoAudio_inputDeviceCallbacks.addTrigger(inputDevices);
180  VuoAudio_outputDeviceCallbacks.addTrigger(outputDevices);
181  inputDevices(VuoAudio_getInputDevices());
182  outputDevices(VuoAudio_getOutputDevices());
183 }
184 
192 {
193  VuoAudio_inputDeviceCallbacks.removeTrigger(inputDevices);
194  VuoAudio_outputDeviceCallbacks.removeTrigger(outputDevices);
195 }
196 
200 typedef std::map<void *, std::queue<VuoList_VuoAudioSamples> > pendingOutputType;
201 
205 typedef std::map<void *, std::map<VuoInteger, VuoReal> > lastOutputSampleType;
206 
210 typedef struct _VuoAudio_internal
211 {
212  RtAudio *rta;
215 
218 
219  dispatch_queue_t pendingOutputQueue;
221 
223 
226 
230 int VuoAudio_receivedEvent(void *outputBuffer, void *inputBuffer, unsigned int nBufferFrames, double streamTime, RtAudioStreamStatus status, void *userData)
231 {
232  VuoAudio_internal ai = (VuoAudio_internal)userData;
233 
234 // if (streamTime - ai->priorStreamTime > ((float)VuoAudioSamples_bufferSize/VuoAudioSamples_sampleRate)*1.02)
235 // VLog("called too late: %g (should have been < %g)", streamTime-ai->priorStreamTime, ((float)VuoAudioSamples_bufferSize/VuoAudioSamples_sampleRate)*1.02);
236 // ai->priorStreamTime=streamTime;
237 
238  if (status)
239  VUserLog("Stream %s (%d) on %s.",
240  status == RTAUDIO_INPUT_OVERFLOW ? "overflow"
241  : (status == RTAUDIO_OUTPUT_UNDERFLOW ? "underflow" : "error"),
242  status,
243  ai->inputDevice.name);
244 
245  // Fire triggers requesting audio output buffers.
246  ai->outputTriggers.fire(streamTime);
247 
248  // Fire triggers providing audio input buffers.
249  if (ai->inputTriggers.size())
250  {
252  VuoRetain(channels);
253 
254  unsigned int samplesPerSecond = ai->rta->getStreamSampleRate();
255 
256  // When creating the stream, we requested that inputBuffer be non-interleaved, so the samples should appear consecutively.
257  for (VuoInteger i = 0; i < ai->inputDevice.channelCount; ++i)
258  {
259  VuoAudioSamples samples = VuoAudioSamples_alloc(nBufferFrames);
260  samples.samplesPerSecond = samplesPerSecond;
261  memcpy(samples.samples, (VuoReal *)inputBuffer + i*nBufferFrames, sizeof(VuoReal)*nBufferFrames);
262  VuoListAppendValue_VuoAudioSamples(channels, samples);
263  }
264 
265  ai->inputTriggers.fire(channels);
266  VuoRelease(channels);
267  }
268 
269  // Zero the final output buffer.
270  double *outputBufferDouble = (double *)outputBuffer;
271  unsigned int outputChannelCount = ai->outputDevice.channelCount;
272  if (outputBuffer)
273  memset(outputBufferDouble, 0, nBufferFrames*sizeof(VuoReal)*outputChannelCount);
274 
275  // Process the pending output buffers.
276  dispatch_sync(ai->pendingOutputQueue, ^{
277  if (!outputBuffer)
278  {
279  // If there are any pending buffers, squander them.
280  bool pendingOutput = false;
281  for (pendingOutputType::iterator it = ai->pendingOutput.begin(); it != ai->pendingOutput.end(); ++it)
282  while (!it->second.empty())
283  {
284  VuoRelease(it->second.front());
285  it->second.pop();
286  pendingOutput = true;
287  }
288 
289  if (pendingOutput)
290  VUserLog("This audio device (%s) doesn't support output.", ai->outputDevice.name);
291  return;
292  }
293 
294  // Mix the next buffer from each source into a single output buffer.
296  for (pendingOutputType::iterator it = ai->pendingOutput.begin(); it != ai->pendingOutput.end(); ++it)
297  {
298  void *id = it->first;
299  if (it->second.empty()) // No pending buffers for this source.
300  {
301  if (ai->lastOutputSample.find(id) == ai->lastOutputSample.end())
302  continue;
303 
304  for (unsigned int channel = 0; channel < outputChannelCount; ++channel)
305  {
306  if (ai->lastOutputSample[id].find(channel) == ai->lastOutputSample[id].end())
307  continue;
308 
309  // Since this channel was previously playing audio, smoothly fade the last amplitude to zero...
310  VuoReal lastOutputSample = ai->lastOutputSample[id][channel];
311  for (VuoInteger i=0; i<nBufferFrames; ++i)
312  outputBufferDouble[nBufferFrames*(channel) + i] = VuoReal_lerp(lastOutputSample, 0, (float)i/nBufferFrames);
313 
314  // ...and indicate that we already faded out.
315  ai->lastOutputSample[id].erase(ai->lastOutputSample[id].find(channel));
316  }
317  }
318  else // Have pending buffers for this source.
319  {
320  VuoList_VuoAudioSamples channels = it->second.front();
321 
322  for (unsigned int channel = 0; channel < outputChannelCount; ++channel)
323  {
324  VuoAudioSamples as = VuoListGetValue_VuoAudioSamples(channels, channel+1);
325  if (!as.samples || as.sampleCount == 0)
326  continue;
327 
328  if (ai->lastOutputSample[id].find(channel) == ai->lastOutputSample[id].end())
329  {
330  // This is the first sample buffer ever, or the first after a dropout.
331  // Make sure the queue is primed with a few buffers before we start draining it.
332  if (it->second.size() < VuoAudio_queueSize)
333  goto skipPop;
334 
335  // Smoothly fade from zero to the sample buffer.
336  for (VuoInteger i=0; i<nBufferFrames; ++i)
337  outputBufferDouble[nBufferFrames*channel + i] = (float)i/nBufferFrames * as.samples[i];
338  }
339  else
340  {
341  // We were previously playing audio, so just copy the samples intact.
342  for (VuoInteger i=0; i<nBufferFrames; ++i)
344  outputBufferDouble[nBufferFrames*(channel) + i] += as.samples[i];
345  }
346 
347  ai->lastOutputSample[id][channel] = as.samples[as.sampleCount-1];
348  }
349 
350  it->second.pop();
351  VuoRelease(channels);
352 
353  skipPop:;
354  }
355  }
356  });
357  return 0;
358 }
359 
361 VUOKEYEDPOOL(unsigned int, VuoAudio_internal);
362 static void VuoAudio_destroy(VuoAudio_internal ai);
363 VuoAudio_internal VuoAudio_make(unsigned int deviceId)
364 {
365  VuoAudio_internal ai = NULL;
366  try
367  {
368  Class avCaptureDeviceClass = objc_getClass("AVCaptureDevice");
369  if (class_getClassMethod(avCaptureDeviceClass, sel_getUid("authorizationStatusForMediaType:")))
370  {
371  CFStringRef mediaType = CFStringCreateWithCString(NULL, "soun", kCFStringEncodingUTF8);
372  long status = (long)objc_msgSend((id)avCaptureDeviceClass, sel_getUid("authorizationStatusForMediaType:"), mediaType);
373  CFRelease(mediaType);
374 
375  if (status == 0 /* AVAuthorizationStatusNotDetermined */)
376  VUserLog("Warning: Audio input may be unavailable due to system restrictions. Check System Preferences > Security & Privacy > Privacy > Microphone.");
377  else if (status == 1 /* AVAuthorizationStatusRestricted */
378  || status == 2 /* AVAuthorizationStatusDenied */)
379  VUserLog("Error: Audio input is unavailable due to system restrictions. Check System Preferences > Security & Privacy > Privacy > Microphone.");
380  }
381 
382  ai = new _VuoAudio_internal;
383  ai->inputDevice.id = deviceId;
384  ai->outputDevice.id = deviceId;
385 
386  ai->pendingOutputQueue = dispatch_queue_create("VuoAudio pending output", VuoEventLoop_getDispatchInteractiveAttribute());
387 
388  // Though neither RtAudio's documentation nor Apple's documentation
389  // specify that audio must be initialized on the main thread,
390  // some audio drivers (such as Jack) make that assumption.
391  // https://b33p.net/kosada/node/12068
393  ai->rta = new RtAudio();
394  });
395 
396  RtAudio::StreamParameters inputParameters;
397  inputParameters.deviceId = deviceId;
398  inputParameters.nChannels = ai->rta->getDeviceInfo(deviceId).inputChannels;
399  ai->inputDevice.name = VuoText_make(ai->rta->getDeviceInfo(deviceId).name.c_str());
401  ai->inputDevice.channelCount = inputParameters.nChannels;
402 
403  RtAudio::StreamParameters outputParameters;
404  outputParameters.deviceId = deviceId;
405  outputParameters.nChannels = ai->rta->getDeviceInfo(deviceId).outputChannels;
406  ai->outputDevice.name = VuoText_make(ai->rta->getDeviceInfo(deviceId).name.c_str());
408  ai->outputDevice.channelCount = outputParameters.nChannels;
409 
410  RtAudio::StreamOptions options;
411  options.flags = RTAUDIO_NONINTERLEAVED;
412 
413  unsigned int bufferFrames = VuoAudioSamples_bufferSize;
414 
415  ai->rta->openStream(
416  outputParameters.nChannels ? &outputParameters : NULL,
417  inputParameters.nChannels ? &inputParameters : NULL,
418  RTAUDIO_FLOAT64,
420  &bufferFrames,
422  ai,
423  &options
424  );
425  ai->rta->startStream();
426  }
427  catch (RtAudioError &error)
428  {
430  VUserLog("Failed to open the audio device (%d): %s", deviceId, error.what());
431 
432  if (ai)
433  {
434  delete ai->rta;
435  delete ai;
436  ai = NULL;
437  }
438  }
439 
440  if (ai)
442 
443  return ai;
444 }
446 {
447  VuoAudio_internalPool->removeSharedInstance(ai->inputDevice.id);
448 
449  try
450  {
451  if (ai->rta->isStreamOpen())
452  {
453  ai->rta->stopStream();
454  ai->rta->closeStream();
455  }
456  }
457  catch (RtAudioError &error)
458  {
459  VUserLog("Failed to close the audio device (%s): %s", ai->inputDevice.name, error.what());
460  }
461 
462  // Now that the audio stream is stopped (and the last callback has returned), it's safe to delete the queue.
463  dispatch_release(ai->pendingOutputQueue);
464 
465  // Release any leftover buffers.
466  for (pendingOutputType::iterator it = ai->pendingOutput.begin(); it != ai->pendingOutput.end(); ++it)
467  while (!it->second.empty())
468  {
469  VuoRelease(it->second.front());
470  it->second.pop();
471  }
472 
473  delete ai->rta;
476  delete ai;
477 }
480 
485 {
486  VuoAudioInputDevice realizedDevice;
487  if (!VuoAudioInputDevice_realize(aid, &realizedDevice))
488  return nullptr;
489 
490  VuoAudioInputDevice_retain(realizedDevice);
491  VuoAudioIn ai = (VuoAudioIn)VuoAudio_internalPool->getSharedInstance(realizedDevice.id);
492  VuoAudioInputDevice_release(realizedDevice);
493 
494  return ai;
495 }
496 
501 {
502  VuoAudioOutputDevice realizedDevice;
503  if (!VuoAudioOutputDevice_realize(aod, &realizedDevice))
504  return nullptr;
505 
506  VuoAudioOutputDevice_retain(realizedDevice);
507  VuoAudioOut ao = (VuoAudioOut)VuoAudio_internalPool->getSharedInstance(realizedDevice.id);
508  VuoAudioOutputDevice_release(realizedDevice);
509 
510  return ao;
511 }
512 
519 (
520  VuoAudioIn ai,
521  VuoOutputTrigger(receivedChannels, VuoList_VuoAudioSamples)
522 )
523 {
524  if (!ai)
525  return;
526 
528  aii->inputTriggers.addTrigger(receivedChannels);
529 }
530 
537 (
538  VuoAudioOut ao,
539  VuoOutputTrigger(requestedChannels, VuoReal)
540 )
541 {
542  if (!ao)
543  return;
544 
546  aii->outputTriggers.addTrigger(requestedChannels);
547 }
548 
555 (
556  VuoAudioIn ai,
557  VuoOutputTrigger(receivedChannels, VuoList_VuoAudioSamples)
558 )
559 {
560  if (!ai)
561  return;
562 
564  aii->inputTriggers.removeTrigger(receivedChannels);
565 }
566 
573 (
574  VuoAudioOut ao,
575  VuoOutputTrigger(requestedChannels, VuoReal)
576 )
577 {
578  if (!ao)
579  return;
580 
582  aii->outputTriggers.removeTrigger(requestedChannels);
583 }
584 
596 {
597  if (!ao)
598  return;
599 
600  VuoRetain(channels);
602  dispatch_async(aii->pendingOutputQueue, ^{
603  aii->pendingOutput[id].push(channels);
604  });
605 }
606 
608 #define setRealizedDevice(newDevice) \
609  realizedDevice->id = newDevice.id; \
610  realizedDevice->modelUid = VuoText_make(newDevice.modelUid); \
611  realizedDevice->name = VuoText_make(newDevice.name); \
612  realizedDevice->channelCount = newDevice.channelCount;
613 
627 {
628  // Already have channel count; nothing to do.
629  if (device.channelCount > 0)
630  {
631  setRealizedDevice(device);
632  return true;
633  }
634 
635  // Otherwise, try to find a matching device.
636 
637  VDebugLog("Requested device: %s", json_object_to_json_string(VuoAudioInputDevice_getJson(device)));
639  VuoLocal(devices);
640  __block bool found = false;
641 
642  // First pass: try to find an exact match by ID.
644  if (device.id != -1 && device.id == item.id)
645  {
646  VDebugLog("Matched by ID: %s",json_object_to_json_string(VuoAudioInputDevice_getJson(item)));
647  setRealizedDevice(item);
648  found = true;
649  return false;
650  }
651  return true;
652  });
653 
654  // Second pass: try to find a match by model AND name.
655  // (Try AND first since, for example, Soundflower creates multiple devices with different names but the same model.)
656  if (!found)
658  if (device.id == -1 && !VuoText_isEmpty(device.modelUid) && !VuoText_isEmpty(device.name)
659  && VuoText_compare(item.modelUid, (VuoTextComparison){VuoTextComparison_Contains, true, ""}, device.modelUid)
660  && VuoText_compare(item.name, (VuoTextComparison){VuoTextComparison_Contains, true, ""}, device.name))
661  {
662  VDebugLog("Matched by model and name: %s",json_object_to_json_string(VuoAudioInputDevice_getJson(item)));
663  setRealizedDevice(item);
664  found = true;
665  return false;
666  }
667  return true;
668  });
669 
670  // Third pass: try to find a loose match by model OR name.
671  if (!found)
673  if ((!VuoText_isEmpty(device.modelUid) && VuoText_compare(item.modelUid, (VuoTextComparison){VuoTextComparison_Contains, true, ""}, device.modelUid))
674  || (!VuoText_isEmpty(device.name) && VuoText_compare(item.name, (VuoTextComparison){VuoTextComparison_Contains, true, ""}, device.name)))
675  {
676  VDebugLog("Matched by model or name: %s",json_object_to_json_string(VuoAudioInputDevice_getJson(item)));
677  setRealizedDevice(item);
678  found = true;
679  return false;
680  }
681  return true;
682  });
683 
684  // Fourth pass: if the user hasn't specified a device, use the default device.
685  if (!found && device.id == -1 && VuoText_isEmpty(device.modelUid) && VuoText_isEmpty(device.name))
686  {
687  __block RtAudio *temporaryRTA; // Just for getting device info prior to opening a shared device.
688 
689  try
690  {
691  // https://b33p.net/kosada/node/12068
693  temporaryRTA = new RtAudio();
694  });
695 
696  int defaultID = temporaryRTA->getDefaultInputDevice();
698  if (item.id == defaultID)
699  {
700  VDebugLog("Using default device: %s",json_object_to_json_string(VuoAudioInputDevice_getJson(item)));
701  setRealizedDevice(item);
702  found = true;
703  return false;
704  }
705  return true;
706  });
707 
708  delete temporaryRTA;
709  }
710  catch (RtAudioError &error)
711  {
712  VUserLog("Error: Couldn't enumerate audio devices: %s", error.what());
713  delete temporaryRTA;
714  }
715  }
716 
717  if (!found)
718  VDebugLog("No matching device found.");
719 
720  return found;
721 }
722 
736 {
737  // Already have channel count; nothing to do.
738  if (device.channelCount > 0)
739  {
740  setRealizedDevice(device);
741  return true;
742  }
743 
744  // Otherwise, try to find a matching device.
745 
746  VDebugLog("Requested device: %s", json_object_to_json_string(VuoAudioOutputDevice_getJson(device)));
748  VuoLocal(devices);
749  __block bool found = false;
750 
751  // First pass: try to find an exact match by ID.
753  if (device.id != -1 && device.id == item.id)
754  {
755  VDebugLog("Matched by ID: %s",json_object_to_json_string(VuoAudioOutputDevice_getJson(item)));
756  setRealizedDevice(item);
757  found = true;
758  return false;
759  }
760  return true;
761  });
762 
763  // Second pass: try to find a match by model AND name.
764  // (Try AND first since, for example, Soundflower creates multiple devices with different names but the same model.)
765  if (!found)
767  if (device.id == -1 && !VuoText_isEmpty(device.modelUid) && !VuoText_isEmpty(device.name)
768  && VuoText_compare(item.modelUid, (VuoTextComparison){VuoTextComparison_Contains, true, ""}, device.modelUid)
769  && VuoText_compare(item.name, (VuoTextComparison){VuoTextComparison_Contains, true, ""}, device.name))
770  {
771  VDebugLog("Matched by model and name: %s",json_object_to_json_string(VuoAudioOutputDevice_getJson(item)));
772  setRealizedDevice(item);
773  found = true;
774  return false;
775  }
776  return true;
777  });
778 
779  // Third pass: try to find a loose match by model OR name.
780  if (!found)
782  if ((!VuoText_isEmpty(device.modelUid) && VuoText_compare(item.modelUid, (VuoTextComparison){VuoTextComparison_Contains, true, ""}, device.modelUid))
783  || (!VuoText_isEmpty(device.name) && VuoText_compare(item.name, (VuoTextComparison){VuoTextComparison_Contains, true, ""}, device.name)))
784  {
785  VDebugLog("Matched by model or name: %s",json_object_to_json_string(VuoAudioOutputDevice_getJson(item)));
786  setRealizedDevice(item);
787  found = true;
788  return false;
789  }
790  return true;
791  });
792 
793  // Fourth pass: if the user hasn't specified a device, use the default device.
794  if (!found && device.id == -1 && VuoText_isEmpty(device.modelUid) && VuoText_isEmpty(device.name))
795  {
796  __block RtAudio *temporaryRTA; // Just for getting device info prior to opening a shared device.
797 
798  try
799  {
800  // https://b33p.net/kosada/node/12068
802  temporaryRTA = new RtAudio();
803  });
804 
805  int defaultID = temporaryRTA->getDefaultOutputDevice();
807  if (item.id == defaultID)
808  {
809  VDebugLog("Using default device: %s",json_object_to_json_string(VuoAudioOutputDevice_getJson(item)));
810  setRealizedDevice(item);
811  found = true;
812  return false;
813  }
814  return true;
815  });
816 
817  delete temporaryRTA;
818  }
819  catch (RtAudioError &error)
820  {
821  VUserLog("Error: Couldn't enumerate audio devices: %s", error.what());
822  delete temporaryRTA;
823  }
824  }
825 
826  if (!found)
827  VDebugLog("No matching device found.");
828 
829  return found;
830 }