Vuo  2.0.0
VuoAudio.cc
Go to the documentation of this file.
1 
10 #include "VuoAudio.h"
11 #include "VuoPool.hh"
12 #include "VuoTriggerSet.hh"
13 #include "VuoApp.h"
14 #include "VuoEventLoop.h"
15 #include "VuoOsStatus.h"
16 
17 #pragma clang diagnostic push
18 #pragma clang diagnostic ignored "-Wdocumentation"
19 #include <RtAudio/RtAudio.h>
20 #pragma clang diagnostic pop
21 
22 #include <queue>
23 #include <CoreAudio/CoreAudio.h>
24 #include <objc/objc-runtime.h>
25 
26 
27 extern "C"
28 {
29 
30 #ifdef VUO_COMPILER
32  "title" : "VuoAudio",
33  "dependencies" : [
34  "VuoAudioSamples",
35  "VuoAudioInputDevice",
36  "VuoAudioOutputDevice",
37  "VuoList_VuoAudioSamples",
38  "VuoList_VuoAudioInputDevice",
39  "VuoList_VuoAudioOutputDevice",
40  "VuoOsStatus",
41  "rtaudio",
42  "CoreAudio.framework"
43  ]
44  });
45 #endif
46 }
47 
49 
52 unsigned int VuoAudio_useCount = 0;
53 
59 static void __attribute__((constructor)) VuoAudio_init()
60 {
61  // Some audio drivers (such as Jack) assume that they're being initialized on the main thread,
62  // so our first audio-related call (which initializes the drivers) should be on the main thread.
63  // https://b33p.net/kosada/node/12798
64  dispatch_async(dispatch_get_main_queue(), ^{
65  CFRunLoopRef theRunLoop = NULL;
66  AudioObjectPropertyAddress theAddress = { kAudioHardwarePropertyRunLoop, kAudioObjectPropertyScopeGlobal, kAudioObjectPropertyElementMaster };
67  AudioObjectSetPropertyData(kAudioObjectSystemObject, &theAddress, 0, NULL, sizeof(CFRunLoopRef), &theRunLoop);
68  });
69 }
70 
75 {
76  RtAudio rta;
77  unsigned int deviceCount = rta.getDeviceCount();
79  for (unsigned int i = 0; i < deviceCount; ++i)
80  {
81  RtAudio::DeviceInfo info = rta.getDeviceInfo(i);
82  if (info.inputChannels)
83  VuoListAppendValue_VuoAudioInputDevice(inputDevices, VuoAudioInputDevice_make(i, VuoText_make(info.modelUid.c_str()), VuoText_make(info.name.c_str()), info.inputChannels));
84  }
85  return inputDevices;
86 }
87 
92 {
93  RtAudio rta;
94  unsigned int deviceCount = rta.getDeviceCount();
96  for (unsigned int i = 0; i < deviceCount; ++i)
97  {
98  RtAudio::DeviceInfo info = rta.getDeviceInfo(i);
99  if (info.outputChannels)
100  VuoListAppendValue_VuoAudioOutputDevice(outputDevices, VuoAudioOutputDevice_make(i, VuoText_make(info.modelUid.c_str()), VuoText_make(info.name.c_str()), info.outputChannels));
101  }
102  return outputDevices;
103 }
104 
108 static OSStatus VuoAudio_reconfigurationCallback(AudioObjectID inObjectID, UInt32 inNumberAddresses, const AudioObjectPropertyAddress *inAddresses, void *inClientData)
109 {
112  return noErr;
113 }
114 
121 void VuoAudio_use(void)
122 {
123  if (__sync_add_and_fetch(&VuoAudio_useCount, 1) == 1)
124  {
125  AudioObjectPropertyAddress address;
126  address.mSelector = kAudioHardwarePropertyDevices;
127  address.mScope = kAudioObjectPropertyScopeGlobal;
128  address.mElement = kAudioObjectPropertyElementMaster;
129  OSStatus ret = AudioObjectAddPropertyListener(kAudioObjectSystemObject, &address, &VuoAudio_reconfigurationCallback, NULL);
130  if (ret)
131  {
132  char *description = VuoOsStatus_getText(ret);
133  VUserLog("Error: Couldn't register device change listener: %s", description);
134  free(description);
135  }
136  }
137 }
138 
145 void VuoAudio_disuse(void)
146 {
147  if (VuoAudio_useCount <= 0)
148  {
149  VUserLog("Error: Unbalanced VuoAudio_use() / _disuse() calls.");
150  return;
151  }
152 
153  if (__sync_sub_and_fetch(&VuoAudio_useCount, 1) == 0)
154  {
155  AudioObjectPropertyAddress address;
156  address.mSelector = kAudioHardwarePropertyDevices;
157  address.mScope = kAudioObjectPropertyScopeGlobal;
158  address.mElement = kAudioObjectPropertyElementMaster;
159  OSStatus ret = AudioObjectRemovePropertyListener(kAudioObjectSystemObject, &address, &VuoAudio_reconfigurationCallback, NULL);
160  if (ret)
161  {
162  char *description = VuoOsStatus_getText(ret);
163  VUserLog("Error: Couldn't unregister device change listener: %s", description);
164  free(description);
165  }
166  }
167 }
168 
178 {
179  VuoAudio_inputDeviceCallbacks.addTrigger(inputDevices);
180  VuoAudio_outputDeviceCallbacks.addTrigger(outputDevices);
181  inputDevices(VuoAudio_getInputDevices());
182  outputDevices(VuoAudio_getOutputDevices());
183 }
184 
192 {
193  VuoAudio_inputDeviceCallbacks.removeTrigger(inputDevices);
194  VuoAudio_outputDeviceCallbacks.removeTrigger(outputDevices);
195 }
196 
200 typedef std::map<void *, std::queue<VuoList_VuoAudioSamples> > pendingOutputType;
201 
205 typedef std::map<void *, std::map<VuoInteger, VuoReal> > lastOutputSampleType;
206 
210 typedef struct _VuoAudio_internal
211 {
212  RtAudio *rta;
215 
218 
219  dispatch_queue_t pendingOutputQueue;
221 
223 
226 
230 int VuoAudio_receivedEvent(void *outputBuffer, void *inputBuffer, unsigned int nBufferFrames, double streamTime, RtAudioStreamStatus status, void *userData)
231 {
232  VuoAudio_internal ai = (VuoAudio_internal)userData;
233 
234 // if (streamTime - ai->priorStreamTime > ((float)VuoAudioSamples_bufferSize/VuoAudioSamples_sampleRate)*1.02)
235 // VLog("called too late: %g (should have been < %g)", streamTime-ai->priorStreamTime, ((float)VuoAudioSamples_bufferSize/VuoAudioSamples_sampleRate)*1.02);
236 // ai->priorStreamTime=streamTime;
237 
238  if (status)
239  VUserLog("Stream %s (%d) on %s.",
240  status == RTAUDIO_INPUT_OVERFLOW ? "overflow"
241  : (status == RTAUDIO_OUTPUT_UNDERFLOW ? "underflow" : "error"),
242  status,
243  ai->inputDevice.name);
244 
245  // Fire triggers requesting audio output buffers.
246  ai->outputTriggers.fire(streamTime);
247 
248  // Fire triggers providing audio input buffers.
249  if (ai->inputTriggers.size())
250  {
252  VuoRetain(channels);
253 
254  unsigned int samplesPerSecond = ai->rta->getStreamSampleRate();
255 
256  // When creating the stream, we requested that inputBuffer be non-interleaved, so the samples should appear consecutively.
257  for (VuoInteger i = 0; i < ai->inputDevice.channelCount; ++i)
258  {
259  VuoAudioSamples samples = VuoAudioSamples_alloc(nBufferFrames);
260  samples.samplesPerSecond = samplesPerSecond;
261  memcpy(samples.samples, (VuoReal *)inputBuffer + i*nBufferFrames, sizeof(VuoReal)*nBufferFrames);
262  VuoListAppendValue_VuoAudioSamples(channels, samples);
263  }
264 
265  ai->inputTriggers.fire(channels);
266  VuoRelease(channels);
267  }
268 
269  // Zero the final output buffer.
270  double *outputBufferDouble = (double *)outputBuffer;
271  unsigned int outputChannelCount = ai->outputDevice.channelCount;
272  if (outputBuffer)
273  memset(outputBufferDouble, 0, nBufferFrames*sizeof(VuoReal)*outputChannelCount);
274 
275  // Process the pending output buffers.
276  dispatch_sync(ai->pendingOutputQueue, ^{
277  if (!outputBuffer)
278  {
279  // If there are any pending buffers, squander them.
280  bool pendingOutput = false;
281  for (pendingOutputType::iterator it = ai->pendingOutput.begin(); it != ai->pendingOutput.end(); ++it)
282  while (!it->second.empty())
283  {
284  VuoRelease(it->second.front());
285  it->second.pop();
286  pendingOutput = true;
287  }
288 
289  if (pendingOutput)
290  VUserLog("This audio device (%s) doesn't support output.", ai->outputDevice.name);
291  return;
292  }
293 
294  // Mix the next buffer from each source into a single output buffer.
296  for (pendingOutputType::iterator it = ai->pendingOutput.begin(); it != ai->pendingOutput.end(); ++it)
297  {
298  void *id = it->first;
299  if (it->second.empty()) // No pending buffers for this source.
300  {
301  if (ai->lastOutputSample.find(id) == ai->lastOutputSample.end())
302  continue;
303 
304  for (unsigned int channel = 0; channel < outputChannelCount; ++channel)
305  {
306  if (ai->lastOutputSample[id].find(channel) == ai->lastOutputSample[id].end())
307  continue;
308 
309  // Since this channel was previously playing audio, smoothly fade the last amplitude to zero...
310  VuoReal lastOutputSample = ai->lastOutputSample[id][channel];
311  for (VuoInteger i=0; i<nBufferFrames; ++i)
312  outputBufferDouble[nBufferFrames*(channel) + i] = VuoReal_lerp(lastOutputSample, 0, (float)i/nBufferFrames);
313 
314  // ...and indicate that we already faded out.
315  ai->lastOutputSample[id].erase(ai->lastOutputSample[id].find(channel));
316  }
317  }
318  else // Have pending buffers for this source.
319  {
320  VuoList_VuoAudioSamples channels = it->second.front();
321 
322  for (unsigned int channel = 0; channel < outputChannelCount; ++channel)
323  {
324  VuoAudioSamples as = VuoListGetValue_VuoAudioSamples(channels, channel+1);
325  if (!as.samples || as.sampleCount == 0)
326  continue;
327 
328  if (ai->lastOutputSample[id].find(channel) == ai->lastOutputSample[id].end())
329  {
330  // This is the first sample buffer ever, or the first after a dropout.
331  // Make sure the queue is primed with a few buffers before we start draining it.
332  if (it->second.size() < VuoAudio_queueSize)
333  goto skipPop;
334 
335  // Smoothly fade from zero to the sample buffer.
336  for (VuoInteger i=0; i<nBufferFrames; ++i)
337  outputBufferDouble[nBufferFrames*channel + i] = (float)i/nBufferFrames * as.samples[i];
338  }
339  else
340  {
341  // We were previously playing audio, so just copy the samples intact.
342  for (VuoInteger i=0; i<nBufferFrames; ++i)
344  outputBufferDouble[nBufferFrames*(channel) + i] += as.samples[i];
345  }
346 
347  ai->lastOutputSample[id][channel] = as.samples[as.sampleCount-1];
348  }
349 
350  it->second.pop();
351  VuoRelease(channels);
352 
353  skipPop:;
354  }
355  }
356  });
357  return 0;
358 }
359 
361 VUOKEYEDPOOL(unsigned int, VuoAudio_internal);
362 static void VuoAudio_destroy(VuoAudio_internal ai);
363 VuoAudio_internal VuoAudio_make(unsigned int deviceId)
364 {
365  VuoAudio_internal ai = NULL;
366  try
367  {
368  Class avCaptureDeviceClass = objc_getClass("AVCaptureDevice");
369  if (class_getClassMethod(avCaptureDeviceClass, sel_getUid("authorizationStatusForMediaType:")))
370  {
371  CFStringRef mediaType = CFStringCreateWithCString(NULL, "soun", kCFStringEncodingUTF8);
372  long status = (long)objc_msgSend((id)avCaptureDeviceClass, sel_getUid("authorizationStatusForMediaType:"), mediaType);
373  CFRelease(mediaType);
374 
375  if (status == 0 /* AVAuthorizationStatusNotDetermined */)
376  VUserLog("Warning: Audio input may be unavailable due to system restrictions. Check System Preferences > Security & Privacy > Privacy > Microphone.");
377  else if (status == 1 /* AVAuthorizationStatusRestricted */
378  || status == 2 /* AVAuthorizationStatusDenied */)
379  VUserLog("Error: Audio input is unavailable due to system restrictions. Check System Preferences > Security & Privacy > Privacy > Microphone.");
380  }
381 
382  ai = new _VuoAudio_internal;
383  ai->inputDevice.id = deviceId;
384  ai->outputDevice.id = deviceId;
385 
386  ai->pendingOutputQueue = dispatch_queue_create("VuoAudio pending output", VuoEventLoop_getDispatchInteractiveAttribute());
387 
388  // Though neither RtAudio's documentation nor Apple's documentation
389  // specify that audio must be initialized on the main thread,
390  // some audio drivers (such as Jack) make that assumption.
391  // https://b33p.net/kosada/node/12068
393  ai->rta = new RtAudio();
394  });
395 
396  RtAudio::StreamParameters inputParameters;
397  inputParameters.deviceId = deviceId;
398  inputParameters.nChannels = ai->rta->getDeviceInfo(deviceId).inputChannels;
399  ai->inputDevice.name = VuoText_make(ai->rta->getDeviceInfo(deviceId).name.c_str());
401  ai->inputDevice.channelCount = inputParameters.nChannels;
402 
403  RtAudio::StreamParameters outputParameters;
404  outputParameters.deviceId = deviceId;
405  outputParameters.nChannels = ai->rta->getDeviceInfo(deviceId).outputChannels;
406  ai->outputDevice.name = VuoText_make(ai->rta->getDeviceInfo(deviceId).name.c_str());
408  ai->outputDevice.channelCount = outputParameters.nChannels;
409 
410  RtAudio::StreamOptions options;
411  options.flags = RTAUDIO_NONINTERLEAVED;
412 
413  unsigned int bufferFrames = VuoAudioSamples_bufferSize;
414 
415  ai->rta->openStream(
416  outputParameters.nChannels ? &outputParameters : NULL,
417  inputParameters.nChannels ? &inputParameters : NULL,
418  RTAUDIO_FLOAT64,
420  &bufferFrames,
422  ai,
423  &options
424  );
425  ai->rta->startStream();
426  }
427  catch (RtAudioError &error)
428  {
430  VUserLog("Failed to open the audio device (%d) :: %s.\n", deviceId, error.what());
431 
432  if (ai)
433  {
434  delete ai->rta;
435  delete ai;
436  ai = NULL;
437  }
438  }
439 
440  if (ai)
442 
443  return ai;
444 }
446 {
447  VuoAudio_internalPool->removeSharedInstance(ai->inputDevice.id);
448 
449  try
450  {
451  if (ai->rta->isStreamOpen())
452  {
453  ai->rta->stopStream();
454  ai->rta->closeStream();
455  }
456  }
457  catch (RtAudioError &error)
458  {
459  VUserLog("Failed to close the audio device (%s) :: %s.\n", ai->inputDevice.name, error.what());
460  }
461 
462  // Now that the audio stream is stopped (and the last callback has returned), it's safe to delete the queue.
463  dispatch_release(ai->pendingOutputQueue);
464 
465  // Release any leftover buffers.
466  for (pendingOutputType::iterator it = ai->pendingOutput.begin(); it != ai->pendingOutput.end(); ++it)
467  while (!it->second.empty())
468  {
469  VuoRelease(it->second.front());
470  it->second.pop();
471  }
472 
473  delete ai->rta;
476  delete ai;
477 }
480 
485 {
486  int deviceId = -1;
487 
488  __block RtAudio *temporaryRTA; // Just for getting device info prior to opening a shared device.
489 
490  try
491  {
492  // https://b33p.net/kosada/node/12068
494  temporaryRTA = new RtAudio();
495  });
496 
497  if (aid.id == -1)
498  {
499  if (VuoText_isEmpty(aid.modelUid) && VuoText_isEmpty(aid.name))
500  {
501  // Choose the default device
502  deviceId = temporaryRTA->getDefaultInputDevice();
503 
504  if (VuoIsDebugEnabled())
505  {
506  RtAudio::DeviceInfo di = temporaryRTA->getDeviceInfo(deviceId);
507  if (di.inputChannels)
508  VUserLog("Using default device #%d, name \"%s\".", deviceId, di.name.c_str());
509  }
510  }
511  else
512  {
513  if (!VuoText_isEmpty(aid.modelUid))
514  {
515  // Choose the first input device whose name contains aid.modelUid
516  VDebugLog("Trying to match model UID \"%s\"…", aid.modelUid);
517  unsigned int deviceCount = temporaryRTA->getDeviceCount();
518  for (unsigned int i = 0; i < deviceCount; ++i)
519  {
520  RtAudio::DeviceInfo di = temporaryRTA->getDeviceInfo(i);
521  if (di.inputChannels && di.modelUid.find(aid.modelUid) != std::string::npos)
522  {
523  VDebugLog("Matched device #%d, name \"%s\".", i, di.name.c_str());
524  deviceId = i;
525  break;
526  }
527  }
528  }
529 
530  if (deviceId == -1 && !VuoText_isEmpty(aid.name))
531  {
532  // Choose the first input device whose name contains aid.name
533  VDebugLog("Trying to match name \"%s\"…", aid.name);
534  unsigned int deviceCount = temporaryRTA->getDeviceCount();
535  for (unsigned int i = 0; i < deviceCount; ++i)
536  {
537  RtAudio::DeviceInfo di = temporaryRTA->getDeviceInfo(i);
538  if (di.inputChannels && di.name.find(aid.name) != std::string::npos)
539  {
540  VDebugLog("Matched device #%d, name \"%s\".", i, di.name.c_str());
541  deviceId = i;
542  break;
543  }
544  }
545  }
546 
547  if (deviceId == -1)
548  {
549  VUserLog("Couldn't find a matching audio device.");
550  return NULL;
551  }
552  }
553  }
554  else
555  {
556  // Choose the device specified by aid.id
557  deviceId = aid.id;
558 
559  if (VuoIsDebugEnabled())
560  {
561  RtAudio::DeviceInfo di = temporaryRTA->getDeviceInfo(deviceId);
562  if (di.inputChannels)
563  VUserLog("Using device #%d, name \"%s\".", deviceId, di.name.c_str());
564  }
565  }
566 
567  delete temporaryRTA;
568  }
569  catch (RtAudioError &error)
570  {
572  VUserLog("Failed to enumerate audio devices :: %s.\n", error.what());
573  delete temporaryRTA;
574  return NULL;
575  }
576 
577  return (VuoAudioIn)VuoAudio_internalPool->getSharedInstance(deviceId);
578 }
579 
584 {
585  int deviceId = -1;
586 
587  __block RtAudio *temporaryRTA; // Just for getting device info prior to opening a shared device.
588 
589  try
590  {
591  // https://b33p.net/kosada/node/12068
593  temporaryRTA = new RtAudio();
594  });
595 
596  if (aod.id == -1)
597  {
598  if (VuoText_isEmpty(aod.modelUid) && VuoText_isEmpty(aod.name))
599  {
600  // Choose the default device
601  deviceId = temporaryRTA->getDefaultOutputDevice();
602 
603  if (VuoIsDebugEnabled())
604  {
605  RtAudio::DeviceInfo di = temporaryRTA->getDeviceInfo(deviceId);
606  if (di.outputChannels)
607  VUserLog("Using default device #%d, name \"%s\".", deviceId, di.name.c_str());
608  }
609  }
610  else
611  {
612  if (!VuoText_isEmpty(aod.modelUid))
613  {
614  // Choose the first output device whose name contains aid.modelUid
615  VDebugLog("Trying to match model UID \"%s\"…", aod.modelUid);
616  unsigned int deviceCount = temporaryRTA->getDeviceCount();
617  for (unsigned int i = 0; i < deviceCount; ++i)
618  {
619  RtAudio::DeviceInfo di = temporaryRTA->getDeviceInfo(i);
620  if (di.outputChannels && di.modelUid.find(aod.modelUid) != std::string::npos)
621  {
622  VDebugLog("Matched device #%d, name \"%s\".", i, di.name.c_str());
623  deviceId = i;
624  break;
625  }
626  }
627  }
628 
629  if (deviceId == -1 && !VuoText_isEmpty(aod.name))
630  {
631  // Choose the first output device whose name contains aid.name
632  VDebugLog("Trying to match name \"%s\"…", aod.name);
633  unsigned int deviceCount = temporaryRTA->getDeviceCount();
634  for (unsigned int i = 0; i < deviceCount; ++i)
635  {
636  RtAudio::DeviceInfo di = temporaryRTA->getDeviceInfo(i);
637  if (di.outputChannels && di.name.find(aod.name) != std::string::npos)
638  {
639  VDebugLog("Matched device #%d, name \"%s\".", i, di.name.c_str());
640  deviceId = i;
641  break;
642  }
643  }
644  }
645 
646  if (deviceId == -1)
647  {
648  VUserLog("Couldn't find a matching audio device.");
649  return NULL;
650  }
651  }
652  }
653  else
654  {
655  // Choose the device specified by aid.id
656  deviceId = aod.id;
657 
658  if (VuoIsDebugEnabled())
659  {
660  RtAudio::DeviceInfo di = temporaryRTA->getDeviceInfo(deviceId);
661  if (di.outputChannels)
662  VUserLog("Using device #%d, name \"%s\".", deviceId, di.name.c_str());
663  }
664  }
665 
666  delete temporaryRTA;
667  }
668  catch (RtAudioError &error)
669  {
671  VUserLog("Failed to enumerate audio devices :: %s.\n", error.what());
672  delete temporaryRTA;
673  return NULL;
674  }
675 
676  return (VuoAudioOut)VuoAudio_internalPool->getSharedInstance(deviceId);
677 }
678 
685 (
686  VuoAudioIn ai,
687  VuoOutputTrigger(receivedChannels, VuoList_VuoAudioSamples)
688 )
689 {
690  if (!ai)
691  return;
692 
694  aii->inputTriggers.addTrigger(receivedChannels);
695 }
696 
703 (
704  VuoAudioOut ao,
705  VuoOutputTrigger(requestedChannels, VuoReal)
706 )
707 {
708  if (!ao)
709  return;
710 
712  aii->outputTriggers.addTrigger(requestedChannels);
713 }
714 
721 (
722  VuoAudioIn ai,
723  VuoOutputTrigger(receivedChannels, VuoList_VuoAudioSamples)
724 )
725 {
726  if (!ai)
727  return;
728 
730  aii->inputTriggers.removeTrigger(receivedChannels);
731 }
732 
739 (
740  VuoAudioOut ao,
741  VuoOutputTrigger(requestedChannels, VuoReal)
742 )
743 {
744  if (!ao)
745  return;
746 
748  aii->outputTriggers.removeTrigger(requestedChannels);
749 }
750 
762 {
763  if (!ao)
764  return;
765 
766  VuoRetain(channels);
768  dispatch_async(aii->pendingOutputQueue, ^{
769  aii->pendingOutput[id].push(channels);
770  });
771 }