Vuo  2.0.0
VuoAvWriterObject.m
Go to the documentation of this file.
1 
10 #include "module.h"
11 #include "VuoAvWriterObject.h"
12 #include "VuoOsStatus.h"
13 #include <CoreVideo/CoreVideo.h>
14 #import <AVFoundation/AVFoundation.h>
15 #include <OpenGL/CGLMacro.h>
16 #include <sys/stat.h>
17 
18 #ifdef VUO_COMPILER
19 VuoModuleMetadata({
20  "title" : "VuoAvWriterObject",
21  "dependencies" : [
22  "VuoFont",
23  "VuoImage",
24  "VuoImageText",
25  "VuoAudioSamples",
26  "VuoImageRenderer",
27  "VuoOsStatus",
28  "AVFoundation.framework",
29  "CoreMedia.framework",
30  "CoreVideo.framework"
31  ]
32  });
33 #endif
34 
35 const double TIMEBASE = 1000.;
36 
37 const long MIN_AUDIO_BITRATE = 64000;
38 const long MAX_AUDIO_BITRATE = 320000;
39 
43 @interface VuoAvWriterObject()
44 @property (retain) AVAssetWriter* assetWriter;
45 @property (retain) AVAssetWriterInput* videoInput;
46 @property (retain) AVAssetWriterInput* audioInput;
47 @property (retain) AVAssetWriterInputPixelBufferAdaptor* avAdaptor;
48 
49 @property CMFormatDescriptionRef audio_fmt_desc;
50 @property long audio_sample_position;
51 
52 @property (retain) NSDate* startDate;
53 @property CMTime lastImageTimestamp;
54 @property CMTime lastAudioTimestamp;
55 
56 @property int originalChannelCount;
57 @property int originalWidth;
58 @property int originalHeight;
59 @property bool firstFrame;
60 @end
61 
62 @implementation VuoAvWriterObject
63 
64 - (BOOL) isRecording
65 {
66  return self.assetWriter != nil;
67 }
68 
69 - (BOOL) setupAssetWriterWithUrl:(NSURL*) fileUrl imageWidth:(int)width imageHeight:(int)height channelCount:(int)channels movieFormat:(VuoMovieFormat)format
70 {
71  NSError *error = nil;
72 
73  self.lastImageTimestamp = kCMTimeNegativeInfinity;
74  self.lastAudioTimestamp = kCMTimeNegativeInfinity;
75 
76  self.originalWidth = width;
77  self.originalHeight = height;
78 
79  NSString* videoEncoding = AVVideoCodecJPEG;
80 
81  if (format.imageEncoding == VuoMovieImageEncoding_H264)
82  videoEncoding = AVVideoCodecH264;
83  else if(format.imageEncoding == VuoMovieImageEncoding_ProRes4444)
84  videoEncoding = AVVideoCodecAppleProRes4444;
85  else if(format.imageEncoding == VuoMovieImageEncoding_ProRes422)
86  videoEncoding = AVVideoCodecAppleProRes422;
87  else if (format.imageEncoding == VuoMovieImageEncoding_ProRes422HQ)
88  {
89  if (NSProcessInfo.processInfo.operatingSystemVersion.minorVersion >= 15)
90  videoEncoding = @"apch"; // AVVideoCodecTypeAppleProRes422HQ
91  else
92  {
93  VUserLog("Error: macOS 10.%d doesn't support ProRes 422 HQ.", (int)NSProcessInfo.processInfo.operatingSystemVersion.minorVersion);
94  return NO;
95  }
96  }
97  else if (format.imageEncoding == VuoMovieImageEncoding_ProRes422LT)
98  {
99  if (NSProcessInfo.processInfo.operatingSystemVersion.minorVersion >= 15)
100  videoEncoding = @"apcs"; // AVVideoCodecTypeAppleProRes422LT
101  else
102  {
103  VUserLog("Error: macOS 10.%d doesn't support ProRes 422 LT.", (int)NSProcessInfo.processInfo.operatingSystemVersion.minorVersion);
104  return NO;
105  }
106  }
107  else if (format.imageEncoding == VuoMovieImageEncoding_ProRes422Proxy)
108  {
109  if (NSProcessInfo.processInfo.operatingSystemVersion.minorVersion >= 15)
110  videoEncoding = @"apco"; // AVVideoCodecTypeAppleProRes422Proxy
111  else
112  {
113  VUserLog("Error: macOS 10.%d doesn't support ProRes 422 Proxy.", (int)NSProcessInfo.processInfo.operatingSystemVersion.minorVersion);
114  return NO;
115  }
116  }
117  else if (format.imageEncoding == VuoMovieImageEncoding_HEVC)
118  {
119  if (NSProcessInfo.processInfo.operatingSystemVersion.minorVersion >= 13)
120  videoEncoding = @"hvc1"; // AVVideoCodecTypeHEVC
121  else
122  {
123  VUserLog("Error: macOS 10.%d doesn't support HEVC/h.265.", (int)NSProcessInfo.processInfo.operatingSystemVersion.minorVersion);
124  return NO;
125  }
126  }
127  else if (format.imageEncoding == VuoMovieImageEncoding_HEVCAlpha)
128  {
129  if (NSProcessInfo.processInfo.operatingSystemVersion.minorVersion >= 15)
130  videoEncoding = @"muxa"; // AVVideoCodecTypeHEVCWithAlpha
131  else
132  {
133  VUserLog("Error: macOS 10.%d doesn't support HEVC/h.265 with alpha channel.", (int)NSProcessInfo.processInfo.operatingSystemVersion.minorVersion);
134  return NO;
135  }
136  }
137 
138  // allocate the writer object with our output file URL
139  self.assetWriter = [[[AVAssetWriter alloc] initWithURL:fileUrl fileType:AVFileTypeQuickTimeMovie error:&error] autorelease];
140  _assetWriter.movieFragmentInterval = CMTimeMake(TIMEBASE*10, TIMEBASE);
141 
142  if (error) {
143  VUserLog("AVAssetWriter initWithURL failed with error %s", [[error localizedDescription] UTF8String]);
144  return NO;
145  }
146 
147  // https://developer.apple.com/library/mac/documentation/AVFoundation/Reference/AVFoundation_Constants/#//apple_ref/doc/constant_group/Video_Settings
148  NSMutableDictionary *videoOutputSettings = [@{
149  AVVideoCodecKey: videoEncoding,
150  AVVideoWidthKey: [NSNumber numberWithInt:self.originalWidth],
151  AVVideoHeightKey: [NSNumber numberWithInt:self.originalHeight]
152  } mutableCopy];
153 
154  float fudge = 0.75;
155  float clampedQuality = MAX(format.imageQuality, 0.01);
156  float bitrate = clampedQuality * width * height * 60. * fudge;
157 
158  if( [videoEncoding isEqualToString:AVVideoCodecJPEG] )
159  videoOutputSettings[AVVideoCompressionPropertiesKey] = @{
160  AVVideoQualityKey: @(clampedQuality),
161  };
162  else if ([videoEncoding isEqualToString:AVVideoCodecH264]
163  || [videoEncoding isEqualToString:@"hvc1" /*AVVideoCodecTypeHEVC*/])
164  videoOutputSettings[AVVideoCompressionPropertiesKey] = @{
165  AVVideoAverageBitRateKey: @(bitrate),
166  };
167  else if ([videoEncoding isEqualToString:@"muxa" /*AVVideoCodecTypeHEVCWithAlpha*/])
168  {
169  videoOutputSettings[AVVideoCompressionPropertiesKey] = @{
170  AVVideoAverageBitRateKey: @(bitrate),
171  @"TargetQualityForAlpha" /*kVTCompressionPropertyKey_TargetQualityForAlpha*/: @(clampedQuality),
172  @"AlphaChannelMode" /*kVTCompressionPropertyKey_AlphaChannelMode*/: @"PremultipliedAlpha", // kVTAlphaChannelMode_PremultipliedAlpha
173  };
174  }
175 
176  self.videoInput = [[[AVAssetWriterInput alloc] initWithMediaType:AVMediaTypeVideo outputSettings:videoOutputSettings] autorelease];
177  [videoOutputSettings release];
178  [self.videoInput setExpectsMediaDataInRealTime:YES];
179 
180  NSDictionary *pa = @{
181  (NSString *)kCVPixelBufferPixelFormatTypeKey: [NSNumber numberWithInt:kCVPixelFormatType_32BGRA],
182  (NSString *)kCVPixelBufferWidthKey: [NSNumber numberWithInt:self.originalWidth],
183  (NSString *)kCVPixelBufferHeightKey: [NSNumber numberWithInt:self.originalHeight],
184  };
185 
186  self.avAdaptor = [AVAssetWriterInputPixelBufferAdaptor assetWriterInputPixelBufferAdaptorWithAssetWriterInput:self.videoInput sourcePixelBufferAttributes:pa];
187 
188  if ([self.assetWriter canAddInput:self.videoInput])
189  {
190  [self.assetWriter addInput:self.videoInput];
191  }
192  else
193  {
194  VUserLog("Failed adding a video input to the AVWriter.");
195  self.assetWriter = nil;
196  self.videoInput = nil;
197  self.avAdaptor = nil;
198 
199  return NO;
200  }
201 
202  self.originalChannelCount = channels;
203 
205  if(channels > 0)
206  {
207  AudioStreamBasicDescription audioFormat;
208  bzero(&audioFormat, sizeof(audioFormat));
209 
210  audioFormat.mSampleRate = VuoAudioSamples_sampleRate;
211  audioFormat.mFormatID = kAudioFormatLinearPCM;
212  audioFormat.mFramesPerPacket = 1;
213  audioFormat.mChannelsPerFrame = channels;
214  int bytes_per_sample = sizeof(float);
215  audioFormat.mFormatFlags = kAudioFormatFlagIsFloat;
216  audioFormat.mBitsPerChannel = bytes_per_sample * 8;
217  audioFormat.mBytesPerPacket = bytes_per_sample * channels;
218  audioFormat.mBytesPerFrame = bytes_per_sample * channels;
219 
220  CMFormatDescriptionRef fmt;
221  CMAudioFormatDescriptionCreate(kCFAllocatorDefault,
222  &audioFormat,
223  0,
224  NULL,
225  0,
226  NULL,
227  NULL,
228  &fmt
229  );
230  self.audio_fmt_desc = fmt;
231 
232  AudioChannelLayout acl;
233  bzero( &acl, sizeof(acl));
234  acl.mChannelLayoutTag = channels > 1 ? kAudioChannelLayoutTag_Stereo : kAudioChannelLayoutTag_Mono;
235 
236  int audioEncoding = kAudioFormatLinearPCM;
237  NSDictionary* audioOutputSettings;
238 
239  if(format.audioEncoding == VuoAudioEncoding_LinearPCM)
240  {
241  audioEncoding = kAudioFormatLinearPCM;
242  audioOutputSettings = [NSDictionary dictionaryWithObjectsAndKeys:
243  [NSNumber numberWithInt: audioEncoding ], AVFormatIDKey,
244  [NSNumber numberWithInt: channels ], AVNumberOfChannelsKey,
245  [NSNumber numberWithFloat: VuoAudioSamples_sampleRate], AVSampleRateKey,
246  [NSNumber numberWithInt: channels],AVNumberOfChannelsKey,
247  [NSNumber numberWithInt:16], AVLinearPCMBitDepthKey,
248  [NSNumber numberWithBool:NO], AVLinearPCMIsBigEndianKey,
249  [NSNumber numberWithBool:NO], AVLinearPCMIsFloatKey,
250  [NSNumber numberWithBool:NO], AVLinearPCMIsNonInterleaved,
251  [ NSData dataWithBytes: &acl length: sizeof( acl ) ], AVChannelLayoutKey,
252  nil];
253  }
254  else
255  {
256  audioEncoding = kAudioFormatMPEG4AAC;
257  long audioBitrate = (long)(MIN_AUDIO_BITRATE + ((float)(MAX_AUDIO_BITRATE-MIN_AUDIO_BITRATE) * format.audioQuality));
258 
259  audioOutputSettings = [NSDictionary dictionaryWithObjectsAndKeys:
260  [ NSNumber numberWithInt: audioEncoding ], AVFormatIDKey,
261  [ NSNumber numberWithInt: channels ], AVNumberOfChannelsKey,
262  [ NSNumber numberWithFloat: VuoAudioSamples_sampleRate], AVSampleRateKey,
263  [ NSNumber numberWithInt:audioBitrate], AVEncoderBitRateKey,
264  [ NSData dataWithBytes: &acl length: sizeof( acl ) ], AVChannelLayoutKey,
265  nil];
266  }
267 
268  self.audioInput = [AVAssetWriterInput assetWriterInputWithMediaType:AVMediaTypeAudio outputSettings:audioOutputSettings];
269  self.audioInput.expectsMediaDataInRealTime=YES;
270  self.audio_sample_position = 0;
271 
272  if([self.assetWriter canAddInput:self.audioInput])
273  {
274  [self.assetWriter addInput:self.audioInput];
275  }
276  else
277  {
278  VUserLog("Could not add audio input.");
279  self.audioInput = nil;
280  }
281  }
282 
283  // initiates a sample-writing at time 0
284  [self.assetWriter startWriting];
285  [self.assetWriter startSessionAtSourceTime:kCMTimeZero];
286 
287  self.firstFrame = true;
288 
289  return YES;
290 }
291 
292 - (void) appendImage:(VuoImage)image presentationTime:(double)timestamp blockIfNotReady:(BOOL)blockIfNotReady
293 {
294  if(image->pixelsWide != self.originalWidth || image->pixelsHigh != self.originalHeight)
295  {
296  VUserLog("Error: Can't append image because it is not the same dimensions as the the current movie.");
297  return;
298  }
299 
300  if(!self.videoInput.readyForMoreMediaData)
301  {
302  if (blockIfNotReady)
303  {
304  VUserLog("Warning: The AVFoundation asset writer isn't keeping up; waiting for it to catch up before writing this video frame.");
305  while (!self.videoInput.readyForMoreMediaData)
306  usleep(USEC_PER_SEC/10);
307  }
308  else
309  {
310  VUserLog("Error: The AVFoundation asset writer isn't keeping up; dropping this video frame.");
311  return;
312  }
313  }
314 
315  CVPixelBufferRef pb = nil;
316 
317  const unsigned char *buf = VuoImage_getBuffer(image, GL_BGRA);
318 
319  CVReturn ret = CVPixelBufferPoolCreatePixelBuffer(nil, [self.avAdaptor pixelBufferPool], &pb);
320 
321  if(ret != kCVReturnSuccess)
322  {
323  VUserLog("Error: Couldn't get PixelBuffer from pool. %d", ret);
324  return;
325  }
326 
327  ret = CVPixelBufferLockBaseAddress(pb, 0);
328 
329  if(ret != kCVReturnSuccess)
330  {
331  VUserLog("Error: Couldn't lock PixelBuffer base address");
332  return;
333  }
334 
335  unsigned char *bytes = (unsigned char*)CVPixelBufferGetBaseAddress(pb);
336 
337  if(!bytes)
338  {
339  VUserLog("Error: Couldn't get the pixel buffer base address");
340  return;
341  }
342 
343  unsigned int bytesPerRow = CVPixelBufferGetBytesPerRow(pb);
344 
345  for(unsigned long y = 0; y < self.originalHeight; y++)
346  memcpy(bytes + bytesPerRow * (self.originalHeight - y - 1), buf + self.originalWidth * y * 4, self.originalWidth * 4);
347 
348  ret = CVPixelBufferUnlockBaseAddress(pb, 0);
349 
350  if(ret != kCVReturnSuccess)
351  {
352  VUserLog("Error: Couldn't unlock pixelbuffer base address. %d", ret);
353  return;
354  }
355 
356  CMTime presentationTime = CMTimeMakeWithSeconds(timestamp, TIMEBASE);
357 
358  while (CMTimeCompare(presentationTime, self.lastImageTimestamp) <= 0)
359  presentationTime.value++;
360 
361  self.lastImageTimestamp = presentationTime;
362 
363 // VLog("video time: %lld %f pts: %f ts=%g", presentationTime.value, [[NSDate date] timeIntervalSinceDate:self.startDate], CMTimeGetSeconds(presentationTime),timestamp);
364 
365  if (![self.avAdaptor appendPixelBuffer:pb withPresentationTime:presentationTime])
366  {
367  VUserLog("Couldn't write frame %lld (%fs): %s", presentationTime.value, CMTimeGetSeconds(presentationTime), [[_assetWriter.error description] UTF8String]);
368  }
369 
370  if(pb)
371  CVPixelBufferRelease(pb);
372 }
373 
374 - (void) appendAudio:(VuoList_VuoAudioSamples) samples presentationTime:(VuoReal)timestamp blockIfNotReady:(BOOL)blockIfNotReady
375 {
376  long sampleCount = VuoAudioSamples_bufferSize;
377  long channelCount = VuoListGetCount_VuoAudioSamples(samples);
378 
379  if(channelCount != self.originalChannelCount)
380  {
381  if (self.originalChannelCount == -1)
382  VUserLog("Error: Attempting to write %lu audio channels to a silent movie.", channelCount);
383  else
384  VUserLog("Error: Attempting to write %lu audio channels to a movie with %d channels.", channelCount, self.originalChannelCount);
385  return;
386  }
387 
388  if ( !self.audioInput.readyForMoreMediaData) {
389  if (blockIfNotReady)
390  {
391  VUserLog("Warning: The AVFoundation asset writer isn't keeping up; waiting for it to catch up before writing this audio frame.");
392  while (!self.audioInput.readyForMoreMediaData)
393  usleep(USEC_PER_SEC/10);
394  }
395  else
396  {
397  VUserLog("Error: The AVFoundation asset writer isn't keeping up; dropping this audio frame.");
398  return;
399  }
400  }
401 
402  OSStatus status;
403  CMBlockBufferRef bbuf = NULL;
404  CMSampleBufferRef sbuf = NULL;
405 
406  size_t buflen = sampleCount * channelCount * sizeof(float);
407 
408  float* interleaved = (float*)malloc(sizeof(float) * sampleCount * channelCount);
409  VuoDefer(^{ free(interleaved); });
410 
411  for(int n = 0; n < channelCount; n++)
412  {
413  VuoAudioSamples s = VuoListGetValue_VuoAudioSamples(samples, n+1);
414  double *channel = s.samples;
415  if (!channel)
416  {
417  VUserLog("Error: Attempting to write a NULL audio sample buffer. Skipping this audio frame.");
418  return;
419  }
420  if (s.sampleCount != VuoAudioSamples_bufferSize)
421  {
422  VUserLog("Error: Attempting to write an audio sample buffer that has %lld samples, but expected %lld. Skipping this audio frame.", s.sampleCount, VuoAudioSamples_bufferSize);
423  return;
424  }
425 
426  for(int i = 0; i < sampleCount; i++)
427  {
428  interleaved[i*channelCount + n] = channel[i];
429  }
430  }
431 
432  // Create sample buffer for adding to the audio input.
433  CMBlockBufferRef tmp;
434  status = CMBlockBufferCreateWithMemoryBlock( kCFAllocatorDefault,
435  interleaved,
436  buflen,
437  kCFAllocatorNull,
438  NULL,
439  0,
440  buflen,
441  0,
442  &tmp);
443  if (status != noErr) {
444  char *s = VuoOsStatus_getText(status);
445  VUserLog("CMBlockBufferCreateWithMemoryBlock error: %s", s);
446  free(s);
447  return;
448  }
449 
450  status = CMBlockBufferCreateContiguous(kCFAllocatorDefault, tmp, kCFAllocatorDefault, NULL, 0,
451  buflen, kCMBlockBufferAlwaysCopyDataFlag, &bbuf);
452 
453  CFRelease(tmp);
454 
455  if (status != noErr) {
456  char *s = VuoOsStatus_getText(status);
457  VUserLog("CMBlockBufferCreateContiguous error: %s", s);
458  free(s);
459  return;
460  }
461 
462  CMTime presentationTime = CMTimeMakeWithSeconds(timestamp, TIMEBASE);
463 
464  // if(self.firstFrame)
465  // {
466  // self.firstFrame = false;
467  // self.startDate = [NSDate date];
468  // timestamp = CMTimeMake(0, AUDIO_TIMEBASE);
469  // }
470  // else
471  // {
472  // timestamp = CMTimeMake([[NSDate date] timeIntervalSinceDate:self.startDate] * AUDIO_TIMEBASE, AUDIO_TIMEBASE);
473  // }
474 
475  // CMTime timestamp = CMTimeMake(self.audio_sample_position, VuoAudioSamples_sampleRate);
476  // self.audio_sample_position += sampleCount;
477 
478  while (CMTimeCompare(presentationTime, self.lastAudioTimestamp) <= 0)
479  presentationTime.value++;
480 
481  self.lastAudioTimestamp = presentationTime;
482 
483 // VLog("audio time: %lld %f pts: %f ts=%g", presentationTime.value, [[NSDate date] timeIntervalSinceDate:self.startDate], CMTimeGetSeconds(presentationTime),timestamp);
484 
485  status = CMAudioSampleBufferCreateWithPacketDescriptions( kCFAllocatorDefault,
486  bbuf,
487  TRUE,
488  0,
489  NULL,
490  self.audio_fmt_desc,
491  VuoAudioSamples_bufferSize,
492  presentationTime,
493  NULL, &sbuf);
494 
495  if (status != noErr) {
496  VUserLog("CMSampleBufferCreate error");
497  return;
498  }
499 
500  if ( ![self.audioInput appendSampleBuffer:sbuf] )
501  {
502  VUserLog("AppendSampleBuffer error");
503  }
504 
505  CFRelease(bbuf);
506  CFRelease(sbuf);
507  bbuf = nil;
508  sbuf = nil;
509 }
510 
511 - (void) finalizeRecording
512 {
513  if( self.assetWriter )
514  {
515  [self.videoInput markAsFinished];
516 
517  if(self.audioInput != nil)
518  [self.audioInput markAsFinished];
519 
520  dispatch_semaphore_t finishedWriting = dispatch_semaphore_create(0);
521  [_assetWriter finishWritingWithCompletionHandler:^{
522  dispatch_semaphore_signal(finishedWriting);
523  }];
524  dispatch_semaphore_wait(finishedWriting, DISPATCH_TIME_FOREVER);
525  dispatch_release(finishedWriting);
526 
527  if (_assetWriter.status != AVAssetWriterStatusCompleted)
528  VUserLog("Error: %s", [[_assetWriter.error localizedDescription] UTF8String]);
529 
530  self.assetWriter = nil;
531  self.videoInput = nil;
532  self.audioInput = nil;
533  }
534 }
535 
536 - (void) dealloc
537 {
538  if(self.assetWriter)
539  [self finalizeRecording];
540 
541  [super dealloc];
542 }
543 
544 @end
Generated on Wed Feb 26 2020 23:19:42 for Vuo by   doxygen 1.8.17