Vuo  2.3.2
VuoRunnerCocoa+Conversion.mm
Go to the documentation of this file.
1 
11 
12 #include <OpenGL/CGLMacro.h>
13 #include <QuartzCore/CoreImage.h>
14 #include <QuartzCore/CoreVideo.h>
15 
16 #pragma clang diagnostic push
17 #pragma clang diagnostic ignored "-Wdocumentation"
18 #include <json-c/json.h>
19 #pragma clang diagnostic pop
20 
21 #include <dlfcn.h>
22 #include <vector>
23 #include "VuoStringUtilities.hh"
24 #include "VuoType.hh"
25 
26 extern "C" {
27 #include "VuoHeap.h"
28 #include "VuoGlContext.h"
29 
30 #include "../node/node.h"
31 }
32 
33 @implementation VuoRunnerCocoa (Conversion)
34 
40 + (id)cocoaObjectWithVuoValue:(json_object *)vuoValue ofType:(string)type
41 {
42  if (type == "VuoBoolean")
43  {
44  VuoBoolean v = VuoBoolean_makeFromJson(vuoValue);
45  return [NSNumber numberWithBool:v];
46  }
47  if (type == "VuoList_VuoBoolean")
48  {
50  unsigned long count = VuoListGetCount_VuoBoolean(l);
51  NSMutableArray *a = [[NSMutableArray new] autorelease];
52  for (unsigned long i=0; i<count; ++i)
53  [a addObject:[NSNumber numberWithBool:VuoListGetValue_VuoBoolean(l, i+1)]];
54  return a;
55  }
56 
57  if (type == "VuoInteger")
58  {
59  VuoInteger v = VuoInteger_makeFromJson(vuoValue);
60  return [NSNumber numberWithLong:v];
61  }
62  if (type == "VuoList_VuoInteger")
63  {
65  unsigned long count = VuoListGetCount_VuoInteger(l);
66  NSMutableArray *a = [[NSMutableArray new] autorelease];
67  for (unsigned long i=0; i<count; ++i)
68  [a addObject:[NSNumber numberWithLong:VuoListGetValue_VuoInteger(l, i+1)]];
69  return a;
70  }
71 
72  if (type == "VuoReal")
73  {
74  VuoReal v = VuoReal_makeFromJson(vuoValue);
75  return [NSNumber numberWithDouble:v];
76  }
77  if (type == "VuoList_VuoReal")
78  {
80  unsigned long count = VuoListGetCount_VuoReal(l);
81  NSMutableArray *a = [[NSMutableArray new] autorelease];
82  for (unsigned long i=0; i<count; ++i)
83  [a addObject:[NSNumber numberWithDouble:VuoListGetValue_VuoReal(l, i+1)]];
84  return a;
85  }
86 
87  if (type == "VuoText")
88  {
89  VuoText v = VuoText_makeFromJson(vuoValue);
90  VuoRetain(v);
91  NSString *s = [NSString stringWithUTF8String:v];
92  VuoRelease(v);
93  return s;
94  }
95  if (type == "VuoList_VuoText")
96  {
98  unsigned long count = VuoListGetCount_VuoText(l);
99  NSMutableArray *a = [[NSMutableArray new] autorelease];
100  for (unsigned long i=0; i<count; ++i)
101  [a addObject:[NSString stringWithUTF8String:VuoListGetValue_VuoText(l, i+1)]];
102  return a;
103  }
104 
105  if (type == "VuoColor")
106  {
107  VuoColor v = VuoColor_makeFromJson(vuoValue);
108  CGFloat c[4] = {v.r, v.g, v.b, v.a};
109  return [NSColor colorWithColorSpace:[NSColorSpace sRGBColorSpace] components:c count:4];
110  }
111  if (type == "VuoList_VuoColor")
112  {
114  unsigned long count = VuoListGetCount_VuoColor(l);
115  NSMutableArray *a = [[NSMutableArray new] autorelease];
116  for (unsigned long i=0; i<count; ++i)
117  {
119  CGFloat c[4] = {v.r, v.g, v.b, v.a};
120  [a addObject:[NSColor colorWithColorSpace:[NSColorSpace sRGBColorSpace] components:c count:4]];
121  }
122  return a;
123  }
124 
125  if (type == "VuoImage")
126  {
127  VuoImage v = VuoImage_makeFromJson(vuoValue);
128  VuoRetain(v);
129  NSImage *im = [self nsImageWithVuoImage:v];
130  VuoRelease(v);
131  return im;
132  }
133  if (type == "VuoList_VuoImage")
134  {
136  unsigned long count = VuoListGetCount_VuoImage(l);
137  NSMutableArray *a = [[NSMutableArray new] autorelease];
138  for (unsigned long i=0; i<count; ++i)
139  {
141  VuoRetain(v);
142  [a addObject:[self nsImageWithVuoImage:v]];
143  VuoRelease(v);
144  }
145  return a;
146  }
147 
148  if (type == "VuoPoint2d")
149  {
150  VuoPoint2d v = VuoPoint2d_makeFromJson(vuoValue);
151  return [NSValue valueWithPoint:NSMakePoint(v.x, v.y)];
152  }
153  if (type == "VuoList_VuoPoint2d")
154  {
156  unsigned long count = VuoListGetCount_VuoPoint2d(l);
157  NSMutableArray *a = [[NSMutableArray new] autorelease];
158  for (unsigned long i=0; i<count; ++i)
159  {
160  VuoPoint2d v = VuoListGetValue_VuoPoint2d(l, i+1);
161  [a addObject:[NSValue valueWithPoint:NSMakePoint(v.x, v.y)]];
162  }
163  return a;
164  }
165 
166  if (type == "VuoPoint3d")
167  {
168  VuoPoint3d v = VuoPoint3d_makeFromJson(vuoValue);
169  size_t s = sizeof(double)*3;
170  double *d = (double *)malloc(s);
171  d[0] = v.x;
172  d[1] = v.y;
173  d[2] = v.z;
174  return [NSData dataWithBytesNoCopy:d length:s];
175  }
176  if (type == "VuoList_VuoPoint3d")
177  {
179  unsigned long count = VuoListGetCount_VuoPoint3d(l);
180  NSMutableArray *a = [[NSMutableArray new] autorelease];
181  for (unsigned long i=0; i<count; ++i)
182  {
183  VuoPoint3d v = VuoListGetValue_VuoPoint3d(l, i+1);
184  size_t s = sizeof(double)*3;
185  double *d = (double *)malloc(s);
186  d[0] = v.x;
187  d[1] = v.y;
188  d[2] = v.z;
189  [a addObject:[NSData dataWithBytesNoCopy:d length:s]];
190  }
191  return a;
192  }
193 
194  // Maybe it's a list of enum values?
196  {
198  string allowedValuesFunctionName = itemType + "_getAllowedValues";
199  typedef void *(*allowedValuesFunctionType)(void);
200  allowedValuesFunctionType allowedValuesFunction = (allowedValuesFunctionType)dlsym(RTLD_SELF, allowedValuesFunctionName.c_str());
201  if (allowedValuesFunction)
202  if (json_object_is_type(vuoValue, json_type_array))
203  {
204  unsigned long arrayLength = json_object_array_length(vuoValue);
205  if (!arrayLength)
206  return [[NSArray new] autorelease];
207 
208  else if (json_object_is_type(json_object_array_get_idx(vuoValue, 0), json_type_string))
209  {
210  NSMutableArray *a = [[NSMutableArray new] autorelease];
211  for (unsigned long i = 0; i < arrayLength; ++i)
212  [a addObject:[NSString stringWithUTF8String:json_object_get_string(json_object_array_get_idx(vuoValue, i))]];
213  return a;
214  }
215  }
216  }
217 
218  // Maybe it's an enum?
219  string allowedValuesFunctionName = type + "_getAllowedValues";
220  typedef void *(*allowedValuesFunctionType)(void);
221  allowedValuesFunctionType allowedValuesFunction = (allowedValuesFunctionType)dlsym(RTLD_SELF, allowedValuesFunctionName.c_str());
222  if (allowedValuesFunction)
223  if (json_object_is_type(vuoValue, json_type_string))
224  return [NSString stringWithUTF8String:json_object_get_string(vuoValue)];
225 
226  VUserLog("Type %s isn't supported yet. Please https://vuo.org/contact us if you'd like support for this type.", type.c_str());
227  return nil;
228 }
229 
233 static void VuoRunnerCocoa_doNothingCallback(VuoImage imageToFree)
234 {
235 }
236 
242 + (json_object *)vuoValueWithCocoaObject:(id)value
243 {
244  if (!value)
245  return NULL;
246 
247  // Is this a Vuo-compatible NSObject (or CFType that's toll-free bridged to one)?
248  if ([value respondsToSelector:@selector(vuoValue)])
249  return [value vuoValue];
250 
251  // Is this a (non-toll-free bridged) CFType?
252  if (strcmp(object_getClassName(value), "__NSCFType") == 0)
253  {
254  CFTypeID type = CFGetTypeID(value);
255 
256  if (type == CGColorGetTypeID())
257  {
258  NSColorSpace *nscs = [[NSColorSpace alloc] initWithCGColorSpace:CGColorGetColorSpace((CGColorRef)value)];
259  NSColor *color = [NSColor colorWithColorSpace:nscs components:CGColorGetComponents((CGColorRef)value) count:CGColorGetNumberOfComponents((CGColorRef)value)];
260  [nscs release];
261  return [color vuoValue];
262  }
263 
264  if (type == CGImageGetTypeID())
265  {
266  CGImageRef cgimage = (CGImageRef)value;
267 
268  if (CGImageGetAlphaInfo(cgimage) != kCGImageAlphaPremultipliedLast)
269  {
270  VUserLog("Error: Alpha option %d isn't supported yet. Please use kCGImageAlphaPremultipliedLast, and https://vuo.org/contact us if you'd like support for other alpha options.", CGImageGetAlphaInfo(cgimage));
271  return NULL;
272  }
273 
274  if (CGImageGetBitmapInfo(cgimage) & kCGBitmapFloatComponents)
275  {
276  VUserLog("Error: Floating-point pixel values aren't supported yet. Please use integer pixel values, and https://vuo.org/contact us if you'd like support for floating-point pixel values.");
277  return NULL;
278  }
279 
280  if (CGImageGetBitsPerComponent(cgimage) != 8)
281  {
282  VUserLog("Error: BitsPerComponent=%lu isn't supported yet. Please use 8 bits per component, and https://vuo.org/contact us if you'd like support for other bit depths.", CGImageGetBitsPerComponent(cgimage));
283  return NULL;
284  }
285 
286  if (CGImageGetBitsPerPixel(cgimage) != 32)
287  {
288  VUserLog("Error: BitsPerPixel=%lu isn't supported yet. Please use 32 bits per pixel, and https://vuo.org/contact us if you'd like support for other bit depths.", CGImageGetBitsPerPixel(cgimage));
289  return NULL;
290  }
291 
292  unsigned long width = CGImageGetWidth(cgimage);
293  unsigned long height = CGImageGetHeight(cgimage);
294  if (CGImageGetBytesPerRow(cgimage) != width*4)
295  {
296  VUserLog("Error: BytesPerRow must be width*4. https://vuo.org/contact us if you'd like support for inexact strides.");
297  return NULL;
298  }
299 
300 // NSString *colorSpaceName = (NSString *)CGColorSpaceCopyName(CGImageGetColorSpace(cgimage));
301 // if (![colorSpaceName isEqualToString:(NSString *)kCGColorSpaceSRGB])
302 // {
303 // VUserLog("Error: Colorspace '%s' isn't supported yet. Please use kCGColorSpaceSRGB, and https://vuo.org/contact us if you'd like support for other colorspaces.", [colorSpaceName UTF8String]);
304 // [colorSpaceName release];
305 // return NULL;
306 // }
307 // CFRelease(colorSpaceName);
308 
309  CGDataProviderRef provider = CGImageGetDataProvider(cgimage);
310  CFDataRef data = CGDataProviderCopyData(provider);
311 
312  char *bitmapData = (char *)CFDataGetBytePtr(data);
313  int bytesPerRow = width*4;
314  char *flippedBitmapData = (char *)malloc(bytesPerRow*height);
315 
316  // Flip the image data (CGImage is unflipped, but VuoImage_makeFromBuffer() expects flipped).
317  for (unsigned long y = 0; y < height; ++y)
318  memcpy(flippedBitmapData + bytesPerRow * (height - y - 1), bitmapData + bytesPerRow * y, bytesPerRow);
319 
320  CFRelease(data);
321 
322  VuoImage vi = VuoImage_makeFromBuffer(flippedBitmapData, GL_RGBA, width, height, VuoImageColorDepth_8, ^(void *buffer){ free(flippedBitmapData); });
323  VuoRetain(vi);
324 
326  VuoRelease(vi);
327  return js;
328  }
329 
330  if (type == CVPixelBufferGetTypeID())
331  {
332  CVPixelBufferRef cvpb = (CVPixelBufferRef)value;
333 
334  if (CVPixelBufferIsPlanar(cvpb))
335  {
336  VUserLog("Error: Planar buffers aren't supported yet. Please use chunky buffers, and https://vuo.org/contact us if you'd like support for planar buffers.");
337  return NULL;
338  }
339 
340  if (CVPixelBufferGetPixelFormatType(cvpb) != kCVPixelFormatType_32BGRA)
341  {
342  unsigned long pf = CVPixelBufferGetPixelFormatType(cvpb);
343  VUserLog("Error: Pixel format %lu isn't supported yet. Please use kCVPixelFormatType_32BGRA, and https://vuo.org/contact us if you'd like support for other pixel formats.", pf);
344  return NULL;
345  }
346 
347  unsigned long width = CVPixelBufferGetWidth(cvpb);
348  unsigned long height = CVPixelBufferGetHeight(cvpb);
349  if (CVPixelBufferGetBytesPerRow(cvpb) != width*4)
350  {
351  VUserLog("Error: BytesPerRow must be width*4. https://vuo.org/contact us if you'd like support for inexact strides.");
352  return NULL;
353  }
354 
355  CVReturn ret = CVPixelBufferLockBaseAddress(cvpb, kCVPixelBufferLock_ReadOnly);
356  if (ret != kCVReturnSuccess)
357  {
358  VUserLog("CVPixelBufferLockBaseAddress() failed: %d", ret);
359  return NULL;
360  }
361 
362  const unsigned char *pixels = (const unsigned char *)CVPixelBufferGetBaseAddress(cvpb);
363  if (!pixels)
364  {
365  VUserLog("Error: CVPixelBufferGetBaseAddress() returned NULL.");
366  return NULL;
367  }
368 
369  CVPixelBufferRetain(cvpb);
370  VuoImage vi = VuoImage_makeFromBuffer(pixels, GL_BGRA, width, height, VuoImageColorDepth_8, ^(void *buffer){ CVPixelBufferRelease(cvpb); });
371  VuoRetain(vi);
372 
373  ret = CVPixelBufferUnlockBaseAddress(cvpb, kCVPixelBufferLock_ReadOnly);
374  if (ret != kCVReturnSuccess)
375  {
376  VUserLog("CVPixelBufferUnlockBaseAddress() failed: %d", ret);
377  return NULL;
378  }
379 
381  VuoRelease(vi);
382  return js;
383  }
384 
385 #pragma clang diagnostic push
386 #pragma clang diagnostic ignored "-Wdeprecated-declarations"
387  if (type == CVOpenGLTextureGetTypeID())
388  {
389  CVOpenGLTextureRef cvgl = (CVOpenGLTextureRef)value;
390 
391  GLuint name = CVOpenGLTextureGetName(cvgl);
392  if (!name)
393  {
394  VUserLog("Error: CVOpenGLTextureGetName() returned 0.");
395  return NULL;
396  }
397 
398  GLfloat lowerLeft[2];
399  GLfloat lowerRight[2];
400  GLfloat upperRight[2];
401  GLfloat upperLeft[2];
402  CVOpenGLTextureGetCleanTexCoords(cvgl, lowerLeft, lowerRight, upperRight, upperLeft);
403 
404  unsigned int width = abs((int)(lowerRight[0] - lowerLeft[0]));
405  unsigned int height = abs((int)(upperLeft[1] - lowerLeft[1]));
406 // VLog("%dx%d",width,height);
407 
408  VuoImage vi;
409  GLenum target = CVOpenGLTextureGetTarget(cvgl);
410  if (target == GL_TEXTURE_2D)
411  vi = VuoImage_makeClientOwned(name, GL_RGBA, width, height, VuoRunnerCocoa_doNothingCallback, NULL);
412  else if (target == GL_TEXTURE_RECTANGLE_ARB)
413  vi = VuoImage_makeClientOwnedGlTextureRectangle(name, GL_RGBA, width, height, VuoRunnerCocoa_doNothingCallback, NULL);
414  else
415  {
416  VUserLog("Error: GL texture target %d isn't supported yet. Please use GL_TEXTURE_2D or GL_TEXTURE_RECTANGLE_ARB, and https://vuo.org/contact us if you'd like support for other targets.", target);
417  return NULL;
418  }
419 
420  VuoRetain(vi);
422  VuoRelease(vi);
423  return js;
424  }
425 #pragma clang diagnostic pop
426 
427  NSString *typeDescription = (NSString *)CFCopyTypeIDDescription(type);
428  VUserLog("Error: Unknown CFType '%s'", [typeDescription UTF8String]);
429  [typeDescription release];
430  return NULL;
431  }
432 
433  VUserLog("Error: Unknown type '%s' for object '%s'", object_getClassName(value), [[value description] UTF8String]);
434  return NULL;
435 }
436 
440 + (NSImage *)nsImageWithVuoImage:(VuoImage)vi
441 {
442  if (!vi)
443  return nil;
444 
445  // Allocate memory to store the image data.
446  unsigned int bytesPerRow = 4 * vi->pixelsWide;
447  NSBitmapImageRep *nbi = [[NSBitmapImageRep alloc] initWithBitmapDataPlanes:NULL
448  pixelsWide:vi->pixelsWide
449  pixelsHigh:vi->pixelsHigh
450  bitsPerSample:8
451  samplesPerPixel:4
452  hasAlpha:YES
453  isPlanar:NO
454  colorSpaceName:NSDeviceRGBColorSpace
455  bytesPerRow:bytesPerRow
456  bitsPerPixel:0];
457  if (!nbi)
458  return nil;
459 
460  // Download the image data from the GPU.
461  unsigned char *bitmapData = [nbi bitmapData];
462  VuoGlContext_perform(^(CGLContextObj cgl_ctx){
463  glBindTexture(GL_TEXTURE_2D, vi->glTextureName);
464  glGetTexImage(GL_TEXTURE_2D, 0, GL_RGBA, GL_UNSIGNED_BYTE, bitmapData);
465  glBindTexture(GL_TEXTURE_2D, 0);
466  });
467 
468  // Flip the image data (OpenGL returns flipped data, but NSBitmapImageRep assumes it is not flipped).
469  unsigned char *tmp = (unsigned char *)malloc(bytesPerRow);
470  for (unsigned long y = 0; y < vi->pixelsHigh / 2; ++y)
471  {
472  memcpy(tmp, bitmapData + bytesPerRow * y, bytesPerRow);
473  memcpy(bitmapData + bytesPerRow * y, bitmapData + bytesPerRow * (vi->pixelsHigh - y - 1), bytesPerRow);
474  memcpy(bitmapData + bytesPerRow * (vi->pixelsHigh - y - 1), tmp, bytesPerRow);
475  }
476  free(tmp);
477 
478  NSImage *ni = [[NSImage alloc] initWithSize:[nbi size]];
479  [ni addRepresentation:nbi];
480  [nbi release];
481  return [ni autorelease];
482 }
483 
484 @end
485 
493 {
494  char type = [self objCType][0];
495 
496  if (type == 'B')
497  return VuoBoolean_getJson([self boolValue]);
498 
499  if (type == 'f' || type == 'd')
500  return VuoReal_getJson([self doubleValue]);
501 
502  if (type == 'c' || type == 'i' || type == 's' || type == 'l' || type == 'q'
503  || type == 'C' || type == 'I' || type == 'S' || type == 'L' || type == 'Q')
504  return VuoInteger_getJson([self longLongValue]);
505 
506  VUserLog("Error: Unknown type '%s'", [self objCType]);
507  return NULL;
508 }
509 @end
510 
518 {
519  return VuoText_getJson([self UTF8String]);
520 }
521 @end
522 
530 {
531  CGFloat c[4] = {0,0,0,1};
532  NSColor *color = [self colorUsingColorSpace:[NSColorSpace sRGBColorSpace]];
533  [color getComponents:c];
534  return VuoColor_getJson(VuoColor_makeWithRGBA(c[0], c[1], c[2], c[3]));
535 }
536 @end
537 
545 {
546  NSColorSpace *nscs = [[NSColorSpace alloc] initWithCGColorSpace:[self colorSpace]];
547  NSColor *color = [NSColor colorWithColorSpace:nscs components:[self components] count:[self numberOfComponents]];
548  [nscs release];
549  return [color vuoValue];
550 }
551 @end
552 
560 {
561  float scale = [self recommendedLayerContentsScale:0];
562  unsigned int pixelsWide = [self size].width * scale;
563  unsigned int pixelsHigh = [self size].height * scale;
564 
565  NSBitmapImageRep *nbir = [[NSBitmapImageRep alloc]
566  initWithBitmapDataPlanes: NULL
567  pixelsWide:pixelsWide
568  pixelsHigh:pixelsHigh
569  bitsPerSample:8
570  samplesPerPixel:4
571  hasAlpha:YES
572  isPlanar:NO
573  colorSpaceName:NSDeviceRGBColorSpace
574  bytesPerRow:pixelsWide*4
575  bitsPerPixel:32];
576 
577  NSGraphicsContext *ngc = [NSGraphicsContext graphicsContextWithBitmapImageRep:nbir];
578  [NSGraphicsContext saveGraphicsState];
579  [NSGraphicsContext setCurrentContext:ngc];
580  [self drawInRect:NSMakeRect(0,0,pixelsWide,pixelsHigh) fromRect:NSZeroRect operation:NSCompositingOperationSourceOver fraction:1 respectFlipped:YES hints:nil];
581  [ngc flushGraphics];
582  [NSGraphicsContext setCurrentContext:nil];
583  [NSGraphicsContext restoreGraphicsState];
584 
585  json_object *js = [nbir vuoValue];
586  [nbir release];
587  return js;
588 }
589 @end
590 
598 {
599  // Flip the image data (OpenGL expects flipped data, but NSBitmapImageRep is not flipped).
600  unsigned char *bitmapData = [self bitmapData];
601  unsigned long width = [self size].width;
602  unsigned long height = [self size].height;
603  unsigned long bytesPerRow = width * 4;
604  unsigned char *bitmapDataFlipped = (unsigned char *)malloc(bytesPerRow * height);
605  for (unsigned long y = 0; y < height; ++y)
606  for (unsigned long x = 0; x < width; ++x)
607  {
608  // Swizzle RGBA -> BGRA (so the GPU doesn't have to swizzle during each fetch)
609  bitmapDataFlipped[bytesPerRow * (height - y - 1) + x * 4 + 2] = bitmapData[bytesPerRow * y + x * 4 + 0];
610  bitmapDataFlipped[bytesPerRow * (height - y - 1) + x * 4 + 1] = bitmapData[bytesPerRow * y + x * 4 + 1];
611  bitmapDataFlipped[bytesPerRow * (height - y - 1) + x * 4 + 0] = bitmapData[bytesPerRow * y + x * 4 + 2];
612  bitmapDataFlipped[bytesPerRow * (height - y - 1) + x * 4 + 3] = bitmapData[bytesPerRow * y + x * 4 + 3];
613  }
614 
615  VuoImage vi = VuoImage_makeFromBuffer(bitmapDataFlipped, GL_BGRA, width, height, VuoImageColorDepth_8, ^(void *buffer){ free(buffer); });
616  VuoRetain(vi);
618  VuoRelease(vi);
619  return json;
620 }
621 @end
622 
630 {
631  NSPoint p = [self pointValue];
632  return VuoPoint2d_getJson(VuoPoint2d_make(p.x, p.y));
633 }
634 @end
635 
636 @implementation NSData (VuoRunnerCocoaConversion)
643 {
644  double p[3] = {0,0,0};
645  [self getBytes:p length:sizeof(double)*3];
646  return VuoPoint3d_getJson(VuoPoint3d_make(p[0], p[1], p[2]));
647 }
648 @end
649 
657 {
658  json_object *arrayJson = json_object_new_array();
659  for (id item in self)
660  {
661  json_object *itemJson = [item vuoValue];
662  json_object_array_add(arrayJson, itemJson);
663  }
664  return arrayJson;
665 }
666 @end