Vuo  2.0.0
VuoCompiler.cc
Go to the documentation of this file.
1 
10 #include <string.h>
11 #include <sys/param.h>
12 #include <sys/stat.h>
13 #include <sys/time.h>
14 #include <sstream>
15 #include <CoreFoundation/CoreFoundation.h>
16 #include "VuoCompiler.hh"
20 #include "VuoCompilerException.hh"
22 #include "VuoCompilerGraph.hh"
24 #include "VuoCompilerIssue.hh"
25 #include "VuoCompilerNode.hh"
26 #include "VuoCompilerPort.hh"
27 #include "VuoCompilerPortClass.hh"
31 #include "VuoComposition.hh"
32 #include "VuoException.hh"
33 #include "VuoGenericType.hh"
34 #include "VuoModuleCompiler.hh"
36 #include "VuoNode.hh"
37 #include "VuoNodeClass.hh"
38 #include "VuoNodeSet.hh"
39 #include "VuoPublishedPort.hh"
40 #include "VuoRunner.hh"
42 #include "VuoStringUtilities.hh"
43 
44 #if VUO_PRO
45 #include "VuoPro.hh"
46 #endif
47 
48 map<string, VuoFileUtilities::File *> VuoCompiler::Environment::moduleCacheFileForLocking;
49 dispatch_queue_t VuoCompiler::Environment::moduleCacheBuildingQueue = dispatch_queue_create("org.vuo.compiler.cache", NULL);
50 const string VuoCompiler::Environment::pidCacheDirPrefix = "pid-";
51 set<VuoCompiler *> VuoCompiler::allCompilers;
52 dispatch_queue_t VuoCompiler::environmentQueue = dispatch_queue_create("org.vuo.compiler.environment", NULL);
53 vector< vector<VuoCompiler::Environment *> > VuoCompiler::sharedEnvironments;
54 map<string, vector<VuoCompiler::Environment *> > VuoCompiler::environmentsForCompositionFamily;
55 dispatch_group_t VuoCompiler::moduleSourceCompilersExist = dispatch_group_create();
56 string VuoCompiler::vuoFrameworkInProgressPath;
57 
58 dispatch_queue_t llvmQueue = NULL;
59 
63 static void __attribute__((constructor)) VuoCompiler_init(void)
64 {
65  // Increase the open-files limit (macOS defaults to 256).
66  struct rlimit rl{OPEN_MAX, OPEN_MAX};
67  getrlimit(RLIMIT_NOFILE, &rl);
68  rl.rlim_cur = MIN(OPEN_MAX, rl.rlim_max);
69  if (setrlimit(RLIMIT_NOFILE, &rl))
70  VUserLog("Warning: Couldn't set open-files limit: %s", strerror(errno));
71 
72 
73  llvmQueue = dispatch_queue_create("org.vuo.compiler.llvm", NULL);
74 
75  dispatch_sync(llvmQueue, ^{
76  llvm::InitializeNativeTarget();
77 
78  // If the Vuo compiler/linker...
79  // 1. Loads a node class that uses dispatch_object_t.
80  // 2. Generates code that uses dispatch_object_t.
81  // 3. Links the node class into a composition.
82  // 4. Generates more code that uses dispatch_object_t.
83  // ... then Step 4 ends up with the wrong llvm::Type for dispatch_object_t.
84  //
85  // A workaround is to generate some code that uses dispatch_object_t before doing Step 1.
86  //
87  // https://b33p.net/kosada/node/3845
88  // http://lists.cs.uiuc.edu/pipermail/llvmdev/2012-December/057075.html
89  Module module("", getGlobalContext());
91 
92  // Workaround for a possibly related error where the compiler ends up with the wrong
93  // llvm::Type for dispatch_semaphore_s. https://b33p.net/kosada/node/10545
95 
96  // Load the NodeContext and PortContext struct types preemptively to make sure that
97  // their fields get the right dispatch types. If these struct types were to be loaded
98  // first from a subcomposition module, they'd get the wrong dispatch types.
99  // https://b33p.net/kosada/node/11160
102  });
103 }
104 
108 VuoCompiler::ModuleInfo::ModuleInfo(Environment *environment, const string &searchPath, const string &relativePath,
109  bool isSourceFile, bool isSubcomposition)
110 {
111  VuoFileUtilities::File *file = new VuoFileUtilities::File(searchPath, relativePath);
112  initialize(environment, searchPath, file, isSourceFile, isSubcomposition);
113 }
114 
118 VuoCompiler::ModuleInfo::ModuleInfo(Environment *environment, const string &searchPath, VuoFileUtilities::File *file,
119  bool isSourceFile, bool isSubcomposition)
120 {
121  initialize(environment, searchPath, file, isSourceFile, isSubcomposition);
122 }
123 
127 void VuoCompiler::ModuleInfo::initialize(Environment *environment, const string &searchPath, VuoFileUtilities::File *file,
128  bool isSourceFile, bool isSubcomposition)
129 {
130  this->environment = environment;
131  this->searchPath = searchPath;
132  this->file = file;
133  moduleKey = getModuleKeyForPath(file->getRelativePath());
134  sourceCodeOverridden = false;
135  lastModified = VuoFileUtilities::getFileLastModifiedInSeconds(file->isInArchive() ? file->getArchivePath() : file->path());
136  longestDownstreamPath = 0;
137  attempted = false;
138  loading = nullptr;
139 
140 #if VUO_PRO
141  init_Pro();
142 #endif
143 
144  if (isSourceFile)
145  {
146  loading = dispatch_group_create();
147  sourceCode = file->getContentsAsString();
148 
149  if (isSubcomposition)
150  {
151  try
152  {
154  }
155  catch (...)
156  {
157  // Issues parsing the composition will be caught later when compiling it.
158  }
159  }
160  }
161 }
162 
163 VuoCompiler::ModuleInfo::~ModuleInfo(void)
164 {
165 #if VUO_PRO
166  fini_Pro();
167 #endif
168 
169  delete file;
170 
171  if (loading)
172  dispatch_release(loading);
173 }
174 
175 VuoCompiler::Environment * VuoCompiler::ModuleInfo::getEnvironment(void) const
176 {
177  return environment;
178 }
179 
180 string VuoCompiler::ModuleInfo::getSearchPath(void) const
181 {
182  return searchPath;
183 }
184 
185 string VuoCompiler::ModuleInfo::getModuleKey(void) const
186 {
187  return moduleKey;
188 }
189 
190 VuoFileUtilities::File * VuoCompiler::ModuleInfo::getFile(void) const
191 {
192  return file;
193 }
194 
195 string VuoCompiler::ModuleInfo::getSourceCode(void) const
196 {
197  return sourceCode;
198 }
199 
200 void VuoCompiler::ModuleInfo::setSourceCode(const string &sourceCode)
201 {
202  this->sourceCode = sourceCode;
203 }
204 
205 void VuoCompiler::ModuleInfo::revertSourceCode(void)
206 {
207  sourceCode = file->getContentsAsString();
208 }
209 
210 bool VuoCompiler::ModuleInfo::isSourceCodeOverridden(void) const
211 {
212  return sourceCodeOverridden;
213 }
214 
215 void VuoCompiler::ModuleInfo::setSourceCodeOverridden(bool overridden)
216 {
217  sourceCodeOverridden = overridden;
218 }
219 
220 bool VuoCompiler::ModuleInfo::isNewerThan(ModuleInfo *other) const
221 {
222  return lastModified > other->lastModified;
223 }
224 
225 bool VuoCompiler::ModuleInfo::isNewerThan(unsigned long ageInSeconds) const
226 {
227  return lastModified > ageInSeconds;
228 }
229 
230 bool VuoCompiler::ModuleInfo::isOlderThan(unsigned long ageInSeconds) const
231 {
232  return lastModified < ageInSeconds;
233 }
234 
235 void VuoCompiler::ModuleInfo::setLastModifiedToNow(void)
236 {
237  struct timeval t;
238  gettimeofday(&t, NULL);
239  lastModified = t.tv_sec;
240 }
241 
242 set<string> VuoCompiler::ModuleInfo::getContainedNodeClasses(void) const
243 {
244  return containedNodeClasses;
245 }
246 
247 int VuoCompiler::ModuleInfo::getLongestDownstreamPath(void) const
248 {
249  return longestDownstreamPath;
250 }
251 
252 void VuoCompiler::ModuleInfo::setLongestDownstreamPath(int pathLength)
253 {
254  longestDownstreamPath = pathLength;
255 }
256 
257 bool VuoCompiler::ModuleInfo::getAttempted(void) const
258 {
259  return attempted;
260 }
261 
262 void VuoCompiler::ModuleInfo::setAttempted(bool attempted)
263 {
264  this->attempted = attempted;
265 }
266 
267 dispatch_group_t VuoCompiler::ModuleInfo::getLoadingGroup(void) const
268 {
269  return loading;
270 }
271 
272 void VuoCompiler::ModuleInfo::dump() const
273 {
274  fprintf(stderr, "module %s:\n"
275  "\tloadingGroup=%p\n"
276  "\tsearchPath=%s\n"
277  "\tattempted=%d\n"
278  "\tenvironment=%s\n"
279  "\tfile=%s%s%s\n"
280  "\tsourceCodeOverridden=%d\n"
281  "\tsourceCode=%s\n"
282  "\tcontainedNodeClasses:\n",
283  moduleKey.c_str(),
284  loading,
285  searchPath.c_str(),
286  attempted,
287  environment->getCompiledModuleCachePath().c_str(),
288  file->isInArchive() ? file->getArchivePath().c_str() : "",
289  file->isInArchive() ? "::" : "",
290  file->isInArchive() ? file->getRelativePath().c_str() : file->path().c_str(),
291  sourceCodeOverridden,
292  sourceCode.c_str());
293  for (auto i: containedNodeClasses)
294  fprintf(stderr, "\t\t%s\n", i.c_str());
295 }
296 
297 VuoCompiler::DependencyGraphVertex * VuoCompiler::DependencyGraphVertex::vertexForDependency(const string &dependency, VuoDirectedAcyclicGraph *graph)
298 {
299  VuoDirectedAcyclicGraph::Vertex *v = graph->findVertex(dependency);
300  if (v)
301  return dynamic_cast<DependencyGraphVertex *>(v);
302 
303  DependencyGraphVertex *vv = new DependencyGraphVertex();
304  vv->dependency = dependency;
305  vv->environment = NULL;
306  vv->compatible = true;
307  return vv;
308 }
309 
310 string VuoCompiler::DependencyGraphVertex::getDependency(void)
311 {
312  return dependency;
313 }
314 
315 VuoCompiler::Environment * VuoCompiler::DependencyGraphVertex::getEnvironment(void)
316 {
317  return environment;
318 }
319 
320 void VuoCompiler::DependencyGraphVertex::setEnvironment(Environment *environment)
321 {
322  this->environment = environment;
323 }
324 
325 bool VuoCompiler::DependencyGraphVertex::isCompatible(void)
326 {
327  return compatible;
328 }
329 
330 void VuoCompiler::DependencyGraphVertex::setCompatible(bool compatible)
331 {
332  this->compatible = compatible;
333 }
334 
335 string VuoCompiler::DependencyGraphVertex::key(void)
336 {
337  return dependency;
338 }
339 
345 VuoCompiler::ModuleInfoIterator::ModuleInfoIterator(map<string, map<string, ModuleInfo *> > *allModuleInfos)
346 {
347  this->allModuleInfos = allModuleInfos;
348  hasSearchModuleKeys = false;
349  initialize();
350 }
351 
357 VuoCompiler::ModuleInfoIterator::ModuleInfoIterator(map<string, map<string, ModuleInfo *> > *allModuleInfos, const set<string> &searchModuleKeys)
358 {
359  this->allModuleInfos = allModuleInfos;
360  this->searchModuleKeys = searchModuleKeys;
361  hasSearchModuleKeys = true;
362  initialize();
363 }
364 
370 void VuoCompiler::ModuleInfoIterator::initialize(void)
371 {
372  currSearchPath = allModuleInfos->begin();
373  hasCurrModuleKey = false;
374 }
375 
381 VuoCompiler::ModuleInfo * VuoCompiler::ModuleInfoIterator::next(void)
382 {
383  for ( ; currSearchPath != allModuleInfos->end(); ++currSearchPath)
384  {
385  if (! hasCurrModuleKey)
386  {
387  currModuleKey = currSearchPath->second.begin();
388  hasCurrModuleKey = true;
389  }
390 
391  for ( ; currModuleKey != currSearchPath->second.end(); ++currModuleKey)
392  {
393  if (! hasSearchModuleKeys || searchModuleKeys.find(currModuleKey->first) != searchModuleKeys.end())
394  {
395  ModuleInfo *moduleInfo = currModuleKey->second;
396  ++currModuleKey;
397  return moduleInfo;
398  }
399  }
400 
401  hasCurrModuleKey = false;
402  }
403 
404  return NULL;
405 }
406 
410 VuoCompiler::Environment::Environment(void)
411 {
412  compilersToNotifyQueue = dispatch_queue_create("org.vuo.compiler.notify", 0);
413  moduleSearchPathContentsChangedQueue = dispatch_queue_create("org.vuo.compiler.watch", 0);
414  isModuleCacheableDataDirty = false;
415  isModuleCacheInitialized = false;
416  isModuleCacheAvailable = false;
417  dependencyGraph = new VuoDirectedAcyclicGraph();
418  compositionDependencyGraph = new VuoDirectedAcyclicGraph();
419  moduleCompilationQueue = new VuoModuleCompilationQueue();
420 }
421 
425 VuoCompiler::Environment::~Environment(void)
426 {
427  stopWatchingModuleSearchPaths();
428  dispatch_sync(moduleSearchPathContentsChangedQueue, ^{});
429 
430  dispatch_release(moduleSearchPathContentsChangedQueue);
431  dispatch_release(compilersToNotifyQueue);
432 
433  for (map<string, VuoCompilerNodeClass *>::iterator i = nodeClasses.begin(); i != nodeClasses.end(); ++i)
434  destroyModule(i->second);
435 
436  for (map<string, VuoCompilerType *>::iterator i = types.begin(); i != types.end(); ++i)
437  destroyModule(i->second);
438 
439  for (map<string, VuoCompilerModule *>::iterator i = libraryModules.begin(); i != libraryModules.end(); ++i)
440  destroyModule(i->second);
441 
442  for (set<VuoCompilerGenericType *>::iterator i = genericTypes.begin(); i != genericTypes.end(); ++i)
443  {
444  VuoType *base = (*i)->getBase();
445  delete *i;
446  delete base;
447  }
448 
449  for (map<string, VuoNodeSet *>::iterator i = nodeSetForName.begin(); i != nodeSetForName.end(); ++i)
450  delete i->second;
451 
452  for (map<string, map<string, ModuleInfo *> >::iterator i = moduleFilesAtSearchPath.begin(); i != moduleFilesAtSearchPath.end(); ++i)
453  for (map<string, ModuleInfo *>::iterator j = i->second.begin(); j != i->second.end(); ++j)
454  delete j->second;
455 
456  for (map<string, map<string, ModuleInfo *> >::iterator i = sourceFilesAtSearchPath.begin(); i != sourceFilesAtSearchPath.end(); ++i)
457  for (map<string, ModuleInfo *>::iterator j = i->second.begin(); j != i->second.end(); ++j)
458  delete j->second;
459 
460  delete dependencyGraph;
461  delete compositionDependencyGraph;
462 }
463 
467 void VuoCompiler::Environment::addCompilerToNotify(VuoCompiler *compiler)
468 {
469  dispatch_sync(compilersToNotifyQueue, ^{
470  compilersToNotify.insert(compiler);
471  });
472 }
473 
477 void VuoCompiler::Environment::removeCompilerToNotify(VuoCompiler *compiler)
478 {
479  dispatch_sync(compilersToNotifyQueue, ^{
480  compilersToNotify.erase(compiler);
481  });
482 }
483 
489 map<string, VuoCompilerNodeClass *> VuoCompiler::Environment::getNodeClasses(void)
490 {
491  return nodeClasses;
492 }
493 
499 VuoCompilerNodeClass * VuoCompiler::Environment::getNodeClass(const string &moduleKey)
500 {
501  map<string, VuoCompilerNodeClass *>::iterator nodeClassIter = nodeClasses.find(moduleKey);
502  if (nodeClassIter != nodeClasses.end())
503  return nodeClassIter->second;
504 
505  return NULL;
506 }
507 
512 VuoCompilerNodeClass * VuoCompiler::Environment::takeNodeClass(const string &moduleKey)
513 {
514  VuoCompilerNodeClass *nodeClass = NULL;
515 
516  map<string, VuoCompilerNodeClass *>::iterator nodeClassIter = nodeClasses.find(moduleKey);
517  if (nodeClassIter != nodeClasses.end())
518  {
519  nodeClass = nodeClassIter->second;
520 
521  nodeClasses.erase(nodeClassIter);
522  removeFromDependencyGraph(nodeClass);
523  modulesChanged();
524  }
525 
526  return nodeClass;
527 }
528 
534 map<string, VuoCompilerType *> VuoCompiler::Environment::getTypes(void)
535 {
536  return types;
537 }
538 
544 VuoCompilerType * VuoCompiler::Environment::getType(const string &moduleKey)
545 {
546  map<string, VuoCompilerType *>::iterator typeIter = types.find(moduleKey);
547  if (typeIter != types.end())
548  return typeIter->second;
549 
550  return NULL;
551 }
552 
558 map<string, VuoNodeSet *> VuoCompiler::Environment::getNodeSets(void)
559 {
560  return nodeSetForName;
561 }
562 
568 map<string, VuoCompilerModule *> VuoCompiler::Environment::getLibraryModules(void)
569 {
570  return libraryModules;
571 }
572 
579 VuoCompilerModule * VuoCompiler::Environment::findModule(const string &moduleKey)
580 {
581  map<string, VuoCompilerNodeClass *>::iterator nodeClassIter = nodeClasses.find(moduleKey);
582  if (nodeClassIter != nodeClasses.end())
583  return nodeClassIter->second;
584 
585  map<string, VuoCompilerType *>::iterator typeIter = types.find(moduleKey);
586  if (typeIter != types.end())
587  return typeIter->second;
588 
589  map<string, VuoCompilerModule *>::iterator libraryIter = libraryModules.find(moduleKey);
590  if (libraryIter != libraryModules.end())
591  return libraryIter->second;
592 
593  return NULL;
594 }
595 
601 VuoNodeSet * VuoCompiler::Environment::findNodeSet(const string &name)
602 {
603 
604  map<string, VuoNodeSet *>::iterator nodeSetIter = nodeSetForName.find(name);
605  if (nodeSetIter != nodeSetForName.end())
606  return nodeSetIter->second;
607 
608  return NULL;
609 }
610 
617 void VuoCompiler::Environment::addModuleSearchPath(const string &path, bool shouldWatch)
618 {
619  moduleSearchPaths.push_back(path);
620 
621  updateModulesAtSearchPath(path);
622  updateSourceFilesAtSearchPath(path);
623 
624  if (shouldWatch)
625  startWatchingModuleSearchPath(path);
626 }
627 
633 vector<string> VuoCompiler::Environment::getModuleSearchPaths(void)
634 {
635  return moduleSearchPaths;
636 }
637 
643 void VuoCompiler::Environment::addHeaderSearchPath(const string &path)
644 {
645  headerSearchPaths.push_back(path);
646 }
647 
653 vector<string> VuoCompiler::Environment::getHeaderSearchPaths(void)
654 {
655  return headerSearchPaths;
656 }
657 
663 void VuoCompiler::Environment::addLibrarySearchPath(const string &path)
664 {
665  librarySearchPaths.push_back(path);
666 }
667 
673 vector<string> VuoCompiler::Environment::getLibrarySearchPaths(void)
674 {
675  return librarySearchPaths;
676 }
677 
683 void VuoCompiler::Environment::addFrameworkSearchPath(const string &path)
684 {
685  frameworkSearchPaths.push_back(path);
686 }
687 
693 vector<string> VuoCompiler::Environment::getFrameworkSearchPaths(void)
694 {
695  return frameworkSearchPaths;
696 }
697 
698 VuoDirectedAcyclicGraph * VuoCompiler::Environment::getDependencyGraph(void)
699 {
700  return dependencyGraph;
701 }
702 
703 VuoDirectedAcyclicGraph * VuoCompiler::Environment::getCompositionDependencyGraph(void)
704 {
705  return compositionDependencyGraph;
706 }
707 
713 void VuoCompiler::Environment::setModuleCachePath(const string &path)
714 {
715  moduleCachePath = path;
716 }
717 
723 string VuoCompiler::Environment::getCompiledModuleCachePath(void)
724 {
725  return (moduleCachePath.empty() ? "" : moduleCachePath + "/Modules");
726 }
727 
733 string VuoCompiler::Environment::getOverriddenCompiledModuleCachePath(void)
734 {
735  if (moduleCachePath.empty())
736  return "";
737 
738  ostringstream pid;
739  pid << getpid();
740 
741  string dir, moduleCacheDirName, ext;
742  VuoFileUtilities::splitPath(moduleCachePath, dir, moduleCacheDirName, ext);
743 
744  return (VuoFileUtilities::getCachePath() + "/" + pidCacheDirPrefix + pid.str() + "/" + moduleCacheDirName + "/Modules");
745 }
746 
754 void VuoCompiler::Environment::addExpatriateSourceFile(const string &sourcePath)
755 {
756  expatriateSourceFiles.push_back(sourcePath);
757 
758  if (find(moduleSearchPaths.begin(), moduleSearchPaths.end(), "") == moduleSearchPaths.end())
759  moduleSearchPaths.push_back("");
760 
761  auto iter = sourceFilesAtSearchPath.find("");
762  if (iter != sourceFilesAtSearchPath.end())
763  sourceFilesAtSearchPath.erase(iter);
764 }
765 
771 void VuoCompiler::Environment::removeExpatriateSourceFile(const string &sourcePath)
772 {
773  for (auto i = expatriateSourceFiles.begin(); i != expatriateSourceFiles.end(); ++i)
774  {
775  if (VuoFileUtilities::arePathsEqual(*i, sourcePath))
776  {
777  expatriateSourceFiles.erase(i);
778 
779  auto iter = sourceFilesAtSearchPath.find("");
780  if (iter != sourceFilesAtSearchPath.end())
781  sourceFilesAtSearchPath.erase(iter);
782 
783  break;
784  }
785  }
786 }
787 
800 void VuoCompiler::Environment::updateModulesAtSearchPath(const string &path, bool shouldCleanModuleCache)
801 {
802  if (moduleFilesAtSearchPath.find(path) != moduleFilesAtSearchPath.end())
803  return;
804 
805  set<string> moduleExtensions;
806  moduleExtensions.insert("vuonode");
807  moduleExtensions.insert("vuonode+");
808  moduleExtensions.insert("bc");
809  moduleExtensions.insert("bc+");
810  set<string> archiveExtensions;
811  archiveExtensions.insert("vuonode");
812  set<VuoFileUtilities::File *> moduleFiles = VuoFileUtilities::findFilesInDirectory(path, moduleExtensions, archiveExtensions);
813 
814  map<string, ModuleInfo *> fileForModuleKey;
815  for (set<VuoFileUtilities::File *>::iterator i = moduleFiles.begin(); i != moduleFiles.end(); ++i)
816  {
817  VuoFileUtilities::File *moduleFile = *i;
818 
819  ModuleInfo *m = new ModuleInfo(this, path, moduleFile, false, false);
820  fileForModuleKey[m->getModuleKey()] = m;
821  }
822 
823  if (shouldCleanModuleCache && path == getCompiledModuleCachePath())
824  {
825  for (map<string, ModuleInfo *>::iterator i = fileForModuleKey.begin(); i != fileForModuleKey.end(); )
826  {
827  ModuleInfo *sourceInfo = listSourceFile(i->first);
828  if (! sourceInfo || sourceInfo->isNewerThan(i->second))
829  {
830  ModuleInfo *m = i->second;
831  VuoFileUtilities::deleteFile(m->getFile()->path());
832  delete m;
833  fileForModuleKey.erase(i++);
834  }
835  else
836  ++i;
837  }
838  }
839 
840  moduleFilesAtSearchPath[path] = fileForModuleKey;
841 }
842 
850 void VuoCompiler::Environment::updateSourceFilesAtSearchPath(const string &path)
851 {
852  map<string, map<string, ModuleInfo *> >::iterator sourceFilesIter = sourceFilesAtSearchPath.find(path);
853  if (sourceFilesIter != sourceFilesAtSearchPath.end())
854  return;
855 
856  set<VuoFileUtilities::File *> sourceFiles;
857  if (! path.empty())
858  {
859  set<string> sourceExtensions;
860  sourceExtensions.insert("vuo");
861  sourceExtensions.insert("fs");
863  sourceExtensions.insert(cext.begin(), cext.end());
864  sourceFiles = VuoFileUtilities::findFilesInDirectory(path, sourceExtensions, set<string>());
865  }
866  else
867  {
868  for (const string &sourcePath : expatriateSourceFiles)
869  {
870  string dir, file, ext;
871  VuoFileUtilities::splitPath(sourcePath, dir, file, ext);
872  VuoFileUtilities::File *sourceFile = new VuoFileUtilities::File(dir, file + "." + ext);
873  sourceFiles.insert(sourceFile);
874  }
875  }
876 
877  map<string, ModuleInfo *> fileForModuleKey;
878  for (set<VuoFileUtilities::File *>::iterator i = sourceFiles.begin(); i != sourceFiles.end(); ++i)
879  {
880  VuoFileUtilities::File *sourceFile = *i;
881 
882  string dir, moduleKey, ext;
883  VuoFileUtilities::splitPath(sourceFile->getRelativePath(), dir, moduleKey, ext);
884  bool isSubcomposition = (ext == "vuo");
885 
886  // Ignore missing expatriateSourceFiles — they might have been deleted in the meantime.
887  if (path.empty() && !sourceFile->exists())
888  continue;
889 
890  ModuleInfo *m = new ModuleInfo(this, path, sourceFile, true, isSubcomposition);
891  if (fileForModuleKey.find(m->getModuleKey()) != fileForModuleKey.end())
892  VUserLog("Warning: Conflicting source files for module %s are installed at %s", m->getModuleKey().c_str(), path.c_str());
893  fileForModuleKey[m->getModuleKey()] = m;
894 
895  if (isSubcomposition)
896  {
897  DependencyGraphVertex *compositionVertex = DependencyGraphVertex::vertexForDependency(moduleKey, compositionDependencyGraph);
898  compositionDependencyGraph->addVertex(compositionVertex);
899 
900  compositionVertex->setEnvironment(this);
901 
902  set<string> dependencies = m->getContainedNodeClasses();
903  for (set<string>::iterator j = dependencies.begin(); j != dependencies.end(); ++j)
904  {
905  DependencyGraphVertex *dependencyVertex = DependencyGraphVertex::vertexForDependency(*j, compositionDependencyGraph);
906  compositionDependencyGraph->addEdge(compositionVertex, dependencyVertex);
907  }
908  }
909  }
910 
911  sourceFilesAtSearchPath[path] = fileForModuleKey;
912 }
913 
919 VuoCompiler::ModuleInfo * VuoCompiler::Environment::listModule(const string &moduleKey)
920 {
921  for (const auto &moduleFiles : moduleFilesAtSearchPath)
922  {
923  map<string, ModuleInfo *>::const_iterator foundIter = moduleFiles.second.find(moduleKey);
924  if (foundIter != moduleFiles.second.end())
925  return foundIter->second;
926  }
927 
928  return NULL;
929 }
930 
936 VuoCompiler::ModuleInfoIterator VuoCompiler::Environment::listModules(const set<string> &moduleKeys)
937 {
938  for (const string &path : moduleSearchPaths)
939  updateModulesAtSearchPath(path);
940 
941  return ModuleInfoIterator(&moduleFilesAtSearchPath, moduleKeys);
942 }
943 
949 VuoCompiler::ModuleInfoIterator VuoCompiler::Environment::listAllModules(void)
950 {
951  for (const string &path : moduleSearchPaths)
952  updateModulesAtSearchPath(path);
953 
954  return ModuleInfoIterator(&moduleFilesAtSearchPath);
955 }
956 
962 VuoCompiler::ModuleInfo * VuoCompiler::Environment::listSourceFile(const string &moduleKey)
963 {
964  for (const auto &sourceFiles : sourceFilesAtSearchPath)
965  {
966  map<string, ModuleInfo *>::const_iterator foundIter = sourceFiles.second.find(moduleKey);
967  if (foundIter != sourceFiles.second.end())
968  return foundIter->second;
969  }
970 
971  return NULL;
972 }
973 
979 VuoCompiler::ModuleInfoIterator VuoCompiler::Environment::listSourceFiles(const set<string> &moduleKeys)
980 {
981  for (const string &path : moduleSearchPaths)
982  updateSourceFilesAtSearchPath(path);
983 
984  return ModuleInfoIterator(&sourceFilesAtSearchPath, moduleKeys);
985 }
986 
992 VuoCompiler::ModuleInfoIterator VuoCompiler::Environment::listAllSourceFiles(void)
993 {
994  for (const string &path : moduleSearchPaths)
995  updateSourceFilesAtSearchPath(path);
996 
997  return ModuleInfoIterator(&sourceFilesAtSearchPath);
998 }
999 
1003 vector<string> VuoCompiler::Environment::getBuiltInModuleSearchPaths(void)
1004 {
1005  vector<string> builtInModuleSearchPaths;
1006 
1007  string vuoFrameworkPath = getVuoFrameworkPath();
1008  if (! vuoFrameworkPath.empty())
1009  {
1010  builtInModuleSearchPaths.push_back(vuoFrameworkPath + "/Modules");
1011  }
1012  else
1013  {
1014  builtInModuleSearchPaths.push_back(VUO_BUILD_DIR "/library");
1015  builtInModuleSearchPaths.push_back(VUO_BUILD_DIR "/node");
1016  builtInModuleSearchPaths.push_back(VUO_BUILD_DIR "/type");
1017  builtInModuleSearchPaths.push_back(VUO_BUILD_DIR "/type/list");
1018  }
1019 
1020  return builtInModuleSearchPaths;
1021 }
1022 
1026 vector<string> VuoCompiler::Environment::getBuiltInHeaderSearchPaths(void)
1027 {
1028  vector<string> builtInHeaderSearchPaths;
1029 
1030  string vuoFrameworkPath = getVuoFrameworkPath();
1031  if (! vuoFrameworkPath.empty())
1032  {
1033  builtInHeaderSearchPaths.push_back(vuoFrameworkPath + "/Headers");
1034  builtInHeaderSearchPaths.push_back(vuoFrameworkPath + "/Headers/macos"); // system headers installed by Xcode Command Line Tools
1035  builtInHeaderSearchPaths.push_back(vuoFrameworkPath + "/Frameworks/llvm.framework/Versions/A/Headers/lib/c++/v1");
1036  }
1037  else
1038  {
1039  builtInHeaderSearchPaths.push_back(VUO_SOURCE_DIR "/library");
1040  builtInHeaderSearchPaths.push_back(VUO_SOURCE_DIR "/node");
1041  builtInHeaderSearchPaths.push_back(VUO_SOURCE_DIR "/type");
1042  builtInHeaderSearchPaths.push_back(VUO_SOURCE_DIR "/type/list");
1043  builtInHeaderSearchPaths.push_back(VUO_SOURCE_DIR "/runtime");
1044  builtInHeaderSearchPaths.push_back(JSONC_ROOT "/include");
1045  builtInHeaderSearchPaths.push_back(LLVM_ROOT "/include/c++/v1");
1046  }
1047 
1048  return builtInHeaderSearchPaths;
1049 }
1050 
1054 vector<string> VuoCompiler::Environment::getBuiltInLibrarySearchPaths(void)
1055 {
1056  vector<string> builtInLibrarySearchPaths;
1057 
1058  string vuoFrameworkPath = getVuoFrameworkPath();
1059  if (! vuoFrameworkPath.empty())
1060  {
1061  builtInLibrarySearchPaths.push_back(vuoFrameworkPath + "/Modules");
1062 
1063  // Ensure we (statically) link to our OpenSSL build when generating Vuo.framework's built-in module cache.
1064  builtInLibrarySearchPaths.push_back(OPENSSL_ROOT "/lib");
1065  }
1066  else
1067  {
1068  builtInLibrarySearchPaths.push_back(VUO_BUILD_DIR "/library");
1069  builtInLibrarySearchPaths.push_back(VUO_BUILD_DIR "/node");
1070  builtInLibrarySearchPaths.push_back(VUO_BUILD_DIR "/type");
1071  builtInLibrarySearchPaths.push_back(VUO_BUILD_DIR "/type/list");
1072  builtInLibrarySearchPaths.push_back(VUO_BUILD_DIR "/runtime");
1073 
1074  builtInLibrarySearchPaths.push_back(LEAP_ROOT);
1075 
1076  vector<string> conanLibDirs = VuoStringUtilities::split(CONAN_LIBRARY_PATHS, ';');
1077  builtInLibrarySearchPaths.insert(builtInLibrarySearchPaths.end(), conanLibDirs.begin(), conanLibDirs.end());
1078  }
1079 
1080  return builtInLibrarySearchPaths;
1081 }
1082 
1086 vector<string> VuoCompiler::Environment::getBuiltInFrameworkSearchPaths(void)
1087 {
1088  vector<string> builtInFrameworkSearchPaths;
1089 
1090  string vuoFrameworkPath = getVuoFrameworkPath();
1091  if (! vuoFrameworkPath.empty())
1092  {
1093  builtInFrameworkSearchPaths.push_back(vuoFrameworkPath + "/Modules/");
1094  builtInFrameworkSearchPaths.push_back(vuoFrameworkPath + "/Frameworks/");
1095  }
1096  else
1097  {
1098  builtInFrameworkSearchPaths.push_back(SYPHON_ROOT);
1099  }
1100 
1101  return builtInFrameworkSearchPaths;
1102 }
1103 
1110 void VuoCompiler::Environment::startWatchingModuleSearchPath(const string &moduleSearchPath)
1111 {
1112  VuoFileWatcher *watcher = new VuoFileWatcher(this, moduleSearchPath);
1113  moduleSearchPathWatchers.insert(watcher);
1114 }
1115 
1121 void VuoCompiler::Environment::stopWatchingModuleSearchPaths(void)
1122 {
1123  for (set<VuoFileWatcher *>::iterator i = moduleSearchPathWatchers.begin(); i != moduleSearchPathWatchers.end(); ++i)
1124  delete *i;
1125 
1126  moduleSearchPathWatchers.clear();
1127 }
1128 
1133 void VuoCompiler::Environment::fileChanged(const string &moduleSearchPath)
1134 {
1135  dispatch_sync(moduleSearchPathContentsChangedQueue, ^{
1136  moduleSearchPathContentsChanged(moduleSearchPath);
1137  });
1138 }
1139 
1147 void VuoCompiler::Environment::moduleSearchPathContentsChanged(const string &moduleSearchPath, const string &moduleAddedOrModifiedPath,
1148  const string &moduleAddedOrModifiedSourceCode,
1149  std::function<void(void)> moduleLoadedCallback,
1150  VuoCompiler *compiler, VuoCompilerIssues *issues)
1151 {
1152  //VLog(" E=%p -- %s", this, moduleSearchPath.c_str());
1153  VuoCompiler *compilerForLoading = (compiler ? compiler : new VuoCompiler(moduleSearchPath + "/unused"));
1154 
1155  __block set<string> modulesAdded;
1156  __block set<string> modulesModifed;
1157  __block set<string> modulesRemoved;
1158  __block set<string> compositionsAdded;
1159  __block set<string> compositionsModifed;
1160  __block set<string> compositionsRemoved;
1161 
1162  dispatch_sync(environmentQueue, ^{
1163 
1164  // Remove the old file records from the environment.
1165 
1166  map<string, ModuleInfo *> oldModules;
1167  map<string, ModuleInfo *> oldCompositions;
1168 
1169  map<string, map<string, ModuleInfo *> >::iterator mf = moduleFilesAtSearchPath.find(moduleSearchPath);
1170  if (mf != moduleFilesAtSearchPath.end())
1171  {
1172  for (map<string, ModuleInfo *>::iterator j = mf->second.begin(); j != mf->second.end(); ++j)
1173  {
1174  oldModules[j->first] = j->second;
1175  }
1176 
1177  moduleFilesAtSearchPath.erase(mf);
1178  }
1179 
1180  map<string, map<string, ModuleInfo *> >::iterator cf = sourceFilesAtSearchPath.find(moduleSearchPath);
1181  if (cf != sourceFilesAtSearchPath.end())
1182  {
1183  for (map<string, ModuleInfo *>::iterator j = cf->second.begin(); j != cf->second.end(); ++j)
1184  {
1185  oldCompositions[j->first] = j->second;
1186 
1187  VuoDirectedAcyclicGraph::Vertex *vertex = compositionDependencyGraph->findVertex(j->first);
1188  compositionDependencyGraph->removeVertex(vertex);
1189  }
1190 
1191  sourceFilesAtSearchPath.erase(cf);
1192  }
1193 
1194  // Compare the old and new file records to see what has changed.
1195 
1196  updateModulesAtSearchPath(moduleSearchPath, moduleAddedOrModifiedPath.empty());
1197  updateSourceFilesAtSearchPath(moduleSearchPath);
1198 
1199  mf = moduleFilesAtSearchPath.find(moduleSearchPath);
1200  if (mf != moduleFilesAtSearchPath.end())
1201  {
1202  for (map<string, ModuleInfo *>::iterator n = mf->second.begin(); n != mf->second.end(); ++n)
1203  {
1204  string moduleKey = n->first;
1205 
1206  map<string, ModuleInfo *>::iterator o = oldModules.find(moduleKey);
1207  if (o != oldModules.end())
1208  {
1209  if (n->second->isNewerThan(o->second) ||
1210  (n->second->getFile() && ! n->second->getFile()->isInArchive() && VuoFileUtilities::arePathsEqual(n->second->getFile()->path(), moduleAddedOrModifiedPath)))
1211  {
1212  modulesModifed.insert(moduleKey);
1213  }
1214  else
1215  {
1216  n->second->setAttempted(o->second->getAttempted());
1217  }
1218 
1219  delete o->second;
1220  oldModules.erase(o);
1221  }
1222  else if (VuoFileUtilities::arePathsEqual(moduleSearchPath, getOverriddenCompiledModuleCachePath()))
1223  {
1224  modulesModifed.insert(moduleKey);
1225  }
1226  else
1227  {
1228  modulesAdded.insert(moduleKey);
1229  }
1230  }
1231  }
1232 
1233  cf = sourceFilesAtSearchPath.find(moduleSearchPath);
1234  if (cf != sourceFilesAtSearchPath.end())
1235  {
1236  for (map<string, ModuleInfo *>::iterator n = cf->second.begin(); n != cf->second.end(); ++n)
1237  {
1238  string moduleKey = n->first;
1239 
1240  map<string, ModuleInfo *>::iterator o = oldCompositions.find(moduleKey);
1241  if (o != oldCompositions.end())
1242  {
1243  if (n->second->isNewerThan(o->second))
1244  {
1245  compositionsModifed.insert(moduleKey);
1246  }
1247  else
1248  {
1249  n->second->setAttempted(o->second->getAttempted());
1250  n->second->setSourceCode(o->second->getSourceCode());
1251  }
1252 
1253  delete o->second;
1254  oldCompositions.erase(o);
1255  }
1256  else
1257  {
1258  compositionsAdded.insert(moduleKey);
1259  }
1260  }
1261  }
1262 
1263  for (map<string, ModuleInfo *>::iterator o = oldModules.begin(); o != oldModules.end(); ++o)
1264  {
1265  delete o->second;
1266  modulesRemoved.insert(o->first);
1267  }
1268 
1269  for (map<string, ModuleInfo *>::iterator o = oldCompositions.begin(); o != oldCompositions.end(); ++o)
1270  {
1271  delete o->second;
1272  compositionsRemoved.insert(o->first);
1273  }
1274 
1275  compilerForLoading->loadModulesAndSources(modulesAdded, modulesModifed, modulesRemoved,
1276  compositionsAdded, compositionsModifed, compositionsRemoved,
1277  false, false, this, issues, moduleLoadedCallback, moduleAddedOrModifiedSourceCode);
1278  });
1279 
1280  if (! compiler)
1281  delete compilerForLoading;
1282 }
1283 
1287 void VuoCompiler::Environment::notifyCompilers(const set<VuoCompilerModule *> &modulesAdded,
1288  const set< pair<VuoCompilerModule *, VuoCompilerModule *> > &modulesModified,
1289  const set<VuoCompilerModule *> &modulesRemoved, VuoCompilerIssues *issues,
1290  bool oldModulesInvalidated)
1291 {
1292  if (! (modulesAdded.empty() && modulesModified.empty() && modulesRemoved.empty() && issues->isEmpty()) )
1293  {
1294  set< pair<VuoCompilerModule *, VuoCompilerModule *> > invalidatedModulesModified;
1295  set<VuoCompilerModule *> invalidatedModulesRemoved;
1296  if (oldModulesInvalidated)
1297  {
1298  invalidatedModulesModified = modulesModified;
1299  invalidatedModulesRemoved = modulesRemoved;
1300  }
1301 
1302  VuoCompilerDelegate::LoadedModulesData *delegateData = new VuoCompilerDelegate::LoadedModulesData(invalidatedModulesModified, invalidatedModulesRemoved, issues);
1303 
1304  map<string, VuoCompilerModule *> modulesAddedMap;
1305  map<string, pair<VuoCompilerModule *, VuoCompilerModule *> > modulesModifiedMap;
1306  map<string, VuoCompilerModule *> modulesRemovedMap;
1307  for (VuoCompilerModule *m : modulesAdded)
1308  modulesAddedMap[m->getPseudoBase()->getModuleKey()] = m;
1309  for (pair<VuoCompilerModule *, VuoCompilerModule *> m : modulesModified)
1310  modulesModifiedMap[m.first->getPseudoBase()->getModuleKey()] = m;
1311  for (VuoCompilerModule *m : modulesRemoved)
1312  modulesRemovedMap[m->getPseudoBase()->getModuleKey()] = m;
1313 
1314  dispatch_sync(compilersToNotifyQueue, ^{
1315  for (set<VuoCompiler *>::iterator i = compilersToNotify.begin(); i != compilersToNotify.end(); ++i) {
1316  delegateData->retain();
1317  }
1318  for (set<VuoCompiler *>::iterator i = compilersToNotify.begin(); i != compilersToNotify.end(); ++i) {
1319  (*i)->loadedModules(modulesAddedMap, modulesModifiedMap, modulesRemovedMap, issues, delegateData, this);
1320  }
1321  });
1322  }
1323  else
1324  {
1325  delete issues;
1326  }
1327 }
1328 
1339 set<VuoCompilerModule *> VuoCompiler::Environment::loadCompiledModules(const set<string> &moduleKeys, const map<string, string> &sourceCodeForModule)
1340 {
1341  ModuleInfoIterator modulesToLoadIter = listModules(moduleKeys);
1342  set<VuoCompilerModule *> modulesLoaded;
1343 
1344  ModuleInfo *moduleInfo;
1345  while ((moduleInfo = modulesToLoadIter.next()))
1346  {
1347  string moduleKey = moduleInfo->getModuleKey();
1348 
1349  // Skip the module if its file is not in this environment, if the module has already been loaded,
1350  // or if the compiler previously tried to load the module and it failed.
1351  if (moduleInfo->getEnvironment() != this || findModule(moduleKey) || moduleInfo->getAttempted())
1352  continue;
1353 
1354  moduleInfo->setAttempted(true);
1355 
1356  // Actually load the module.
1357  VuoCompilerModule *module = loadModule(moduleInfo);
1358 
1359  if (module)
1360  {
1361  modulesLoaded.insert(module);
1362  addToDependencyGraph(module);
1363  modulesChanged();
1364 
1365  // For a compiled subcomposition or other source file, store its source code in the VuoCompilerNodeClass.
1366  string searchPath = moduleInfo->getSearchPath();
1367  if (VuoFileUtilities::arePathsEqual(searchPath, getCompiledModuleCachePath()) ||
1368  VuoFileUtilities::arePathsEqual(searchPath, getOverriddenCompiledModuleCachePath()))
1369  {
1370  VuoCompilerNodeClass *nodeClass = dynamic_cast<VuoCompilerNodeClass *>(module);
1371  if (nodeClass)
1372  {
1373  ModuleInfo *sourceInfo = listSourceFile(moduleKey);
1374  if (sourceInfo)
1375  nodeClass->setSourcePath( sourceInfo->getFile()->path() );
1376 
1377  auto sourceCodeIter = sourceCodeForModule.find(moduleKey);
1378  if (sourceCodeIter != sourceCodeForModule.end())
1379  nodeClass->setSourceCode( sourceCodeIter->second );
1380  }
1381  }
1382  }
1383  }
1384 
1385  return modulesLoaded;
1386 }
1387 
1399 set<dispatch_group_t> VuoCompiler::Environment::loadSpecializedModules(const set<string> &moduleKeys,
1400  VuoCompiler *compiler, dispatch_queue_t llvmQueue)
1401 {
1402  set<dispatch_group_t > loadingGroups;
1403 
1404  for (string moduleKey : moduleKeys)
1405  {
1406  // Skip if it's not a node class.
1407 
1408  if (moduleKey.find(".") == string::npos || VuoStringUtilities::endsWith(moduleKey, ".framework"))
1409  continue;
1410 
1411  // Skip the module if it's already been loaded.
1412 
1413  if (findModule(moduleKey))
1414  continue;
1415 
1416  // Skip the module if it's in the process of being loaded, but have the caller wait for it to finish loading.
1417 
1418  map<string, dispatch_group_t>::iterator iter = specializedModulesLoading.find(moduleKey);
1419  if (iter != specializedModulesLoading.end())
1420  {
1421  loadingGroups.insert(iter->second);
1422  dispatch_retain(iter->second);
1423  continue;
1424  }
1425 
1426  dispatch_group_t loadingGroup = dispatch_group_create();
1427  specializedModulesLoading[moduleKey] = loadingGroup;
1428  loadingGroups.insert(loadingGroup);
1429 
1430  // Generate the module.
1431  // This is done asynchronously since VuoCompilerSpecializedNodeClass::newNodeClass() needs to use environmentQueue
1432  // to compile the generated C code.
1433 
1434  dispatch_queue_t queue = dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0);
1435  dispatch_group_async(loadingGroup, queue, ^{
1436  VuoNodeClass *baseNodeClass = nullptr;
1437  try
1438  {
1439  baseNodeClass = VuoCompilerSpecializedNodeClass::newNodeClass(moduleKey, compiler, llvmQueue);
1440  }
1441  catch (VuoCompilerException &e)
1442  {
1443  VUserLog("Error: %s", e.what());
1444  }
1445 
1446  if (baseNodeClass)
1447  {
1448  VuoCompilerSpecializedNodeClass *specNodeClass = static_cast<VuoCompilerSpecializedNodeClass *>(baseNodeClass->getCompiler());
1449  compiler->loadNodeClassGeneratedAtRuntime(specNodeClass, this);
1450 
1451  set<string> dependencies = specNodeClass->getDependencies();
1452  compiler->loadModulesIfNeeded(dependencies);
1453  }
1454 
1455  dispatch_sync(environmentQueue, ^{
1456  specializedModulesLoading.erase(moduleKey);
1457  });
1458  });
1459  }
1460 
1461  return loadingGroups;
1462 }
1463 
1476 set<dispatch_group_t> VuoCompiler::Environment::compileModulesFromSourceCode(const set<string> &moduleKeys, bool shouldRecompileIfUnchanged)
1477 {
1478  ModuleInfoIterator modulesToLoadIter = listSourceFiles(moduleKeys);
1479 
1480  int environmentIndex = sharedEnvironments.size();
1481  for (int i = 0; i < sharedEnvironments.size(); ++i)
1482  if (this == sharedEnvironments.at(i).at(0))
1483  environmentIndex = i;
1484 
1485  set<dispatch_group_t> sourcesLoading;
1486  int sourcesEnqueued = 0;
1487  ModuleInfo *sourceInfo;
1488  while ((sourceInfo = modulesToLoadIter.next()))
1489  {
1490  string moduleKey = sourceInfo->getModuleKey();
1491 
1492  dispatch_group_t loadingGroup = sourceInfo->getLoadingGroup();
1493  sourcesLoading.insert(loadingGroup);
1494 
1495  // Skip compiling if the source file has already been scheduled for compilation.
1496  // Either its compilation is in progress or it failed to compile.
1497 
1498  if (sourceInfo->getAttempted())
1499  continue;
1500 
1501  // Skip compiling if the compiled module is up-to-date.
1502 
1503  string sourceCode = sourceInfo->getSourceCode();
1504 
1505  if (! shouldRecompileIfUnchanged)
1506  {
1507  VuoCompilerNodeClass *existingNodeClass = getNodeClass(moduleKey);
1508  if (existingNodeClass && existingNodeClass->getSourceCode() == sourceCode)
1509  continue;
1510  }
1511 
1512  // Enqueue the source file to be compiled.
1513 
1514  sourceInfo->setAttempted(true);
1515 
1516  dispatch_group_enter(loadingGroup);
1517  dispatch_group_enter(moduleSourceCompilersExist);
1518 
1520  queueItem->moduleKey = moduleKey;
1521  queueItem->sourcePath = sourceInfo->getFile()->path();
1522  queueItem->sourceCode = sourceCode;
1523  queueItem->sourceFile = sourceInfo->getFile();
1524  queueItem->cachedModulesPath = sourceInfo->isSourceCodeOverridden() ? getOverriddenCompiledModuleCachePath() : getCompiledModuleCachePath();
1525  queueItem->compiledModulePath = queueItem->cachedModulesPath + "/" + moduleKey + ".vuonode";
1526  queueItem->loadingGroup = loadingGroup;
1527  queueItem->priority = { environmentIndex, sourceInfo->getLongestDownstreamPath() };
1528  moduleCompilationQueue->enqueue(queueItem);
1529  ++sourcesEnqueued;
1530  }
1531 
1532  // Compile each enqueued source file. This is done asynchronously for several reasons:
1533  // - To avoid environmentQueue calling compiler code calling environmentQueue.
1534  // - To compile dependencies that were enqueued later while a compilation that was enqueued earlier waits.
1535  // - To be more efficient when compiling multiple source files.
1536 
1537  dispatch_queue_t queue = dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0);
1538  dispatch_async(queue, ^{
1539  for (int i = 0; i < sourcesEnqueued; ++i)
1540  {
1541  __block VuoModuleCompilationQueue::Item *queueItem = moduleCompilationQueue->next();
1542  VUserLog("Compiling %s", queueItem->moduleKey.c_str());
1543 
1544  dispatch_async(queue, ^{
1545  try
1546  {
1547  VuoFileUtilities::makeDir(queueItem->cachedModulesPath);
1548  }
1549  catch (VuoException &e)
1550  {
1551  VuoCompilerIssue issue(VuoCompilerIssue::Error, "compiling subcomposition", "", "Couldn't create cached modules folder", e.what());
1552  VuoCompilerIssues *issues = new VuoCompilerIssues(issue);
1553  notifyCompilers(set<VuoCompilerModule *>(), set< pair<VuoCompilerModule *, VuoCompilerModule *> >(), set<VuoCompilerModule *>(), issues, false);
1554  }
1555 
1556  VuoCompiler *compiler = new VuoCompiler(queueItem->sourcePath);
1557 
1558  VuoModuleCompilationQueue::Item *queueItemForCallback = queueItem;
1559  auto moduleLoadedCallback = [=](){
1560  dispatch_group_leave(queueItemForCallback->loadingGroup);
1561  moduleCompilationQueue->completed(queueItemForCallback);
1562  };
1563 
1564  string ext = queueItem->sourceFile->extension();
1566  {
1567  VuoModuleCompiler *moduleCompiler = NULL;
1568  try
1569  {
1570  moduleCompiler = VuoModuleCompiler::newModuleCompiler("isf", queueItem->moduleKey, queueItem->sourceFile);
1571  }
1572  catch (VuoException &e)
1573  {
1574  VuoCompilerIssues *issues = new VuoCompilerIssues(VuoCompilerIssue(VuoCompilerIssue::Error, "compiling ISF module", queueItem->sourcePath, "", e.what()));
1575  notifyCompilers(set<VuoCompilerModule *>(), set< pair<VuoCompilerModule *, VuoCompilerModule *> >(), set<VuoCompilerModule *>(), issues, false);
1576  }
1577 
1578  if (moduleCompiler)
1579  {
1580  moduleCompiler->overrideSourceCode(queueItem->sourceCode, queueItem->sourceFile);
1581 
1582  auto getType = [&compiler] (const string &moduleKey) { return compiler->getType(moduleKey); };
1583  VuoCompilerIssues *issues = new VuoCompilerIssues();
1584  Module *module = moduleCompiler->compile(getType, llvmQueue, issues);
1585 
1586  if (module)
1587  {
1588  dispatch_sync(llvmQueue, ^{
1589  //setTargetForModule(module, target);
1590  writeModuleToBitcode(module, queueItem->compiledModulePath);
1591  });
1592  }
1593  else
1594  {
1595  issues->setFilePathIfEmpty(queueItem->sourcePath);
1596  }
1597 
1598  moduleSearchPathContentsChanged(queueItem->cachedModulesPath, queueItem->compiledModulePath, queueItem->sourceCode, moduleLoadedCallback, compiler, issues);
1599  }
1600  else
1601  moduleLoadedCallback();
1602  }
1603  else if (VuoFileUtilities::isCSourceExtension(ext) && ! queueItem->cachedModulesPath.empty())
1604  {
1605  try
1606  {
1607  compiler->compileModule(queueItem->sourcePath, queueItem->compiledModulePath, vector<string>());
1608  moduleSearchPathContentsChanged(queueItem->cachedModulesPath, queueItem->compiledModulePath, queueItem->sourceCode, moduleLoadedCallback, compiler);
1609  }
1610  catch (VuoCompilerException &e)
1611  {
1612  VuoCompilerIssues *issues = new VuoCompilerIssues;
1613  for (auto issue : e.getIssues()->getList())
1614  issues->append(issue);
1615  notifyCompilers(set<VuoCompilerModule *>(), set< pair<VuoCompilerModule *, VuoCompilerModule *> >(), set<VuoCompilerModule *>(), issues, false);
1616  moduleLoadedCallback();
1617  }
1618  }
1619  else
1620  {
1621  compiler->setLoadAllModules(false);
1622  compiler->compileSubcompositionString(queueItem->sourceCode, queueItem->compiledModulePath, moduleLoadedCallback, this, NULL, queueItem->sourcePath);
1623  }
1624 
1625  delete compiler;
1626  dispatch_group_leave(moduleSourceCompilersExist);
1627  });
1628  }
1629  });
1630 
1631  return sourcesLoading;
1632 }
1633 
1642 set<VuoCompilerModule *> VuoCompiler::Environment::unloadCompiledModules(const set<string> &moduleKeys)
1643 {
1644  set<VuoCompilerModule *> modulesUnloaded;
1645 
1646  for (set<string>::const_iterator i = moduleKeys.begin(); i != moduleKeys.end(); ++i)
1647  {
1648  string moduleKey = *i;
1649  VuoCompilerModule *module = NULL;
1650 
1651  map<string, VuoCompilerNodeClass *>::iterator nodeClassIter = nodeClasses.find(moduleKey);
1652  if (nodeClassIter != nodeClasses.end())
1653  {
1654  module = nodeClassIter->second;
1655  nodeClasses.erase(nodeClassIter);
1656  }
1657  else
1658  {
1659  map<string, VuoCompilerType *>::iterator typeIter = types.find(moduleKey);
1660  if (typeIter != types.end())
1661  {
1662  module = typeIter->second;
1663  types.erase(typeIter);
1664  }
1665  else
1666  {
1667  map<string, VuoCompilerModule *>::iterator libraryModuleIter = libraryModules.find(moduleKey);
1668  if (libraryModuleIter != libraryModules.end())
1669  {
1670  module = libraryModuleIter->second;
1671  libraryModules.erase(libraryModuleIter);
1672  }
1673  }
1674  }
1675 
1676  if (module)
1677  {
1678  modulesUnloaded.insert(module);
1679  removeFromDependencyGraph(module);
1680  modulesChanged();
1681  }
1682  }
1683 
1684  return modulesUnloaded;
1685 }
1686 
1692 void VuoCompiler::Environment::deleteModulesCompiledFromSourceCode(const set<string> &moduleKeys)
1693 {
1694  for (set<string>::const_iterator i = moduleKeys.begin(); i != moduleKeys.end(); ++i)
1695  {
1696  string moduleKey = *i;
1697 
1698  map<string, ModuleInfo *>::iterator iter = moduleFilesAtSearchPath[getCompiledModuleCachePath()].find(moduleKey);
1699  if (iter != moduleFilesAtSearchPath[getCompiledModuleCachePath()].end())
1700  VuoFileUtilities::deleteFile(iter->second->getFile()->path());
1701  }
1702 }
1703 
1711 VuoCompilerModule * VuoCompiler::Environment::loadModule(ModuleInfo *moduleInfo)
1712 {
1713  string moduleKey = moduleInfo->getModuleKey();
1714 
1715  // Skip certain LLVM modules that definitely aren't Vuo modules to avoid adding struct types defined in them to the LLVM
1716  // context, resulting in mismatched struct types in code generation (e.g. %struct.NodeContext and %struct.NodeContext.1).
1717  if (VuoStringUtilities::beginsWith(moduleKey, "libVuo"))
1718  return NULL;
1719 
1720  __block size_t inputDataBytes;
1721  __block char *rawInputData;
1722  dispatch_sync(llvmQueue, ^{
1723  try
1724  {
1725  rawInputData = moduleInfo->getFile()->getContentsAsRawData(inputDataBytes);
1726  }
1727  catch (VuoException &e)
1728  {
1729  rawInputData = NULL;
1730  VUserLog("Warning: Couldn't load module '%s'. Its file may have been deleted. (%s)", moduleKey.c_str(), e.what());
1731  }
1732  });
1733  if (! rawInputData)
1734  return NULL;
1735 
1736  char *processedInputData;
1737 #if VUO_PRO
1738  processedInputData = loadModule_Pro0(moduleInfo, moduleKey, inputDataBytes, rawInputData);
1739 #else
1740  processedInputData = rawInputData;
1741 #endif
1742 
1743  Module *module = NULL;
1744  bool moduleParseError = !processedInputData;
1745  if (!moduleParseError)
1746  {
1747  string moduleReadError;
1748  VuoLog_status("Loading module \"%s\"", moduleKey.c_str());
1749  module = readModuleFromBitcodeData(processedInputData, inputDataBytes, moduleReadError);
1750  VuoLog_status(NULL);
1751  free(processedInputData);
1752 
1753  if (!module)
1754  {
1755  moduleParseError = true;
1756 
1757  string dir, file, ext;
1758  VuoFileUtilities::splitPath(moduleInfo->getFile()->isInArchive() ? moduleInfo->getFile()->getRelativePath() : moduleInfo->getFile()->path(), dir, file, ext);
1760  if (dir == getCompiledModuleCachePath())
1761  VuoFileUtilities::deleteFile(moduleInfo->getFile()->path());
1762  else
1763  VUserLog("Error: Couldn't parse module '%s': %s.", moduleInfo->getFile()->getRelativePath().c_str(), moduleReadError.c_str());
1764  }
1765  }
1766 
1767  if (!moduleParseError)
1768  {
1769  string modulePath;
1770  if (! moduleInfo->getFile()->isInArchive())
1771  modulePath = moduleInfo->getFile()->path();
1772 
1773  __block VuoCompilerModule *compilerModule;
1774  dispatch_sync(llvmQueue, ^{
1775  compilerModule = VuoCompilerModule::newModule(moduleKey, module, modulePath);
1776  });
1777 
1778  if (compilerModule)
1779  {
1780  if (dynamic_cast<VuoCompilerNodeClass *>(compilerModule))
1781  nodeClasses[moduleKey] = static_cast<VuoCompilerNodeClass *>(compilerModule);
1782  else if (dynamic_cast<VuoCompilerType *>(compilerModule))
1783  types[moduleKey] = static_cast<VuoCompilerType *>(compilerModule);
1784  else
1785  libraryModules[moduleKey] = compilerModule;
1786 
1787  VuoNodeSet *nodeSet = VuoNodeSet::createNodeSetForModule(moduleInfo->getFile());
1788  if (nodeSet)
1789  {
1790  map<string, VuoNodeSet *>::iterator nodeSetIter = nodeSetForName.find(nodeSet->getName());
1791  if (nodeSetIter == nodeSetForName.end())
1792  {
1793  nodeSetForName[nodeSet->getName()] = nodeSet;
1794  }
1795  else
1796  {
1797  delete nodeSet;
1798  nodeSet = nodeSetIter->second;
1799  }
1800  compilerModule->getPseudoBase()->setNodeSet(nodeSet);
1801  }
1802 
1803 #if VUO_PRO
1804  loadModule_Pro1(rawInputData, processedInputData, compilerModule);
1805 #endif
1806 
1807  compilerModule->setBuiltIn( isBuiltIn() );
1808 
1809  return compilerModule;
1810  }
1811  else
1812  {
1813  destroyLlvmModule(module);
1814  }
1815  }
1816 
1817  return NULL;
1818 }
1819 
1825 void VuoCompiler::Environment::replaceNodeClass(VuoCompilerNodeClass *newNodeClass)
1826 {
1827  string moduleKey = newNodeClass->getBase()->getModuleKey();
1828 
1829  VuoCompilerNodeClass *oldNodeClass = nodeClasses[moduleKey];
1830  if (oldNodeClass)
1831  {
1832  removeFromDependencyGraph(oldNodeClass);
1833  destroyModule(oldNodeClass);
1834  }
1835 
1836  nodeClasses[moduleKey] = newNodeClass;
1837  addToDependencyGraph(newNodeClass);
1838  modulesChanged();
1839 }
1840 
1841 void VuoCompiler::Environment::addToDependencyGraph(VuoCompilerModule *module)
1842 {
1843  DependencyGraphVertex *vertex = DependencyGraphVertex::vertexForDependency(module->getPseudoBase()->getModuleKey(), dependencyGraph);
1844  dependencyGraph->addVertex(vertex);
1845 
1846  vertex->setEnvironment(this);
1847 
1848  VuoCompilerTargetSet compositionTargets;
1849  compositionTargets.restrictToCurrentOperatingSystemVersion();
1850  vertex->setCompatible( module->getCompatibleTargets().isCompatibleWithAllOf(compositionTargets) );
1851 
1852  set<string> dependencies = module->getDependencies();
1853  for (set<string>::iterator i = dependencies.begin(); i != dependencies.end(); ++i)
1854  {
1855  DependencyGraphVertex *depVertex = DependencyGraphVertex::vertexForDependency(*i, dependencyGraph);
1856  dependencyGraph->addEdge(vertex, depVertex);
1857  }
1858 }
1859 
1860 void VuoCompiler::Environment::removeFromDependencyGraph(VuoCompilerModule *module)
1861 {
1862  DependencyGraphVertex *vertex = DependencyGraphVertex::vertexForDependency(module->getPseudoBase()->getModuleKey(), dependencyGraph);
1863  dependencyGraph->removeVertex(vertex);
1864 }
1865 
1876 void VuoCompiler::Environment::reifyPortTypes(const map<string, VuoCompilerType *> &inheritedTypes)
1877 {
1878  map<string, VuoCompilerType *> availableTypes = inheritedTypes;
1879  availableTypes.insert(types.begin(), types.end());
1880 
1881  for (map<string, VuoCompilerNodeClass *>::const_iterator i = nodeClasses.begin(); i != nodeClasses.end(); ++i)
1882  {
1883  VuoNodeClass *nodeClass = i->second->getBase();
1884 
1885  vector<VuoPortClass *> inputPortClasses = nodeClass->getInputPortClasses();
1886  vector<VuoPortClass *> outputPortClasses = nodeClass->getOutputPortClasses();
1887  vector<VuoPortClass *> portClasses;
1888  portClasses.insert(portClasses.end(), inputPortClasses.begin(), inputPortClasses.end());
1889  portClasses.insert(portClasses.end(), outputPortClasses.begin(), outputPortClasses.end());
1890 
1891  for (vector<VuoPortClass *>::iterator j = portClasses.begin(); j != portClasses.end(); ++j)
1892  {
1893  VuoCompilerPortClass *portClass = static_cast<VuoCompilerPortClass *>((*j)->getCompiler());
1894  VuoType *baseType = portClass->getDataVuoType();
1895 
1896  if (baseType && ! baseType->hasCompiler())
1897  {
1898  string typeName = baseType->getModuleKey();
1899  VuoCompilerType *reifiedType = NULL;
1900 
1901  VuoGenericType *genericType = dynamic_cast<VuoGenericType *>(baseType);
1902  if (genericType)
1903  {
1904  reifiedType = VuoCompilerGenericType::newGenericType(genericType, availableTypes);
1905  if (reifiedType) {
1906  genericTypes.insert( static_cast<VuoCompilerGenericType *>(reifiedType) );
1907  }
1908  }
1909  else
1910  {
1911  map<string, VuoCompilerType *>::iterator reifiedTypeIter = availableTypes.find(typeName);
1912  if (reifiedTypeIter != availableTypes.end())
1913  {
1914  delete baseType;
1915  reifiedType = reifiedTypeIter->second;
1916  }
1917  }
1918 
1919  if (reifiedType) {
1920  portClass->setDataVuoType(reifiedType->getBase());
1921  }
1922  }
1923  }
1924 
1925  vector<VuoCompilerTriggerDescription *> triggers = nodeClass->getCompiler()->getTriggerDescriptions();
1926  for (vector<VuoCompilerTriggerDescription *>::iterator j = triggers.begin(); j != triggers.end(); ++j)
1927  {
1928  VuoCompilerTriggerDescription *trigger = *j;
1929  VuoType *baseType = trigger->getDataType();
1930 
1931  if (baseType && ! baseType->hasCompiler())
1932  {
1933  string typeName = baseType->getModuleKey();
1934  map<string, VuoCompilerType *>::iterator reifiedTypeIter = availableTypes.find(typeName);
1935  if (reifiedTypeIter != availableTypes.end())
1936  {
1937  delete baseType;
1938  VuoCompilerType *reifiedType = reifiedTypeIter->second;
1939  trigger->setDataType(reifiedType->getBase());
1940  }
1941  }
1942  }
1943  }
1944 }
1945 
1958 void VuoCompiler::Environment::getCacheableModulesAndDependencies(bool builtIn, bool installed, set<string> &cacheableModulesAndDependencies,
1959  set<string> &dylibsNeededToLinkToThisCache,
1960  set<string> &frameworksNeededToLinkToThisCache)
1961 {
1962  if (isModuleCacheInitialized && ! isModuleCacheableDataDirty)
1963  {
1964  cacheableModulesAndDependencies = moduleCacheContents;
1965  dylibsNeededToLinkToThisCache = moduleCacheDylibs;
1966  frameworksNeededToLinkToThisCache = moduleCacheFrameworks;
1967  return;
1968  }
1969 
1970  VuoCompilerTargetSet compositionTargets;
1971  compositionTargets.restrictToCurrentOperatingSystemVersion();
1972 
1973  // Include all modules…
1974  map<string, VuoCompilerModule *> allModules;
1975  allModules.insert(nodeClasses.begin(), nodeClasses.end());
1976  allModules.insert(types.begin(), types.end());
1977  allModules.insert(libraryModules.begin(), libraryModules.end());
1978 
1979  set<string> dependencies;
1980  for (map<string, VuoCompilerModule *>::iterator i = allModules.begin(); i != allModules.end(); ++i)
1981  {
1982  string moduleKey = i->first;
1983  VuoCompilerModule *module = i->second;
1984 
1985 #if VUO_PRO
1986  // … except Pro modules and modules that depend on them…
1987  if (module->requiresPro())
1988  continue;
1989 #endif
1990 
1991  // … and incompatible modules.
1992  if (! module->getCompatibleTargets().isCompatibleWithAllOf(compositionTargets))
1993  continue;
1994 
1995  cacheableModulesAndDependencies.insert(moduleKey);
1996 
1997  set<string> moduleDependencies = module->getDependencies();
1998  dependencies.insert(moduleDependencies.begin(), moduleDependencies.end());
1999  }
2000 
2001  // For the built-in environment, include Vuo's core dependencies.
2002  if (builtIn && installed)
2003  {
2004  vector<string> coreDependencies = getCoreVuoDependencies();
2005  dependencies.insert(coreDependencies.begin(), coreDependencies.end());
2006  }
2007 
2008  // Include all dependencies of the included module that are located in this environment.
2009  // (All modules are already included, so we only need to handle non-module dependencies.)
2010  for (set<string>::iterator i = dependencies.begin(); i != dependencies.end(); ++i)
2011  {
2012  string dependency = *i;
2013  if (allModules.find(dependency) == allModules.end())
2014  {
2015  if (VuoStringUtilities::endsWith(dependency, ".framework"))
2016  frameworksNeededToLinkToThisCache.insert(dependency);
2017  else
2018  {
2019  string dependencyPath = VuoCompiler::getLibraryPath(dependency, librarySearchPaths);
2020  if (! dependencyPath.empty())
2021  {
2022  if (VuoStringUtilities::endsWith(dependencyPath, ".dylib"))
2023  dylibsNeededToLinkToThisCache.insert(dependencyPath);
2024  else
2025  cacheableModulesAndDependencies.insert(dependency);
2026  }
2027  }
2028  }
2029  }
2030 
2031  moduleCacheSuffix = (installed ? "installed" : "generated");
2032  dylibsNeededToLinkToThisCache.insert(moduleCachePath + "/libVuoModuleCache-" + moduleCacheSuffix + ".dylib");
2033 
2034  moduleCacheDylibs = dylibsNeededToLinkToThisCache;
2035  moduleCacheFrameworks = frameworksNeededToLinkToThisCache;
2036 }
2037 
2057 void VuoCompiler::Environment::useModuleCache(bool shouldUseExistingCache, VuoCompiler *compiler, set<string> cacheableModulesAndDependencies,
2058  set<string> dylibsNeededToLinkToCaches, set<string> frameworksNeededToLinkToCaches)
2059 {
2060  // Ignore the cache if the `prelinkCache` preference is false.
2061 
2062  static dispatch_once_t checked = 0;
2063  static bool prelinkCache = true;
2064  dispatch_once(&checked, ^{
2065  Boolean valid;
2066  bool result = CFPreferencesGetAppBooleanValue(CFSTR("prelinkCache"), CFSTR("org.vuo.Editor"), &valid);
2067  if (valid)
2068  prelinkCache = result;
2069  });
2070  if (! prelinkCache)
2071  {
2072  VDebugLog("Ignoring the module cache since the 'prelinkCache' preference is false.");
2073  return;
2074  }
2075 
2076  // Don't do anything if this environment doesn't have a cache configured.
2077 
2078  if (moduleCachePath.empty())
2079  return;
2080 
2081  // Don't bother rechecking the cache if the modules in this environment haven't changed.
2082 
2083  string cacheDescription = string() + "the cache of " + moduleCacheSuffix + " modules at '" + moduleCachePath + "'";
2084  if (isModuleCacheInitialized && ! isModuleCacheableDataDirty)
2085  {
2086  VDebugLog("No need to recheck %s.", cacheDescription.c_str());
2087  return;
2088  }
2089 
2090  try
2091  {
2092  VDebugLog("Checking if %s is up-to-date…", cacheDescription.c_str());
2093 
2094  bool isCacheUpToDate = true;
2095 
2096  if (isModuleCacheInitialized && isModuleCacheableDataDirty)
2097  isCacheUpToDate = false;
2098 
2099  isModuleCacheInitialized = true;
2100  isModuleCacheableDataDirty = false;
2101 
2102  const string dylibFileName = "libVuoModuleCache-" + moduleCacheSuffix + ".dylib";
2103  const string indexFileName = "moduleCache-" + moduleCacheSuffix + ".txt";
2104  string dylibPath = moduleCachePath + "/" + dylibFileName;
2105  string indexPath = moduleCachePath + "/" + indexFileName;
2106 
2107  // Create the cache files if they don't already exist. (If they do exist, don't affect the last-modified times.)
2108 
2109  bool dylibFileExists = false;
2110  bool indexFileExists = false;
2111 
2112  bool dirExists = VuoFileUtilities::fileExists(moduleCachePath);
2113  if (dirExists)
2114  {
2115  dylibFileExists = VuoFileUtilities::fileExists(dylibPath);
2116  indexFileExists = VuoFileUtilities::fileExists(indexPath);
2117  }
2118 
2119  if (! (dirExists && dylibFileExists && indexFileExists))
2120  {
2121  if (shouldUseExistingCache)
2122  throw VuoException("Trying to use the existing cache, but the cache doesn't exist.", false);
2123  else
2124  {
2125  if (! dirExists)
2126  VuoFileUtilities::makeDir(moduleCachePath);
2127  if (! indexFileExists)
2128  VuoFileUtilities::createFile(indexPath);
2129  if (! dylibFileExists)
2130  VuoFileUtilities::createFile(dylibPath);
2131 
2132  isCacheUpToDate = false;
2133  }
2134  }
2135 
2136  // Lock the cache for reading.
2137 
2138  VuoFileUtilities::File *fileForLocking;
2139  {
2140  fileForLocking = moduleCacheFileForLocking[dylibPath];
2141  if (! fileForLocking)
2142  {
2143  fileForLocking = new VuoFileUtilities::File(moduleCachePath, dylibFileName);
2144  moduleCacheFileForLocking[dylibPath] = fileForLocking;
2145  }
2146 
2147  if (!fileForLocking->lockForReading())
2148  VDebugLog("\tWarning: Couldn't lock for reading.");
2149  }
2150 
2151  // Check if the dylib looks remotely valid.
2152 
2153  if (isCacheUpToDate)
2154  {
2155  bool dylibHasData = VuoFileUtilities::fileContainsReadableData(dylibPath);
2156  if (! dylibHasData)
2157  {
2158  if (shouldUseExistingCache)
2159  throw VuoException("Trying to use the existing cache, but the cache doesn't contain readable data.", false);
2160  else
2161  isCacheUpToDate = false;
2162  }
2163  }
2164 
2165  // List the items actually in the cache, according to its index.
2166 
2167  const char separator = '\n';
2168  if (isCacheUpToDate || shouldUseExistingCache)
2169  {
2170  VuoFileUtilities::File indexFile(moduleCachePath, indexFileName);
2171  string index = indexFile.getContentsAsString();
2172  vector<string> actualIndex = VuoStringUtilities::split(index, separator);
2173 
2174  moduleCacheContents.clear();
2175  moduleCacheContents.insert(actualIndex.begin(), actualIndex.end());
2176 
2177  if (shouldUseExistingCache)
2178  {
2179  isModuleCacheAvailable = true;
2180  return;
2181  }
2182  }
2183 
2184  // Check if the list of actual items matches the list of expected items.
2185 
2186  if (isCacheUpToDate)
2187  {
2188  if (moduleCacheContents.size() != cacheableModulesAndDependencies.size())
2189  isCacheUpToDate = false;
2190  else
2191  {
2192  set<string> contentsInBoth;
2193  std::set_intersection(moduleCacheContents.begin(), moduleCacheContents.end(),
2194  cacheableModulesAndDependencies.begin(), cacheableModulesAndDependencies.end(),
2195  std::inserter(contentsInBoth, contentsInBoth.begin()));
2196 
2197  if (contentsInBoth.size() != cacheableModulesAndDependencies.size())
2198  isCacheUpToDate = false;
2199  }
2200  }
2201 
2202  // Check if the cache is newer than all of the modules in it.
2203 
2204  if (isCacheUpToDate)
2205  {
2206  unsigned long cacheLastModified = VuoFileUtilities::getFileLastModifiedInSeconds(dylibPath);
2207 
2208  for (map<string, map<string, ModuleInfo *> >::iterator i = moduleFilesAtSearchPath.begin(); i != moduleFilesAtSearchPath.end(); ++i)
2209  {
2210  for (map<string, ModuleInfo *>::iterator j = i->second.begin(); j != i->second.end(); ++j)
2211  {
2212  if (! j->second->isOlderThan(cacheLastModified))
2213  {
2214  isCacheUpToDate = false;
2215  break;
2216  }
2217  }
2218  }
2219  }
2220 
2221  // If the cache is consistent with this environment, we're done.
2222 
2223  if (isCacheUpToDate)
2224  {
2225  VDebugLog("Up-to-date.");
2226 
2227  isModuleCacheAvailable = true;
2228  return;
2229  }
2230 
2231  // Otherwise, (re)build the cache.
2232 
2233  dispatch_async(moduleCacheBuildingQueue, ^{
2234  VDebugLog("Rebuilding %s…", cacheDescription.c_str());
2235 
2236  set<Module *> modulesToLink;
2237  set<string> librariesToLink;
2238  set<string> frameworksToLink;
2239  {
2240  compiler->getLinkerInputs(cacheableModulesAndDependencies, Optimization_SmallBinary, modulesToLink, librariesToLink, frameworksToLink);
2241 
2242  librariesToLink.insert(dylibsNeededToLinkToCaches.begin(), dylibsNeededToLinkToCaches.end());
2243  set<string>::iterator iter = librariesToLink.find(dylibPath); // getCacheableModulesAndDependencies includes dylibPath, but don't want to link against self
2244  if (iter != librariesToLink.end())
2245  librariesToLink.erase(iter);
2246 
2247  frameworksToLink.insert(frameworksNeededToLinkToCaches.begin(), frameworksNeededToLinkToCaches.end());
2248  }
2249 
2250  bool gotLockForWriting = false;
2251  try
2252  {
2253  // Try to upgrade the file lock for writing.
2254  gotLockForWriting = fileForLocking->lockForWriting(true);
2255  if (! gotLockForWriting)
2256  throw VuoException("The cache file is being used by another process. "
2257  "If any composition windows are open from previous Vuo sessions, quit them. "
2258  "If any processes whose names start with \"VuoComposition\" or one of your composition file names appear in Activity Monitor, force-quit them.",
2259  false);
2260 
2261  // Link the dependencies to create a temporary file.
2262  string dir, file, ext;
2263  VuoFileUtilities::splitPath(dylibFileName, dir, file, ext);
2264  string tmpPath = VuoFileUtilities::makeTmpFile(file, "dylib");
2265  compiler->link(tmpPath, modulesToLink, librariesToLink, frameworksToLink, true);
2266 
2267  // Copy the contents of the temporary file into the cached resources dylib.
2268  // This preserves the lock on the cached resources dylib file (https://b33p.net/kosada/node/12970).
2269  VuoFileUtilities::copyFile(tmpPath, dylibPath, true);
2271 
2273  getVuoFrameworkPath() + "/Helpers/install_name_tool",
2274  "-id",
2275  dylibPath,
2276  dylibPath,
2277  });
2278 
2279  // Write the list of dependencies to the index file.
2280  vector<string> expectedContents(cacheableModulesAndDependencies.begin(), cacheableModulesAndDependencies.end());
2281  string index = VuoStringUtilities::join(expectedContents, separator);
2282  VuoFileUtilities::writeStringToFile(index, indexPath);
2283 
2284  // Downgrade the file lock back to reading.
2285  if (!fileForLocking->lockForReading())
2286  VDebugLog("\tWarning: Couldn't downgrade the lock back to reading.");
2287 
2288  dispatch_sync(environmentQueue, ^{
2289  moduleCacheContents = cacheableModulesAndDependencies;
2290  isModuleCacheAvailable = true;
2291  });
2292  }
2293  catch (VuoException &e)
2294  {
2295  // Downgrade the file lock back to reading.
2296  if (gotLockForWriting)
2297  if (!fileForLocking->lockForReading())
2298  VDebugLog("\tWarning: Couldn't downgrade the lock back to reading.");
2299 
2300  VUserLog("Warning: Couldn't rebuild %s for the \"faster build\" optimization: %s", cacheDescription.c_str(), e.what());
2301  }
2302 
2303  VDebugLog("Done.");
2304  });
2305  }
2306  catch (VuoException &e)
2307  {
2308  VUserLog("Warning: Couldn't use %s for the \"faster build\" optimization: %s", cacheDescription.c_str(), e.what());
2309  }
2310 }
2311 
2315 void VuoCompiler::Environment::waitForModuleCachesToBuild(void)
2316 {
2317  dispatch_sync(moduleCacheBuildingQueue, ^{});
2318 }
2319 
2329 bool VuoCompiler::Environment::findInModuleCache(const string &moduleOrDependency, string &cachePath)
2330 {
2331  if (isModuleCacheAvailable && moduleCacheContents.find(moduleOrDependency) != moduleCacheContents.end())
2332  {
2333  cachePath = moduleCachePath + "/libVuoModuleCache-" + moduleCacheSuffix + ".dylib";
2334  return true;
2335  }
2336 
2337  return false;
2338 }
2339 
2346 void VuoCompiler::Environment::modulesChanged(void)
2347 {
2348  isModuleCacheableDataDirty = true;
2349  isModuleCacheAvailable = false;
2350 }
2351 
2355 bool VuoCompiler::Environment::isBuiltIn()
2356 {
2357  return this == sharedEnvironments[0][0];
2358 }
2359 
2363 string VuoCompiler::Environment::getName()
2364 {
2365  if (isBuiltIn())
2366  return "builtin";
2367  else if (this == sharedEnvironments[0][1])
2368  return "builtin (generated)";
2369  else if (this == sharedEnvironments[1][0])
2370  return "system";
2371  else if (this == sharedEnvironments[1][1])
2372  return "system (generated)";
2373  else if (this == sharedEnvironments[2][0])
2374  return "user";
2375  else if (this == sharedEnvironments[2][1])
2376  return "user (generated)";
2377  return "composition-local";
2378 }
2379 
2383 void VuoCompiler::applyToInstalledEnvironments(void (^doForEnvironment)(Environment *))
2384 {
2385  dispatch_sync(environmentQueue, ^{
2386  for (vector< vector<Environment *> >::iterator i = environments.begin(); i != environments.end(); ++i) {
2387  doForEnvironment((*i)[0]);
2388  }
2389  });
2390 }
2391 
2395 void VuoCompiler::applyToInstalledEnvironments(void (^doForEnvironment)(Environment *, bool *, string), bool *result, string arg)
2396 {
2397  dispatch_sync(environmentQueue, ^{
2398  for (vector< vector<Environment *> >::iterator i = environments.begin(); i != environments.end(); ++i) {
2399  doForEnvironment((*i)[0], result, arg);
2400  }
2401  });
2402 }
2403 
2407 void VuoCompiler::applyToAllEnvironments(void (^doForEnvironment)(Environment *environment))
2408 {
2409  dispatch_sync(environmentQueue, ^{
2410  for (vector< vector<Environment *> >::iterator i = environments.begin(); i != environments.end(); ++i) {
2411  for (vector<Environment *>::iterator j = i->begin(); j != i->end(); ++j) {
2412  doForEnvironment(*j);
2413  }
2414  }
2415  });
2416 }
2417 
2426 VuoCompiler::VuoCompiler(const string &compositionPath)
2427 {
2428 #if VUO_PRO
2429  init_Pro();
2430 #endif
2431 
2432  shouldLoadAllModules = true;
2433  hasLoadedAllModules = false;
2434  modulesToLoadQueue = dispatch_queue_create("org.vuo.compiler.modules", NULL);
2435  moduleCacheBuilding = dispatch_group_create();
2436  dependencyGraph = NULL;
2437  compositionDependencyGraph = NULL;
2438  isVerbose = false;
2439  _shouldShowSplashWindow = false;
2440  delegate = NULL;
2441  delegateQueue = dispatch_queue_create("org.vuo.compiler.delegate", NULL);
2442 
2443  string vuoFrameworkPath = getVuoFrameworkPath();
2444  if (! vuoFrameworkPath.empty())
2445  clangPath = vuoFrameworkPath + "/Helpers/clang";
2446  else
2447  clangPath = llvm::sys::Path(StringRef(LLVM_ROOT "/bin/clang"));
2448 
2449  dispatch_sync(environmentQueue, ^{
2450  allCompilers.insert(this);
2451 
2452  if (sharedEnvironments.empty())
2453  {
2454  sharedEnvironments = vector< vector<Environment *> >(3, vector<Environment *>(2, NULL));
2455  for (vector< vector<Environment *> >::iterator i = sharedEnvironments.begin(); i != sharedEnvironments.end(); ++i) {
2456  for (vector<Environment *>::iterator j = i->begin(); j != i->end(); ++j) {
2457  *j = new Environment();
2458  }
2459  }
2460 
2461  vector<string> builtInModuleSearchPaths = Environment::getBuiltInModuleSearchPaths();
2462  for (vector<string>::iterator i = builtInModuleSearchPaths.begin(); i != builtInModuleSearchPaths.end(); ++i) {
2463  sharedEnvironments[0][0]->addModuleSearchPath(*i, false);
2464  }
2465 
2466  vector<string> builtInHeaderSearchPaths = Environment::getBuiltInHeaderSearchPaths();
2467  for (vector<string>::iterator i = builtInHeaderSearchPaths.begin(); i != builtInHeaderSearchPaths.end(); ++i) {
2468  sharedEnvironments[0][0]->addHeaderSearchPath(*i);
2469  }
2470 
2471  vector<string> builtInLibrarySearchPaths = Environment::getBuiltInLibrarySearchPaths();
2472  for (vector<string>::iterator i = builtInLibrarySearchPaths.begin(); i != builtInLibrarySearchPaths.end(); ++i) {
2473  sharedEnvironments[0][0]->addLibrarySearchPath(*i);
2474  }
2475 
2476  vector<string> builtInFrameworkSearchPaths = Environment::getBuiltInFrameworkSearchPaths();
2477  for (vector<string>::iterator i = builtInFrameworkSearchPaths.begin(); i != builtInFrameworkSearchPaths.end(); ++i) {
2478  sharedEnvironments[0][0]->addFrameworkSearchPath(*i);
2479  }
2480 
2481  // Allow system administrator to override Vuo.framework modules
2482  sharedEnvironments[1][0]->addModuleSearchPath(VuoFileUtilities::getSystemModulesPath());
2483  sharedEnvironments[1][0]->addLibrarySearchPath(VuoFileUtilities::getSystemModulesPath());
2484 
2485  // Allow user to override Vuo.framework and system-wide modules
2486  sharedEnvironments[2][0]->addModuleSearchPath(VuoFileUtilities::getUserModulesPath());
2487  sharedEnvironments[2][0]->addLibrarySearchPath(VuoFileUtilities::getUserModulesPath());
2488 
2489  // Set up module cache paths.
2490  // Since the built-in module caches are part of Vuo.framework (put there by `generateBuiltInModuleCaches`),
2491  // only attempt to use module caches if Vuo.framework exists.
2492  string vuoFrameworkPath = getVuoFrameworkPath();
2493  if (! vuoFrameworkPath.empty())
2494  {
2495  vector<string> moduleCachePaths(3);
2496  moduleCachePaths[0] = vuoFrameworkPath + "/Modules/Builtin";
2497  moduleCachePaths[1] = VuoFileUtilities::getCachePath() + "/System";
2498  moduleCachePaths[2] = VuoFileUtilities::getCachePath() + "/User";
2499 
2500  for (size_t i = 0; i < moduleCachePaths.size(); ++i)
2501  {
2502  string moduleCachePath = moduleCachePaths[i];
2503 
2504  sharedEnvironments[i][0]->setModuleCachePath(moduleCachePath);
2505  sharedEnvironments[i][0]->addModuleSearchPath(moduleCachePath + "/Modules", false);
2506 
2507  sharedEnvironments[i][1]->setModuleCachePath(moduleCachePath);
2508  }
2509  }
2510  }
2511  });
2512 
2513  setCompositionPath(compositionPath);
2514 }
2515 
2520 {
2521 #if VUO_PRO
2522  fini_Pro();
2523 #endif
2524 
2525  dispatch_sync(environmentQueue, ^{
2526  allCompilers.erase(this);
2527  });
2528 
2529  dispatch_group_wait(moduleCacheBuilding, DISPATCH_TIME_FOREVER);
2530  dispatch_release(moduleCacheBuilding);
2531 
2532  for (vector< vector<Environment *> >::iterator i = environments.begin(); i != environments.end(); ++i)
2533  (*i)[0]->removeCompilerToNotify(this);
2534 
2535  dispatch_sync(delegateQueue, ^{});
2536 
2537  dispatch_release(modulesToLoadQueue);
2538  dispatch_release(delegateQueue);
2539 
2540  delete dependencyGraph;
2541  delete compositionDependencyGraph;
2542 }
2543 
2547 void VuoCompiler::reset(void)
2548 {
2549  dispatch_group_wait(moduleSourceCompilersExist, DISPATCH_TIME_FOREVER);
2550 
2551  for (vector< vector<Environment *> >::iterator i = sharedEnvironments.begin(); i != sharedEnvironments.end(); ++i)
2552  {
2553  (*i)[0]->stopWatchingModuleSearchPaths();
2554  dispatch_sync((*i)[0]->moduleSearchPathContentsChangedQueue, ^{});
2555  }
2556 
2557  for (map< string, vector<Environment *> >::iterator i = environmentsForCompositionFamily.begin(); i != environmentsForCompositionFamily.end(); ++i)
2558  {
2559  (i->second)[0]->stopWatchingModuleSearchPaths();
2560  dispatch_sync((i->second)[0]->moduleSearchPathContentsChangedQueue, ^{});
2561  }
2562 
2563  for (vector< vector<Environment *> >::iterator i = sharedEnvironments.begin(); i != sharedEnvironments.end(); ++i)
2564  for (vector<Environment *>::iterator j = i->begin(); j != i->end(); ++j)
2565  delete *j;
2566 
2567  for (map< string, vector<Environment *> >::iterator i = environmentsForCompositionFamily.begin(); i != environmentsForCompositionFamily.end(); ++i)
2568  for (vector<Environment *>::iterator j = i->second.begin(); j != i->second.end(); ++j)
2569  delete *j;
2570 
2571  allCompilers.clear();
2572  sharedEnvironments.clear();
2573  environmentsForCompositionFamily.clear();
2574 }
2575 
2582 {
2583  dispatch_async(delegateQueue, ^{
2584  this->delegate = delegate;
2585  });
2586 }
2587 
2597 void VuoCompiler::setCompositionPath(const string &compositionPath)
2598 {
2599  string compositionModulesDir;
2600  string compositionBaseDir;
2601  lastCompositionIsSubcomposition = false;
2602  if (! compositionPath.empty())
2603  {
2604  compositionModulesDir = VuoFileUtilities::getCompositionLocalModulesPath(compositionPath);
2605 
2606  string file, ext;
2607  VuoFileUtilities::splitPath(compositionModulesDir, compositionBaseDir, file, ext);
2608  VuoFileUtilities::canonicalizePath(compositionBaseDir);
2609 
2610  string compositionDir;
2611  VuoFileUtilities::splitPath(compositionPath, compositionDir, file, ext);
2612  VuoFileUtilities::canonicalizePath(compositionDir);
2613  lastCompositionIsSubcomposition = (compositionDir == compositionModulesDir);
2614  }
2615 
2616  // Set up `environments` to contain all environments available to this compiler, in order from broadest to narrowest.
2617 
2618  dispatch_sync(environmentQueue, ^{
2619  if (! environments.empty() && compositionBaseDir == lastCompositionBaseDir) {
2620  return;
2621  }
2622  lastCompositionBaseDir = compositionBaseDir;
2623 
2624  // Clear out the existing environments for this compiler.
2625 
2626  Environment *oldCompositionFamilyInstalledEnvironment = nullptr;
2627  vector<Environment *> compositionEnvironments;
2628  if (! environments.empty())
2629  {
2630  for (vector< vector<Environment *> >::iterator i = environments.begin(); i != environments.end(); ++i) {
2631  (*i)[0]->removeCompilerToNotify(this);
2632  }
2633 
2634  if (environments.size() >= 5) {
2635  oldCompositionFamilyInstalledEnvironment = environments[3][0];
2636  }
2637 
2638  compositionEnvironments = environments.back();
2639 
2640  environments.clear();
2641  }
2642 
2643  // If the composition is located in one of the shared environments (built-in, system, user),
2644  // add that shared environment and any shared environments at broader scope to the compiler.
2645  // If the composition is not in a shared environment, add all of the shared environments to the compiler.
2646 
2647  bool isCompositionInSharedEnvironment = false;
2648  for (vector< vector<Environment *> >::iterator i = sharedEnvironments.begin(); i != sharedEnvironments.end(); ++i)
2649  {
2650  environments.push_back(*i);
2651 
2652  vector<string> moduleSearchPaths = (*i)[0]->getModuleSearchPaths();
2653  for (vector<string>::iterator j = moduleSearchPaths.begin(); j != moduleSearchPaths.end(); ++j)
2654  {
2655  string moduleSearchPath = *j;
2656  VuoFileUtilities::canonicalizePath(moduleSearchPath);
2657  if (moduleSearchPath == compositionModulesDir)
2658  {
2659  isCompositionInSharedEnvironment = true;
2660  break;
2661  }
2662  }
2663 
2664  if (isCompositionInSharedEnvironment) {
2665  break;
2666  }
2667  }
2668 
2669  // If the composition is not in a shared environment, add the composition-family environment to the compiler.
2670 
2671  Environment *newCompositionFamilyInstalledEnvironment = nullptr;
2672  if (! isCompositionInSharedEnvironment && ! compositionPath.empty())
2673  {
2674  vector<Environment *> compositionFamilyEnvironments = environmentsForCompositionFamily[compositionBaseDir];
2675  if (compositionFamilyEnvironments.empty())
2676  {
2677  compositionFamilyEnvironments = vector<Environment *>(2, NULL);
2678  compositionFamilyEnvironments[0] = new Environment();
2679  compositionFamilyEnvironments[1] = new Environment();
2680  environmentsForCompositionFamily[compositionBaseDir] = compositionFamilyEnvironments;
2681 
2682  // Allow the user to place modules/subcompositions in a Modules folder inside the composition folder.
2683 
2684  compositionFamilyEnvironments[0]->addModuleSearchPath(compositionModulesDir);
2685 
2686  // Locate this environment's cache in a folder whose name is the composition folder path with
2687  // slashes replaced with Unicode Modifier Letter Colon.
2688  string moduleCachePath = getCachePathForComposition(compositionBaseDir);
2689 
2690  compositionFamilyEnvironments[0]->setModuleCachePath(moduleCachePath);
2691  compositionFamilyEnvironments[0]->addModuleSearchPath(moduleCachePath + "/Modules", false);
2692 
2693  compositionFamilyEnvironments[1]->setModuleCachePath(moduleCachePath);
2694  }
2695  environments.push_back(compositionFamilyEnvironments);
2696 
2697  newCompositionFamilyInstalledEnvironment = compositionFamilyEnvironments[0];
2698  }
2699 
2700  // Add the composition environment to the compiler (or add it back in if it already existed).
2701 
2702  if (compositionEnvironments.empty())
2703  {
2704  compositionEnvironments = vector<Environment *>(2, NULL);
2705  compositionEnvironments[0] = new Environment();
2706  compositionEnvironments[1] = new Environment();
2707  }
2708  environments.push_back(compositionEnvironments);
2709 
2710  for (vector< vector<Environment *> >::iterator i = environments.begin(); i != environments.end(); ++i) {
2711  (*i)[0]->addCompilerToNotify(this);
2712  }
2713 
2714  delete dependencyGraph;
2715  delete compositionDependencyGraph;
2716  dependencyGraph = makeDependencyNetwork(environments, ^VuoDirectedAcyclicGraph * (Environment *env) { return env->getDependencyGraph(); });
2717  compositionDependencyGraph = makeDependencyNetwork(environments, ^VuoDirectedAcyclicGraph * (Environment *env) { return env->getCompositionDependencyGraph(); });
2718 
2719  // If the compiler has a different local Modules directory than before, notify the compiler's delegate
2720  // of composition-family modules that are newly available/unavailable.
2721 
2722  if (oldCompositionFamilyInstalledEnvironment != newCompositionFamilyInstalledEnvironment)
2723  {
2724  auto getModules = [] (Environment *env)
2725  {
2726  map<string, VuoCompilerModule *> modules;
2727  if (env)
2728  {
2729  for (auto i : env->getNodeClasses()) {
2730  modules.insert(i);
2731  }
2732  for (auto i : env->getTypes()) {
2733  modules.insert(i);
2734  }
2735  for (auto i : env->getLibraryModules()) {
2736  modules.insert(i);
2737  }
2738  }
2739  return modules;
2740  };
2741 
2742  map<string, VuoCompilerModule *> modulesAdded = getModules(newCompositionFamilyInstalledEnvironment);
2743  map<string, VuoCompilerModule *> modulesRemoved = getModules(oldCompositionFamilyInstalledEnvironment);
2744 
2745  map<string, pair<VuoCompilerModule *, VuoCompilerModule *> > modulesModified;
2746  for (map<string, VuoCompilerModule *>::iterator add = modulesAdded.begin(); add != modulesAdded.end(); )
2747  {
2748  map<string, VuoCompilerModule *>::iterator rem = modulesRemoved.find(add->first);
2749  if (rem != modulesRemoved.end())
2750  {
2751  modulesModified[add->first] = make_pair(rem->second, add->second);
2752  modulesAdded.erase(add++);
2753  modulesRemoved.erase(rem);
2754  }
2755  else
2756  {
2757  ++add;
2758  }
2759  }
2760 
2761  if (! (modulesAdded.empty() && modulesModified.empty() && modulesRemoved.empty()) )
2762  {
2763  VuoCompilerIssues *issues = new VuoCompilerIssues();
2764  VuoCompilerDelegate::LoadedModulesData *delegateData = new VuoCompilerDelegate::LoadedModulesData(set< pair<VuoCompilerModule *, VuoCompilerModule *> >(), set<VuoCompilerModule *>(), issues);
2765  delegateData->retain();
2766 
2767  Environment *scopeEnvironment = newCompositionFamilyInstalledEnvironment;
2768  if (! scopeEnvironment) {
2769  scopeEnvironment = compositionEnvironments.at(0);
2770  }
2771 
2772  loadedModules(modulesAdded, modulesModified, modulesRemoved, issues, delegateData, scopeEnvironment);
2773  }
2774  }
2775  });
2776 }
2777 
2783 VuoDirectedAcyclicNetwork * VuoCompiler::makeDependencyNetwork(const vector< vector<Environment *> > &environments,
2784  VuoDirectedAcyclicGraph * (^graphForEnvironment)(Environment *))
2785 {
2786  if (!graphForEnvironment)
2787  return NULL;
2788 
2790 
2791  for (vector< vector<Environment *> >::const_iterator i = environments.begin(); i != environments.end(); ++i)
2792  {
2793  // Installed environment depends on generated environment in same scope.
2794 
2795  network->addEdge(graphForEnvironment(i->at(0)), graphForEnvironment(i->at(1)));
2796 
2797  // Installed and generated environments depend on installed environments in broader scopes.
2798 
2799  for (vector< vector<Environment *> >::const_iterator ii = environments.begin(); ii != i; ++ii)
2800  {
2801  network->addEdge(graphForEnvironment(i->at(0)), graphForEnvironment(ii->at(0)));
2802  network->addEdge(graphForEnvironment(i->at(1)), graphForEnvironment(ii->at(0)));
2803  }
2804  }
2805 
2806  return network;
2807 }
2808 
2829 void VuoCompiler::loadModulesIfNeeded(const set<string> &moduleKeys)
2830 {
2831  __block bool willLoadAllModules = false;
2832  if (moduleKeys.empty())
2833  {
2834  dispatch_sync(modulesToLoadQueue, ^{
2835  if (shouldLoadAllModules && ! hasLoadedAllModules) {
2836  willLoadAllModules = true;
2837  hasLoadedAllModules = true;
2838  }
2839  });
2840  }
2841 
2842  if (! willLoadAllModules && moduleKeys.empty())
2843  return;
2844 
2845  // Load modules and start sources compiling.
2846 
2847  __block set<dispatch_group_t> sourcesLoading;
2848  dispatch_sync(environmentQueue, ^{
2849  sourcesLoading = loadModulesAndSources(moduleKeys, set<string>(), set<string>(),
2850  moduleKeys, set<string>(), set<string>(),
2851  willLoadAllModules, false, nullptr, nullptr, nullptr, "");
2852  });
2853 
2854  // Wait for sources to finish compiling and their modules to be loaded,
2855  // to ensure that `getNodeClass(subcomposition)` finds the subcomposition node class.
2856 
2857  for (set<dispatch_group_t>::iterator i = sourcesLoading.begin(); i != sourcesLoading.end(); ++i)
2858  {
2859  dispatch_group_wait(*i, DISPATCH_TIME_FOREVER);
2860  dispatch_release(*i);
2861  }
2862 }
2863 
2872 set<dispatch_group_t> VuoCompiler::loadModulesAndSources(const set<string> &modulesAddedKeys, const set<string> &modulesModifiedKeys, const set<string> &modulesRemovedKeys,
2873  const set<string> &sourcesAddedKeys, const set<string> &sourcesModifiedKeys, const set<string> &sourcesRemovedKeys,
2874  bool willLoadAllModules, bool shouldRecompileSourcesIfUnchanged,
2875  Environment *currentEnvironment, VuoCompilerIssues *issuesForCurrentEnvironment,
2876  std::function<void(void)> moduleLoadedCallback, const string &moduleAddedOrModifiedSourceCode)
2877 {
2878  //VLog("C=%p E=%p -- %lu %lu %lu %lu %lu %lu %i %i", this, currentEnvironment,
2879  //modulesAddedKeys.size(), modulesModifiedKeys.size(), modulesRemovedKeys.size(),
2880  //sourcesAddedKeys.size(), sourcesModifiedKeys.size(), sourcesRemovedKeys.size(),
2881  //willLoadAllModules, shouldRecompileSourcesIfUnchanged);
2882  //if (modulesAddedKeys.size() == 1) VLog(" %s", modulesAddedKeys.begin()->c_str());
2883  //if (modulesModifiedKeys.size() == 1) VLog(" %s", modulesModifiedKeys.begin()->c_str());
2884  //if (modulesRemovedKeys.size() == 1) VLog(" %s", modulesRemovedKeys.begin()->c_str());
2885 
2886  // Organize the modules, source files, and issues by environment.
2887 
2888  map<Environment *, set<string> > modulesAdded;
2889  map<Environment *, set<string> > modulesModified;
2890  map<Environment *, set<string> > modulesRemoved;
2891  map<Environment *, set<string> > sourcesAdded;
2892  map<Environment *, set<string> > sourcesModified;
2893  map<Environment *, set<string> > sourcesRemoved;
2894  map<Environment *, set<string> > potentialSpecializedModules;
2895 
2896  if (currentEnvironment)
2897  {
2898  modulesAdded[currentEnvironment] = modulesAddedKeys;
2899  modulesModified[currentEnvironment] = modulesModifiedKeys;
2900  modulesRemoved[currentEnvironment] = modulesRemovedKeys;
2901  sourcesAdded[currentEnvironment] = sourcesAddedKeys;
2902  sourcesModified[currentEnvironment] = sourcesModifiedKeys;
2903  sourcesRemoved[currentEnvironment] = sourcesRemovedKeys;
2904  }
2905  else
2906  {
2907  Environment *genEnv = nullptr;
2908  if (! willLoadAllModules)
2909  {
2910  // If compiling a top-level composition, generated modules that were directly requested (in modulesAddedKeys) are
2911  // added at the composition scope so they won't be cached. This prevents link errors when running multiple
2912  // compositions in the current process (https://b33p.net/kosada/node/14317).
2913 
2914  int scope = environments.size() - (lastCompositionIsSubcomposition ? 2 : 1);
2915  genEnv = environments.at(scope).at(1);
2916  potentialSpecializedModules[genEnv] = modulesAddedKeys;
2917  }
2918 
2919  for (vector< vector<Environment *> >::iterator i = environments.begin(); i != environments.end(); ++i)
2920  {
2921  Environment *env = (*i).at(0);
2922 
2923  ModuleInfoIterator modulesAddedIter = (willLoadAllModules ? env->listAllModules() : env->listModules(modulesAddedKeys));
2924  ModuleInfoIterator sourcesAddedIter = (willLoadAllModules ? env->listAllSourceFiles() : env->listSourceFiles(sourcesAddedKeys));
2925  ModuleInfoIterator sourcesModifiedIter = (willLoadAllModules ? env->listAllSourceFiles() : env->listSourceFiles(sourcesModifiedKeys));
2926 
2927  ModuleInfo *moduleInfo;
2928  while ((moduleInfo = modulesAddedIter.next()))
2929  {
2930  string moduleKey = moduleInfo->getModuleKey();
2931 
2932  modulesAdded[env].insert(moduleKey);
2933 
2934  if (! willLoadAllModules)
2935  {
2936  auto foundIter = potentialSpecializedModules[genEnv].find(moduleKey);
2937  if (foundIter != potentialSpecializedModules[genEnv].end())
2938  potentialSpecializedModules[genEnv].erase(foundIter);
2939  }
2940  }
2941 
2942  // If a source file and a compiled file for the same module are in the same folder,
2943  // the compiled file takes precedence.
2944  auto isCompiledModuleAtSameSearchPath = [&env] (ModuleInfo *sourceInfo)
2945  {
2946  ModuleInfo *compiledModuleInfo = env->listModule(sourceInfo->getModuleKey());
2947  return (compiledModuleInfo && compiledModuleInfo->getSearchPath() == sourceInfo->getSearchPath());
2948  };
2949 
2950  while ((moduleInfo = sourcesAddedIter.next()))
2951  {
2952  if (isCompiledModuleAtSameSearchPath(moduleInfo))
2953  continue;
2954 
2955  sourcesAdded[env].insert( moduleInfo->getModuleKey() );
2956  }
2957 
2958  while ((moduleInfo = sourcesModifiedIter.next()))
2959  {
2960  if (isCompiledModuleAtSameSearchPath(moduleInfo))
2961  continue;
2962 
2963  sourcesModified[env].insert( moduleInfo->getModuleKey() );
2964  }
2965  }
2966  }
2967 
2968  map<Environment *, VuoCompilerIssues *> issues;
2969  for (vector< vector<Environment *> >::iterator i = environments.begin(); i != environments.end(); ++i)
2970  {
2971  Environment *env = (*i).at(0);
2972  issues[env] = (env == currentEnvironment && issuesForCurrentEnvironment ? issuesForCurrentEnvironment : new VuoCompilerIssues());
2973  }
2974 
2975  // Check for circular dependencies in added/modified sources.
2976 
2977  for (vector< vector<Environment *> >::iterator i = environments.begin(); i != environments.end(); ++i)
2978  {
2979  Environment *env = (*i).at(0);
2980 
2981  // Check for circular dependencies involving sources being loaded within this environment.
2982  // For circular dependencies involving sources in different environments,
2983  // an error will be reported elsewhere due to one of them being outside of the other's scope.
2984  set<VuoDirectedAcyclicGraph::Vertex *> circularDependencies = env->getCompositionDependencyGraph()->getCycleVertices();
2985 
2986  set<string> sourcesAddedModified;
2987  sourcesAddedModified.insert(sourcesAdded[env].begin(), sourcesAdded[env].end());
2988  sourcesAddedModified.insert(sourcesModified[env].begin(), sourcesModified[env].end());
2989 
2990  for (set<string>::iterator j = sourcesAddedModified.begin(); j != sourcesAddedModified.end(); ++j)
2991  {
2992  string moduleKey = *j;
2993 
2994  bool found = false;
2995  for (set<VuoDirectedAcyclicGraph::Vertex *>::iterator k = circularDependencies.begin(); k != circularDependencies.end(); ++k)
2996  {
2997  DependencyGraphVertex *vertex = static_cast<DependencyGraphVertex *>(*k);
2998  if (vertex->getDependency() == moduleKey)
2999  {
3000  found = true;
3001  break;
3002  }
3003  }
3004 
3005  if (found)
3006  {
3007  sourcesAdded[env].erase(moduleKey);
3008  sourcesModified[env].erase(moduleKey);
3009 
3010  VuoCompilerIssue issue(VuoCompilerIssue::Error, "compiling subcomposition", moduleKey,
3011  "Subcomposition contains itself",
3012  "%moduleKey contains an instance of itself, "
3013  "or contains another subcomposition that contains an instance of %moduleKey.");
3014  issue.setModuleKey(moduleKey);
3015 
3016  ModuleInfo *sourceInfo = env->listSourceFile(moduleKey);
3017  if (sourceInfo)
3018  issue.setFilePath(sourceInfo->getFile()->path());
3019 
3020  issues[env]->append(issue);
3021  }
3022  }
3023  }
3024 
3025  // Update the longest downstream paths for added/modified sources.
3026 
3027  for (const vector<Environment *> &envs : environments)
3028  {
3029  Environment *env = envs.at(0);
3030 
3031  set<string> sourcesAddedModified;
3032  sourcesAddedModified.insert(sourcesAdded[env].begin(), sourcesAdded[env].end());
3033  sourcesAddedModified.insert(sourcesModified[env].begin(), sourcesModified[env].end());
3034 
3035  for (const string &moduleKey : sourcesAddedModified)
3036  {
3037  VuoDirectedAcyclicGraph::Vertex *vertex = env->getCompositionDependencyGraph()->findVertex(moduleKey);
3038  int pathLength = env->getCompositionDependencyGraph()->getLongestDownstreamPath(vertex);
3039 
3040  ModuleInfo *sourceInfo = env->listSourceFile(moduleKey);
3041  sourceInfo->setLongestDownstreamPath(pathLength);
3042  }
3043  }
3044 
3045  // Find all modules and sources that depend on the modules and sources being modified or removed.
3046  // Those that belong to one of this compiler's environments are used in subsequent stages.
3047  // Those that belong to some other environment are scheduled to be handled by a separate call to this function.
3048 
3049  map<Environment *, set<string> > modulesDepOnModulesModified;
3050  map<Environment *, set<string> > sourcesDepOnModulesModified;
3051  map<Environment *, set<string> > modulesDepOnModulesRemoved;
3052  map<Environment *, set<string> > sourcesDepOnModulesRemoved;
3053 
3054  {
3055  __block map<Environment *, set<string> > modulesDepOnModulesModified_otherCompiler;
3056  __block map<Environment *, set<string> > sourcesDepOnModulesModified_otherCompiler;
3057  __block map<Environment *, set<string> > modulesDepOnModulesRemoved_otherCompiler;
3058  __block map<Environment *, set<string> > sourcesDepOnModulesRemoved_otherCompiler;
3059 
3060  vector<VuoDirectedAcyclicNetwork *> searchDependencyGraphs;
3061  for (VuoCompiler *compiler : allCompilers)
3062  searchDependencyGraphs.push_back(compiler->dependencyGraph);
3063 
3064  VuoDirectedAcyclicGraph *currentEnvironmentDependencyGraph = (currentEnvironment ? currentEnvironment->getDependencyGraph() : nullptr);
3065 
3066  findDependentModulesAndSources(modulesModified, searchDependencyGraphs, currentEnvironmentDependencyGraph,
3067  modulesDepOnModulesModified, modulesDepOnModulesModified_otherCompiler,
3068  sourcesDepOnModulesModified, sourcesDepOnModulesModified_otherCompiler);
3069 
3070  findDependentModulesAndSources(modulesRemoved, searchDependencyGraphs, currentEnvironmentDependencyGraph,
3071  modulesDepOnModulesRemoved, modulesDepOnModulesRemoved_otherCompiler,
3072  sourcesDepOnModulesRemoved, sourcesDepOnModulesRemoved_otherCompiler);
3073 
3074  set<Environment *> otherEnvironments;
3075  for (map<Environment *, set<string> >::iterator i = modulesDepOnModulesModified_otherCompiler.begin(); i != modulesDepOnModulesModified_otherCompiler.end(); ++i)
3076  otherEnvironments.insert(i->first);
3077  for (map<Environment *, set<string> >::iterator i = sourcesDepOnModulesModified_otherCompiler.begin(); i != sourcesDepOnModulesModified_otherCompiler.end(); ++i)
3078  otherEnvironments.insert(i->first);
3079  for (map<Environment *, set<string> >::iterator i = modulesDepOnModulesRemoved_otherCompiler.begin(); i != modulesDepOnModulesRemoved_otherCompiler.end(); ++i)
3080  otherEnvironments.insert(i->first);
3081  for (map<Environment *, set<string> >::iterator i = sourcesDepOnModulesRemoved_otherCompiler.begin(); i != sourcesDepOnModulesRemoved_otherCompiler.end(); ++i)
3082  otherEnvironments.insert(i->first);
3083 
3084  for (Environment *env : otherEnvironments)
3085  {
3086  VuoCompiler *otherCompiler = nullptr;
3087  for (VuoCompiler *c : allCompilers)
3088  for (const vector<Environment *> &ee : c->environments)
3089  for (Environment *e : ee)
3090  if (e == env)
3091  {
3092  otherCompiler = c;
3093  goto foundOtherCompiler;
3094  }
3095  foundOtherCompiler:
3096 
3097  dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{
3098  dispatch_sync(environmentQueue, ^{
3099  otherCompiler->loadModulesAndSources(set<string>(), modulesDepOnModulesModified_otherCompiler[env], modulesDepOnModulesRemoved_otherCompiler[env],
3100  set<string>(), sourcesDepOnModulesModified_otherCompiler[env], sourcesDepOnModulesRemoved_otherCompiler[env],
3101  false, true, env, nullptr, nullptr, "");
3102  });
3103  });
3104  }
3105  }
3106 
3107  // Unload:
3108  // - modules that have been removed or modified
3109  // - modules that depend on them
3110 
3111  map<Environment *, set<VuoCompilerModule *> > actualModulesRemoved;
3112  for (const vector<Environment *> &envs : environments)
3113  {
3114  for (Environment *env : envs)
3115  {
3116  set<string> modulesToUnload;
3117  modulesToUnload.insert(modulesRemoved[env].begin(), modulesRemoved[env].end());
3118  modulesToUnload.insert(modulesModified[env].begin(), modulesModified[env].end());
3119  modulesToUnload.insert(modulesDepOnModulesRemoved[env].begin(), modulesDepOnModulesRemoved[env].end());
3120  modulesToUnload.insert(modulesDepOnModulesModified[env].begin(), modulesDepOnModulesModified[env].end());
3121 
3122  actualModulesRemoved[env] = env->unloadCompiledModules(modulesToUnload);
3123 
3124  if (!env->isBuiltIn() && !actualModulesRemoved[env].empty())
3125  {
3126  set<string> actualModulesRemovedKeys;
3127  for (auto m : actualModulesRemoved[env])
3128  actualModulesRemovedKeys.insert(m->getPseudoBase()->getModuleKey());
3129  VUserLog("Removed from %s environment: %s", env->getName().c_str(), VuoStringUtilities::join(actualModulesRemovedKeys, ", ").c_str());
3130  }
3131  }
3132  }
3133 
3134  // Load:
3135  // - modules that have been added or modified
3136  // - modules that they depend on
3137  // - modules that depend on them that were just unloaded
3138  // Delete:
3139  // - cached module files in `modulesToLoad` whose source files have been removed
3140 
3141  map<Environment *, set<string> > modulesToLoad;
3142  map<Environment *, map<string, string> > modulesToLoadSourceCode;
3143  for (vector< vector<Environment *> >::iterator i = environments.begin(); i != environments.end(); ++i)
3144  {
3145  Environment *env = (*i).at(0);
3146 
3147  if (! modulesAdded[env].empty())
3148  modulesToLoad[env].insert(modulesAdded[env].begin(), modulesAdded[env].end());
3149  if (! modulesModified[env].empty())
3150  modulesToLoad[env].insert(modulesModified[env].begin(), modulesModified[env].end());
3151  if (! modulesDepOnModulesModified[env].empty())
3152  modulesToLoad[env].insert(modulesDepOnModulesModified[env].begin(), modulesDepOnModulesModified[env].end());
3153 
3154  if (env == currentEnvironment && moduleLoadedCallback)
3155  {
3156  if (modulesAdded[env].size() == 1)
3157  modulesToLoadSourceCode[env][*modulesAdded[env].begin()] = moduleAddedOrModifiedSourceCode;
3158  else if (modulesModified[env].size() == 1)
3159  modulesToLoadSourceCode[env][*modulesModified[env].begin()] = moduleAddedOrModifiedSourceCode;
3160  }
3161  }
3162 
3163  map<Environment *, set<VuoCompilerModule *> > actualModulesAdded;
3164  while (! modulesToLoad.empty())
3165  {
3166  set<string> dependenciesToLoad;
3167  map<Environment *, set<string> > potentialSpecializedDependencies;
3168  for (vector< vector<Environment *> >::iterator i = environments.begin(); i != environments.end(); ++i)
3169  {
3170  Environment *env = (*i).at(0);
3171  Environment *genEnv = (*i).at(1);
3172 
3173  set<VuoCompilerModule *> actualModulesLoaded = env->loadCompiledModules(modulesToLoad[env], modulesToLoadSourceCode[env]);
3174 
3175  actualModulesAdded[env].insert(actualModulesLoaded.begin(), actualModulesLoaded.end());
3176  modulesToLoad.erase(env);
3177 
3178  set<string> actualModulesLoadedKeys;
3179  for (set<VuoCompilerModule *>::iterator j = actualModulesLoaded.begin(); j != actualModulesLoaded.end(); ++j)
3180  {
3181  set<string> dependencies = (*j)->getDependencies();
3182  dependenciesToLoad.insert(dependencies.begin(), dependencies.end());
3183  potentialSpecializedDependencies[genEnv].insert(dependencies.begin(), dependencies.end());
3184  actualModulesLoadedKeys.insert((*j)->getPseudoBase()->getModuleKey());
3185  }
3186 
3187  if (!env->isBuiltIn() && !actualModulesLoadedKeys.empty())
3188  VUserLog("Loaded into %s environment: %s", env->getName().c_str(), VuoStringUtilities::join(actualModulesLoadedKeys, ", ").c_str());
3189  }
3190 
3191  for (vector< vector<Environment *> >::iterator i = environments.begin(); i != environments.end(); ++i)
3192  {
3193  Environment *env = (*i).at(0);
3194 
3195  ModuleInfoIterator dependenciesInEnv = env->listModules(dependenciesToLoad);
3196  ModuleInfo *moduleInfo;
3197  while ((moduleInfo = dependenciesInEnv.next()))
3198  {
3199  modulesToLoad[env].insert( moduleInfo->getModuleKey() );
3200 
3201  for (map<Environment *, set<string> >::iterator j = potentialSpecializedDependencies.begin(); j != potentialSpecializedDependencies.end(); ++j)
3202  {
3203  auto foundIter = j->second.find( moduleInfo->getModuleKey() );
3204  if (foundIter != j->second.end())
3205  j->second.erase(foundIter);
3206  }
3207  }
3208  }
3209 
3210  for (vector< vector<Environment *> >::iterator i = environments.begin(); i != environments.end(); ++i)
3211  {
3212  Environment *genEnv = (*i).at(1);
3213  potentialSpecializedModules[genEnv].insert(potentialSpecializedDependencies[genEnv].begin(), potentialSpecializedDependencies[genEnv].end());
3214  }
3215  }
3216 
3217  // Load asynchronously:
3218  // - specializations of generic modules
3219 
3220  set<dispatch_group_t> specializedModulesLoading;
3221  for (vector< vector<Environment *> >::iterator i = environments.begin(); i != environments.end(); ++i)
3222  {
3223  Environment *genEnv = (*i).at(1);
3224  set<dispatch_group_t> s = genEnv->loadSpecializedModules(potentialSpecializedModules[genEnv], this, llvmQueue);
3225  specializedModulesLoading.insert(s.begin(), s.end());
3226  }
3227 
3228  // Notify those waiting on a source file to be compiled that its module has now been loaded.
3229 
3230  if (moduleLoadedCallback)
3231  moduleLoadedCallback();
3232 
3233  // Move modified modules from `actualModulesAdded` and `actualModulesRemoved` to `actualModulesModified`.
3234 
3235  map<Environment *, set< pair<VuoCompilerModule *, VuoCompilerModule *> > > actualModulesModified;
3236  for (vector< vector<Environment *> >::iterator i = environments.begin(); i != environments.end(); ++i)
3237  {
3238  Environment *env = (*i).at(0);
3239 
3240  for (set<VuoCompilerModule *>::iterator add = actualModulesAdded[env].begin(); add != actualModulesAdded[env].end(); )
3241  {
3242  set<VuoCompilerModule *>::iterator rem;
3243  for (rem = actualModulesRemoved[env].begin(); rem != actualModulesRemoved[env].end(); ++rem)
3244  if ((*rem)->getPseudoBase()->getModuleKey() == (*add)->getPseudoBase()->getModuleKey())
3245  break;
3246 
3247  if (rem != actualModulesRemoved[env].end())
3248  {
3249  actualModulesModified[env].insert( make_pair(*rem, *add) );
3250  actualModulesRemoved[env].erase(rem);
3251  actualModulesAdded[env].erase(add++);
3252  }
3253  else
3254  ++add;
3255  }
3256  }
3257 
3258  // Reify port types on node classes (if needed).
3259 
3260  bool wereModulesAddedOrModified = false;
3261  for (vector< vector<Environment *> >::iterator i = environments.begin(); i != environments.end(); ++i)
3262  {
3263  Environment *env = (*i).at(0);
3264  if (! (actualModulesAdded[env].empty() && actualModulesModified[env].empty()) )
3265  {
3266  wereModulesAddedOrModified = true;
3267  break;
3268  }
3269  }
3270 
3271  if (wereModulesAddedOrModified)
3272  {
3273  map<string, VuoCompilerType *> inheritedTypes;
3274  for (const vector<Environment *> &envs : environments)
3275  {
3276  for (Environment *env : envs)
3277  {
3278  env->reifyPortTypes(inheritedTypes);
3279  map<string, VuoCompilerType *> envTypes = env->getTypes();
3280  inheritedTypes.insert(envTypes.begin(), envTypes.end());
3281  }
3282  }
3283  }
3284 
3285  // Delete cached compiled module files and call this function recursively for:
3286  // - modules whose source files have been removed
3287  // - modules whose source files depend on them
3288 
3289  for (vector< vector<Environment *> >::iterator i = environments.begin(); i != environments.end(); ++i)
3290  {
3291  Environment *env = (*i).at(0);
3292 
3293  set<string> sourcesToUnload;
3294  sourcesToUnload.insert(sourcesRemoved[env].begin(), sourcesRemoved[env].end());
3295  sourcesToUnload.insert(sourcesDepOnModulesRemoved[env].begin(), sourcesDepOnModulesRemoved[env].end());
3296  if (! sourcesToUnload.empty())
3297  {
3298  string moduleSearchPath = env->getModuleSearchPaths().front();
3299 
3300  dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{
3301  VuoCompiler *otherCompiler = new VuoCompiler(moduleSearchPath + "/unused");
3302 
3303  dispatch_sync(environmentQueue, ^{
3304  otherCompiler->loadModulesAndSources(set<string>(), set<string>(), sourcesToUnload,
3305  set<string>(), set<string>(), set<string>(),
3306  false, false, env, nullptr, nullptr, "");
3307  });
3308 
3309  delete otherCompiler;
3310  });
3311  }
3312 
3313  if (!env->isBuiltIn() && !sourcesToUnload.empty())
3314  VUserLog("Deleting from %s environment: %s", env->getName().c_str(), VuoStringUtilities::join(sourcesToUnload, ", ").c_str());
3315 
3316  env->deleteModulesCompiledFromSourceCode(sourcesToUnload);
3317  }
3318 
3319  // Compile asynchronously:
3320  // - source files that have been added or modified
3321  // - source files that depend on them
3322  // - source files that depend on modules that have been modified
3323 
3324  map<Environment *, set<string> > sourcesDepOnModulesAdded;
3325  {
3326  map<Environment *, set<string> > modulesDepOnModulesAdded; // unused
3327  __block map<Environment *, set<string> > modulesDepOnModulesAdded_otherCompiler; //
3328  __block map<Environment *, set<string> > sourcesDepOnModulesAdded_otherCompiler;
3329 
3330  map<Environment *, set<string> > actualModuleKeysAdded;
3331  for (const vector<Environment *> &envs : environments)
3332  {
3333  Environment *env = envs.at(0);
3334  for (VuoCompilerModule *module : actualModulesAdded[env])
3335  actualModuleKeysAdded[env].insert( module->getPseudoBase()->getModuleKey() );
3336  }
3337 
3338  vector<VuoDirectedAcyclicNetwork *> searchDependencyGraphs;
3339  searchDependencyGraphs.push_back(compositionDependencyGraph);
3340  for (map<string, vector<Environment *> >::iterator ii = environmentsForCompositionFamily.begin(); ii != environmentsForCompositionFamily.end(); ++ii)
3341  {
3342  vector< vector<Environment *> > otherEnvs = sharedEnvironments;
3343  otherEnvs.push_back(ii->second);
3344  VuoDirectedAcyclicNetwork *other = makeDependencyNetwork(otherEnvs, ^VuoDirectedAcyclicGraph * (Environment *env) { return env->getCompositionDependencyGraph(); });
3345  searchDependencyGraphs.push_back(other);
3346  }
3347 
3348  VuoDirectedAcyclicGraph *currentEnvironmentDependencyGraph = (currentEnvironment ? currentEnvironment->getCompositionDependencyGraph() : nullptr);
3349 
3350  findDependentModulesAndSources(actualModuleKeysAdded, searchDependencyGraphs, currentEnvironmentDependencyGraph,
3351  modulesDepOnModulesAdded, modulesDepOnModulesAdded_otherCompiler,
3352  sourcesDepOnModulesAdded, sourcesDepOnModulesAdded_otherCompiler);
3353 
3354  set<Environment *> otherEnvironments;
3355  for (map<Environment *, set<string> >::iterator i = sourcesDepOnModulesAdded_otherCompiler.begin(); i != sourcesDepOnModulesAdded_otherCompiler.end(); ++i)
3356  otherEnvironments.insert(i->first);
3357 
3358  for (Environment *env : otherEnvironments)
3359  {
3360  dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{
3361  string moduleSearchPath = env->getModuleSearchPaths().front();
3362  VuoCompiler *otherCompiler = new VuoCompiler(moduleSearchPath + "/unused");
3363 
3364  dispatch_sync(environmentQueue, ^{
3365  otherCompiler->loadModulesAndSources(set<string>(), set<string>(), set<string>(),
3366  sourcesDepOnModulesAdded_otherCompiler[env], set<string>(), set<string>(),
3367  false, true, env, nullptr, nullptr, "");
3368  });
3369 
3370  delete otherCompiler;
3371  });
3372  }
3373  }
3374 
3375  set<dispatch_group_t> sourcesLoading;
3376  for (vector< vector<Environment *> >::iterator i = environments.begin(); i != environments.end(); ++i)
3377  {
3378  Environment *env = (*i).at(0);
3379 
3380  set<string> sourcesToCompile;
3381  sourcesToCompile.insert(sourcesAdded[env].begin(), sourcesAdded[env].end());
3382  sourcesToCompile.insert(sourcesModified[env].begin(), sourcesModified[env].end());
3383 
3384  if (sourcesToCompile.size() == 0)
3385  continue;
3386 
3387  set<dispatch_group_t> s = env->compileModulesFromSourceCode(sourcesToCompile, shouldRecompileSourcesIfUnchanged);
3388  sourcesLoading.insert(s.begin(), s.end());
3389  }
3390 
3391  for (vector< vector<Environment *> >::iterator i = environments.begin(); i != environments.end(); ++i)
3392  {
3393  Environment *env = (*i).at(0);
3394 
3395  set<string> sourcesToCompile;
3396  sourcesToCompile.insert(sourcesDepOnModulesAdded[env].begin(), sourcesDepOnModulesAdded[env].end());
3397  sourcesToCompile.insert(sourcesDepOnModulesModified[env].begin(), sourcesDepOnModulesModified[env].end());
3398 
3399  if (sourcesToCompile.size() == 0)
3400  continue;
3401 
3402  env->compileModulesFromSourceCode(sourcesToCompile, true);
3403  }
3404 
3405  // Notify compiler delegates.
3406 
3407  for (vector< vector<Environment *> >::iterator i = environments.begin(); i != environments.end(); ++i)
3408  {
3409  Environment *env = (*i).at(0);
3410  env->notifyCompilers(actualModulesAdded[env], actualModulesModified[env], actualModulesRemoved[env], issues[env]);
3411  }
3412 
3413  // Since the dispatch groups for specialized modules are temporary (caller is responsible for releasing them)
3414  // but the dispatch groups for module sources should stay alive as long as the ModuleInfo that contains them,
3415  // retain the dispatch groups for module sources so that all dispatch groups can be safely released by the caller.
3416 
3417  for (const dispatch_group_t &group : sourcesLoading)
3418  dispatch_retain(group);
3419 
3420  set<dispatch_group_t> loadingGroups;
3421  loadingGroups.insert(specializedModulesLoading.begin(), specializedModulesLoading.end());
3422  loadingGroups.insert(sourcesLoading.begin(), sourcesLoading.end());
3423  return loadingGroups;
3424 }
3425 
3437 void VuoCompiler::findDependentModulesAndSources(map<Environment *, set<string> > &changedModules,
3438  const vector<VuoDirectedAcyclicNetwork *> &searchDependencyGraphs,
3439  VuoDirectedAcyclicGraph *currentEnvironmentDependencyGraph,
3440  map<Environment *, set<string> > &modulesDepOnChangedModules_this,
3441  map<Environment *, set<string> > &modulesDepOnChangedModules_other,
3442  map<Environment *, set<string> > &sourcesDepOnChangedModules_this,
3443  map<Environment *, set<string> > &sourcesDepOnChangedModules_other)
3444 {
3445  for (const vector<Environment *> &envs : environments)
3446  {
3447  Environment *env = envs.at(0);
3448 
3449  for (const string &module : changedModules[env])
3450  {
3451  set<VuoDirectedAcyclicGraph::Vertex *> dependents;
3452  for (VuoDirectedAcyclicNetwork *searchDependencyGraph : searchDependencyGraphs)
3453  {
3454  vector<VuoDirectedAcyclicGraph::Vertex *> moduleVertices;
3455  if (currentEnvironmentDependencyGraph)
3456  {
3457  // If a module with the same module key is installed in multiple locations,
3458  // only consider the one being modified or removed.
3459  VuoDirectedAcyclicGraph::Vertex *mv = currentEnvironmentDependencyGraph->findVertex(module);
3460  if (mv)
3461  moduleVertices.push_back(mv);
3462  }
3463  else
3464  moduleVertices = searchDependencyGraph->findVertex(module);
3465 
3466  for (VuoDirectedAcyclicGraph::Vertex *moduleVertexRaw : moduleVertices)
3467  {
3468  DependencyGraphVertex *moduleVertex = static_cast<DependencyGraphVertex *>(moduleVertexRaw);
3469  if (moduleVertex->getEnvironment())
3470  {
3471  vector<VuoDirectedAcyclicGraph::Vertex *> upstream = searchDependencyGraph->getUpstreamVertices(moduleVertex);
3472  dependents.insert(upstream.begin(), upstream.end());
3473  }
3474  }
3475  }
3476 
3477  set< pair<Environment *, string> > dependentsMap;
3478  for (VuoDirectedAcyclicGraph::Vertex *dependentVertexRaw : dependents)
3479  {
3480  DependencyGraphVertex *v = static_cast<DependencyGraphVertex *>(dependentVertexRaw);
3481  Environment *dependentEnv = v->getEnvironment();
3482  if (! dependentEnv)
3483  continue;
3484 
3485  string dependent = v->getDependency();
3486 
3487  dependentsMap.insert({dependentEnv, dependent});
3488  }
3489 
3490  // In case `module` is a generic node class, check the generated environment at the same scope for any
3491  // specializations of the node class, and add them to the list of dependencies.
3492  // (They aren't in the dependency graph since the graph edge goes from installed to generated.)
3493  for (auto i : envs.at(1)->getNodeClasses())
3494  {
3495  VuoCompilerSpecializedNodeClass *specializedNodeClass = dynamic_cast<VuoCompilerSpecializedNodeClass *>(i.second);
3496  if (specializedNodeClass->getOriginalGenericNodeClassName() == module)
3497  dependentsMap.insert({envs.at(1), i.first});
3498  }
3499 
3500  for (auto i : dependentsMap)
3501  {
3502  Environment *dependentEnv = i.first;
3503  string dependent = i.second;
3504 
3505  // Skip if the dependent module is already being modified/removed in its own right
3506  // (e.g. if the module depends on another in the same node set and the node set is being removed).
3507  if (changedModules[dependentEnv].find(dependent) != changedModules[dependentEnv].end())
3508  continue;
3509 
3510  ModuleInfo *foundSourceInfo = dependentEnv->listSourceFile(dependent);
3511  ModuleInfo *foundModuleInfo = dependentEnv->listModule(dependent);
3512 
3513  bool belongsToCurrentCompiler = false;
3514  for (const vector<Environment *> &envs2 : environments)
3515  {
3516  if (find(envs2.begin(), envs2.end(), dependentEnv) != envs2.end())
3517  {
3518  belongsToCurrentCompiler = true;
3519  break;
3520  }
3521  }
3522 
3523  map<Environment *, set<string> > *whicheverDependents = nullptr;
3524  ModuleInfo *moduleInfo = nullptr;
3525  if (foundSourceInfo)
3526  {
3527  moduleInfo = foundSourceInfo;
3528  whicheverDependents = (belongsToCurrentCompiler ? &sourcesDepOnChangedModules_this : &sourcesDepOnChangedModules_other);
3529  }
3530  else if (foundModuleInfo)
3531  {
3532  moduleInfo = foundModuleInfo;
3533  whicheverDependents = (belongsToCurrentCompiler ? &modulesDepOnChangedModules_this : &modulesDepOnChangedModules_other);
3534  }
3535  else // Module in generated environment
3536  {
3537  whicheverDependents = (belongsToCurrentCompiler ? &modulesDepOnChangedModules_this : &modulesDepOnChangedModules_other);
3538  }
3539 
3540  (*whicheverDependents)[dependentEnv].insert(dependent);
3541  if (moduleInfo)
3542  moduleInfo->setAttempted(false);
3543  }
3544  }
3545  }
3546 }
3547 
3551 void VuoCompiler::loadedModules(map<string, VuoCompilerModule *> modulesAdded,
3552  map<string, pair<VuoCompilerModule *, VuoCompilerModule *> > modulesModified,
3553  map<string, VuoCompilerModule *> modulesRemoved,
3554  VuoCompilerIssues *issues, void *delegateDataV, Environment *currentEnvironment)
3555 {
3556  //VLog("C=%p %lu %lu %lu", this, modulesAdded.size(), modulesModified.size(), modulesRemoved.size());
3557 
3558  // If a module is added, superseding a version of the same module installed at a broader scope,
3559  // notify this VuoCompiler that the module has been modified rather than added.
3560  //
3561  // If a module is added, but a version of the same module is already installed at a narrower scope,
3562  // don't notify this VuoCompiler, since it will continue to use the version at the narrower scope.
3563  //
3564  // Same idea when a module is modified or removed while another version is installed at a different scope.
3565 
3566  auto findVersionsOfModule = [this, currentEnvironment] (const string &moduleKey)
3567  {
3568  vector< pair<Environment *, VuoCompilerModule *> > moduleVersions;
3569  for (const vector<Environment *> &envs : environments)
3570  {
3571  Environment *env = envs.at(0);
3572  VuoCompilerModule *module = env->findModule(moduleKey);
3573  if (module || env == currentEnvironment)
3574  moduleVersions.push_back( make_pair(env, module) );
3575  }
3576  return moduleVersions;
3577  };
3578 
3579  for (map<string, VuoCompilerModule *>::iterator i = modulesAdded.begin(); i != modulesAdded.end(); )
3580  {
3581  string moduleKey = i->first;
3582  VuoCompilerModule *moduleAdded = i->second;
3583 
3584  vector< pair<Environment *, VuoCompilerModule *> > moduleVersions = findVersionsOfModule(moduleKey);
3585 
3586  if (moduleVersions.size() > 1)
3587  {
3588  modulesAdded.erase(i++);
3589 
3590  if (moduleVersions.back().second == moduleAdded)
3591  {
3592  VuoCompilerModule *moduleSuperseded = moduleVersions.at(moduleVersions.size()-2).second;
3593  modulesModified[moduleKey] = make_pair(moduleSuperseded, moduleAdded);
3594  }
3595  }
3596  else
3597  ++i;
3598  }
3599 
3600  for (map<string, pair<VuoCompilerModule *, VuoCompilerModule *> >::iterator i = modulesModified.begin(); i != modulesModified.end(); )
3601  {
3602  string moduleKey = i->first;
3603  VuoCompilerModule *moduleModified = i->second.second;
3604 
3605  vector< pair<Environment *, VuoCompilerModule *> > moduleVersions = findVersionsOfModule(moduleKey);
3606 
3607  if (moduleVersions.size() > 1 && moduleVersions.back().second != moduleModified)
3608  modulesModified.erase(i++);
3609  else
3610  ++i;
3611  }
3612 
3613  for (map<string, VuoCompilerModule *>::iterator i = modulesRemoved.begin(); i != modulesRemoved.end(); )
3614  {
3615  string moduleKey = i->first;
3616  VuoCompilerModule *moduleRemoved = i->second;
3617 
3618  vector< pair<Environment *, VuoCompilerModule *> > moduleVersions = findVersionsOfModule(moduleKey);
3619 
3620  if (moduleVersions.size() > 1)
3621  {
3622  modulesRemoved.erase(i++);
3623 
3624  if (moduleVersions.back().first == currentEnvironment)
3625  {
3626  VuoCompilerModule *moduleUnsuperseded = moduleVersions.at(moduleVersions.size()-2).second;
3627  modulesModified[moduleKey] = make_pair(moduleRemoved, moduleUnsuperseded);
3628  }
3629  }
3630  else
3631  ++i;
3632  }
3633 
3634  dispatch_async(delegateQueue, ^{
3635  VuoCompilerDelegate::LoadedModulesData *delegateData = static_cast<VuoCompilerDelegate::LoadedModulesData *>(delegateDataV);
3636 
3637  if (delegate && ! (modulesAdded.empty() && modulesModified.empty() && modulesRemoved.empty() && issues->isEmpty()))
3638  {
3639  delegate->enqueueData(delegateData);
3640  delegate->loadedModules(modulesAdded, modulesModified, modulesRemoved, issues);
3641  }
3642  else
3643  {
3644  delegateData->release();
3645  }
3646  });
3647 }
3648 
3654 void VuoCompiler::loadNodeClassGeneratedAtRuntime(VuoCompilerNodeClass *nodeClass, Environment *env)
3655 {
3656  Module *module = nodeClass->getModule();
3657  if (module)
3658  {
3659  dispatch_sync(llvmQueue, ^{
3660  setTargetForModule(nodeClass->getModule());
3661  });
3662  }
3663 
3664  dispatch_sync(environmentQueue, ^{
3665  env->replaceNodeClass(nodeClass);
3666  });
3667 
3668  __block map<string, VuoCompilerType *> inheritedTypes;
3669  void (^envReifyPortTypes)(Environment *) = ^void (Environment *env) {
3670  env->reifyPortTypes(inheritedTypes);
3671  map<string, VuoCompilerType *> currentTypes = env->getTypes();
3672  inheritedTypes.insert(currentTypes.begin(), currentTypes.end());
3673  };
3674  applyToAllEnvironments(envReifyPortTypes);
3675 }
3676 
3680 void VuoCompiler::reifyGenericPortTypes(VuoCompilerComposition *composition)
3681 {
3682  for (VuoCompilerNode *node : composition->getCachedGraph(this)->getNodes())
3683  reifyGenericPortTypes(node->getBase());
3684 
3685  composition->invalidateCachedGraph();
3686 }
3687 
3691 void VuoCompiler::reifyGenericPortTypes(VuoNode *node)
3692 {
3694  if (! nodeClass)
3695  return;
3696 
3697  // Reify any generic types on the node that don't already have a compiler detail.
3698 
3699  vector<VuoPort *> inputPorts = node->getInputPorts();
3700  vector<VuoPort *> outputPorts = node->getOutputPorts();
3701  vector<VuoPort *> ports;
3702  ports.insert(ports.end(), inputPorts.begin(), inputPorts.end());
3703  ports.insert(ports.end(), outputPorts.begin(), outputPorts.end());
3704 
3705  for (vector<VuoPort *>::iterator j = ports.begin(); j != ports.end(); ++j)
3706  {
3707  VuoCompilerPort *port = static_cast<VuoCompilerPort *>((*j)->getCompiler());
3708  VuoGenericType *genericType = dynamic_cast<VuoGenericType *>(port->getDataVuoType());
3709  if (! genericType)
3710  continue;
3711 
3712  if (! genericType->hasCompiler())
3713  {
3714  VuoCompilerType * (^compilerGetType)(string) = ^VuoCompilerType * (string moduleKey) {
3715  return getType(moduleKey);
3716  };
3717 
3718  VuoCompilerGenericType *reifiedType = VuoCompilerGenericType::newGenericType(genericType, compilerGetType);
3719  if (reifiedType)
3720  port->setDataVuoType(reifiedType->getBase());
3721  }
3722  }
3723 
3724  // Update the node class's backing to match the node's backing.
3725 
3726  nodeClass->updateBackingNodeClass(node, this);
3727 }
3728 
3735 void VuoCompiler::compileModule(string inputPath, string outputPath)
3736 {
3737  compileModule(inputPath, outputPath, vector<string>());
3738 }
3739 
3747 void VuoCompiler::compileModule(string inputPath, string outputPath, const vector<string> &includePaths)
3748 {
3749  if (isVerbose)
3750  print();
3751 
3752  vector<string> allIncludePaths = includePaths;
3753  string preprocessedInputPath = inputPath;
3754 
3755  string tmpPreprocessedInputDir;
3756  string dir, file, ext;
3757  VuoFileUtilities::splitPath(inputPath, dir, file, ext);
3759  {
3760  string inputContents = VuoFileUtilities::readFileToString(inputPath);
3761  string preprocessedInputContents = inputContents;
3763  if (inputContents != preprocessedInputContents)
3764  {
3765  // Unspecialized generic node class
3766  allIncludePaths.push_back(dir.empty() ? "." : dir);
3767  tmpPreprocessedInputDir = VuoFileUtilities::makeTmpDir(file);
3768  preprocessedInputPath = tmpPreprocessedInputDir + "/" + file + "." + ext;
3769  VuoFileUtilities::preserveOriginalFileName(preprocessedInputContents, file + "." + ext);
3770  VuoFileUtilities::writeStringToFile(preprocessedInputContents, preprocessedInputPath);
3771  }
3772  }
3773  else if (ext == "fs")
3774  {
3775  VuoFileUtilities::File vuf(dir, file + "." + ext);
3776  VuoModuleCompiler *moduleCompiler = VuoModuleCompiler::newModuleCompiler("isf", getModuleKeyForPath(inputPath), &vuf);
3777  if (moduleCompiler)
3778  {
3779  auto getType = [this] (const string &moduleKey) { return this->getType(moduleKey); };
3780  VuoCompilerIssues *issues = new VuoCompilerIssues();
3781  Module *module = moduleCompiler->compile(getType, llvmQueue, issues);
3782  if (module)
3783  dispatch_sync(llvmQueue, ^{
3784  writeModuleToBitcode(module, outputPath);
3785  });
3786 
3787  if (!issues->isEmpty())
3788  throw VuoCompilerException(issues, true);
3789  delete issues;
3790  }
3791  return;
3792  }
3793 
3794  vector<string> extraArgs;
3795  for (vector<string>::iterator i = allIncludePaths.begin(); i != allIncludePaths.end(); ++i)
3796  {
3797  extraArgs.push_back("-I");
3798  extraArgs.push_back(*i);
3799  }
3800 
3801  string macosxSdkFolder = "/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/";
3802  if (VuoFileUtilities::fileExists(macosxSdkFolder + "MacOSX10.10.sdk"))
3803  {
3804  extraArgs.push_back("-isysroot");
3805  extraArgs.push_back(macosxSdkFolder + "MacOSX10.10.sdk");
3806  }
3807 
3808  __block vector<string> headerSearchPaths;
3809  void (^envGetHeaderSearchPaths)(Environment *) = ^void (Environment *env) {
3810  vector<string> result = env->getHeaderSearchPaths();
3811  headerSearchPaths.insert(headerSearchPaths.end(), result.begin(), result.end());
3812  };
3813  applyToInstalledEnvironments(envGetHeaderSearchPaths);
3814 
3815  __block Module *module;
3816  dispatch_sync(llvmQueue, ^{
3817  module = readModuleFromC(preprocessedInputPath, headerSearchPaths, extraArgs);
3818  });
3819  string moduleKey = getModuleKeyForPath(inputPath);
3820  if (! tmpPreprocessedInputDir.empty())
3821  remove(tmpPreprocessedInputDir.c_str());
3822  if (! module)
3823  {
3824  VuoCompilerIssue issue(VuoCompilerIssue::Error, "compiling module", inputPath,
3825  "", "%moduleKey couldn't be compiled as a node class, type, or library. Check the macOS Console for details.");
3826  issue.setModuleKey(moduleKey);
3827  throw VuoCompilerException(issue);
3828  }
3829 
3830  dispatch_sync(llvmQueue, ^{
3831  VuoCompilerModule *compilerModule = VuoCompilerModule::newModule(moduleKey, module, "");
3832  if (! compilerModule)
3833  {
3834  VUserLog("Error: Didn't recognize '%s' as a node class, type, or library.", inputPath.c_str());
3835  return;
3836  }
3837 
3838  setTargetForModule(module, target);
3839  writeModuleToBitcode(module, outputPath);
3840 
3841  delete module;
3842  });
3843 }
3844 
3848 Module * VuoCompiler::compileCompositionToModule(VuoCompilerComposition *composition, const string &moduleKey, bool isTopLevelComposition,
3849  VuoCompilerIssues *issues)
3850 {
3851  composition->check(issues);
3852 
3853  reifyGenericPortTypes(composition);
3854 
3856  isTopLevelComposition,
3857  moduleKey, this);
3858  if (telemetry == "console")
3859  generator->setDebugMode(true);
3860 
3861  __block Module *module;
3862  dispatch_sync(llvmQueue, ^{
3863  module = generator->generateBitcode();
3864  setTargetForModule(module, target);
3865  });
3866 
3867  delete generator;
3868 
3869  return module;
3870 }
3871 
3882 void VuoCompiler::compileComposition(VuoCompilerComposition *composition, string outputPath, bool isTopLevelComposition,
3883  VuoCompilerIssues *issues)
3884 {
3885  string moduleKey = getModuleKeyForPath(outputPath);
3886  Module *module = compileCompositionToModule(composition, moduleKey, isTopLevelComposition, issues);
3887 
3888  dispatch_sync(llvmQueue, ^{
3889  writeModuleToBitcode(module, outputPath);
3890  });
3891 }
3892 
3903 void VuoCompiler::compileComposition(string inputPath, string outputPath, bool isTopLevelComposition,
3904  VuoCompilerIssues *issues)
3905 {
3906  VDebugLog("Compiling '%s'…", inputPath.c_str());
3907  if (isVerbose)
3908  print();
3909 
3911  {
3912  VuoCompilerIssue issue(VuoCompilerIssue::Error, "opening composition", inputPath,
3913  "", "The composition file couldn't be read or was empty.");
3914  throw VuoCompilerException(issue);
3915  }
3916 
3917  try
3918  {
3919  string compositionString = VuoFileUtilities::readFileToString(inputPath);
3920  compileCompositionString(compositionString, outputPath, isTopLevelComposition, issues);
3921  }
3922  catch (VuoCompilerException &e)
3923  {
3924  e.getIssues()->setFilePathIfEmpty(inputPath);
3925  throw;
3926  }
3927 
3928  VDebugLog("Done.");
3929 }
3930 
3941 void VuoCompiler::compileCompositionString(const string &compositionString, string outputPath, bool isTopLevelComposition,
3942  VuoCompilerIssues *issues)
3943 {
3945  compileComposition(composition, outputPath, isTopLevelComposition, issues);
3946 
3947  VuoComposition *baseComposition = composition->getBase();
3948  delete composition;
3949  delete baseComposition;
3950 }
3951 
3955 void VuoCompiler::compileSubcompositionString(const string &compositionString, const string &outputPath,
3956  std::function<void(void)> moduleLoadedCallback, Environment *environment,
3957  VuoCompilerIssues *issues, const string inputPathForIssues)
3958 {
3959  if (! issues)
3960  issues = new VuoCompilerIssues();
3961 
3962  bool compilationSucceeded = false;
3963  try
3964  {
3965  compileCompositionString(compositionString, outputPath, false, issues);
3966  compilationSucceeded = true;
3967  }
3968  catch (VuoCompilerException &e)
3969  {
3970  if (issues != e.getIssues())
3971  issues->append(e.getIssues());
3972  }
3973 
3974  if (! compilationSucceeded)
3975  {
3976  VuoFileUtilities::deleteFile(outputPath);
3977  issues->setFilePathIfEmpty(inputPathForIssues);
3978  }
3979 
3980  string outputDir, file, ext;
3981  VuoFileUtilities::splitPath(outputPath, outputDir, file, ext);
3983 
3984  environment->moduleSearchPathContentsChanged(outputDir, outputPath, compositionString, moduleLoadedCallback, this, issues);
3985 }
3986 
4000 void VuoCompiler::linkCompositionToCreateExecutable(string inputPath, string outputPath, Optimization optimization, string rPath)
4001 {
4002  linkCompositionToCreateExecutableOrDynamicLibrary(inputPath, outputPath, optimization, false, rPath);
4003 }
4004 
4021 void VuoCompiler::linkCompositionToCreateDynamicLibrary(string inputPath, string outputPath, Optimization optimization)
4022 {
4023  linkCompositionToCreateExecutableOrDynamicLibrary(inputPath, outputPath, optimization, true);
4024 }
4025 
4040 void VuoCompiler::linkCompositionToCreateExecutableOrDynamicLibrary(string compiledCompositionPath, string linkedCompositionPath,
4041  Optimization optimization, bool isDylib, string rPath)
4042 {
4043  if (isVerbose)
4044  print();
4045 
4046  if (optimization == Optimization_FastBuildExistingCache)
4047  shouldLoadAllModules = false;
4048 
4049  set<string> dependencies = getDependenciesForComposition(compiledCompositionPath);
4050  dependencies.insert(getRuntimeDependency());
4051  if (! isDylib)
4052  dependencies.insert(getRuntimeMainDependency());
4053 
4054  set<Module *> modules;
4055  set<string> libraries;
4056  set<string> frameworks;
4057  getLinkerInputs(dependencies, optimization, modules, libraries, frameworks);
4058 
4059  libraries.insert(compiledCompositionPath);
4060 
4061  link(linkedCompositionPath, modules, libraries, frameworks, isDylib, rPath);
4062 }
4063 
4078 void VuoCompiler::linkCompositionToCreateDynamicLibraries(string compiledCompositionPath, string linkedCompositionPath,
4079  VuoRunningCompositionLibraries *runningCompositionLibraries)
4080 {
4081  if (isVerbose)
4082  print();
4083 
4084  // Get the dependencies used by the new resources and not the previous resources.
4085 
4086  set<string> carriedOverDependencies = runningCompositionLibraries->getDependenciesLoaded();
4087  set<string> allDependencies = getDependenciesForComposition(compiledCompositionPath);
4088  set<string> addedDependencies;
4089  std::set_difference(allDependencies.begin(), allDependencies.end(),
4090  carriedOverDependencies.begin(), carriedOverDependencies.end(),
4091  std::inserter(addedDependencies, addedDependencies.end()));
4092 
4093  // Get the libraries and frameworks used by the previous resources.
4094 
4095  vector<string> carriedOverNonUnloadableLibraries = runningCompositionLibraries->getNonUnloadableLibrariesLoaded();
4096  vector<string> carriedOverUnloadableLibraries = runningCompositionLibraries->getUnloadableLibrariesLoaded();
4097  set<string> carriedOverExternalLibraries = runningCompositionLibraries->getExternalLibraries();
4098  set<string> carriedOverFrameworks = runningCompositionLibraries->getExternalFrameworks();
4099 
4100  // Link the new resource dylibs, if needed.
4101 
4102  string nonUnloadableResourcePath;
4103  string unloadableResourcePath;
4104  set<string> nonUnloadableDependencies;
4105  set<string> unloadableDependencies;
4106  map<string, set<string> > builtInCacheDependencies;
4107  map<string, set<string> > userCacheDependencies;
4108  set<string> builtInLibraries;
4109  set<string> userLibraries;
4110  set<string> addedExternalLibraries;
4111  set<string> addedFrameworks;
4112  set<string> allFrameworks;
4113  if (! addedDependencies.empty())
4114  {
4115  set<string> builtInModuleAndLibraryDependencies;
4116  set<string> userModuleAndLibraryDependencies;
4117  set<Module *> builtInModules;
4118  set<Module *> userModules;
4119 
4120  getLinkerInputs(addedDependencies, Optimization_FastBuild,
4121  builtInModuleAndLibraryDependencies, userModuleAndLibraryDependencies, builtInCacheDependencies, userCacheDependencies,
4122  builtInModules, userModules, builtInLibraries, userLibraries, addedExternalLibraries, addedFrameworks);
4123 
4124  allFrameworks.insert(carriedOverFrameworks.begin(), carriedOverFrameworks.end());
4125  allFrameworks.insert(addedFrameworks.begin(), addedFrameworks.end());
4126 
4127  string dir, linkedCompositionFile, ext;
4128  VuoFileUtilities::splitPath(linkedCompositionPath, dir, linkedCompositionFile, ext);
4129 
4130  if (! builtInModules.empty() || builtInLibraries.size() > builtInCacheDependencies.size())
4131  {
4132  nonUnloadableResourcePath = VuoFileUtilities::makeTmpFile(linkedCompositionFile + "-resource", "dylib");
4133  nonUnloadableDependencies = builtInModuleAndLibraryDependencies;
4134 
4135  set<string> librariesForNonUnloadableResource = builtInLibraries;
4136  librariesForNonUnloadableResource.insert(carriedOverNonUnloadableLibraries.begin(), carriedOverNonUnloadableLibraries.end());
4137  librariesForNonUnloadableResource.insert(carriedOverExternalLibraries.begin(), carriedOverExternalLibraries.end());
4138  librariesForNonUnloadableResource.insert(addedExternalLibraries.begin(), addedExternalLibraries.end());
4139 
4140  link(nonUnloadableResourcePath, builtInModules, librariesForNonUnloadableResource, allFrameworks, true);\
4141 
4142  for (set<string>::iterator i = builtInLibraries.begin(); i != builtInLibraries.end(); )
4143  {
4144  if (! VuoStringUtilities::endsWith(*i, ".dylib"))
4145  builtInLibraries.erase(i++);
4146  else
4147  i++;
4148  }
4149 
4150  for (set<string>::iterator i = addedExternalLibraries.begin(); i != addedExternalLibraries.end(); )
4151  {
4152  if (! VuoStringUtilities::endsWith(*i, ".dylib"))
4153  addedExternalLibraries.erase(i++);
4154  else
4155  i++;
4156  }
4157  }
4158 
4159  if (! userModules.empty() || userLibraries.size() > userCacheDependencies.size())
4160  {
4161  unloadableResourcePath = VuoFileUtilities::makeTmpFile(linkedCompositionFile + "-resource", "dylib");
4162  unloadableDependencies = userModuleAndLibraryDependencies;
4163 
4164  set<string> librariesForUnloadableResource = userLibraries;
4165  librariesForUnloadableResource.insert(builtInLibraries.begin(), builtInLibraries.end());
4166  librariesForUnloadableResource.insert(carriedOverUnloadableLibraries.begin(), carriedOverUnloadableLibraries.end());
4167  librariesForUnloadableResource.insert(carriedOverNonUnloadableLibraries.begin(), carriedOverNonUnloadableLibraries.end());
4168  librariesForUnloadableResource.insert(carriedOverExternalLibraries.begin(), carriedOverExternalLibraries.end());
4169  librariesForUnloadableResource.insert(addedExternalLibraries.begin(), addedExternalLibraries.end());
4170  if (! nonUnloadableResourcePath.empty())
4171  librariesForUnloadableResource.insert(nonUnloadableResourcePath);
4172 
4173  link(unloadableResourcePath, userModules, librariesForUnloadableResource, allFrameworks, true);
4174 
4175  for (set<string>::iterator i = userLibraries.begin(); i != userLibraries.end(); )
4176  {
4177  if (! VuoStringUtilities::endsWith(*i, ".dylib"))
4178  userLibraries.erase(i++);
4179  else
4180  i++;
4181  }
4182  }
4183  }
4184 
4185  // Get the Vuo runtime dependency.
4186 
4187  set<string> vuoRuntimePaths;
4188  {
4189  set<Module *> modules;
4190  set<string> libraries;
4191  set<string> frameworks;
4192 
4193  set<string> dependencies;
4194  dependencies.insert(getRuntimeDependency());
4195  getLinkerInputs(dependencies, Optimization_FastBuild, modules, libraries, frameworks);
4196  vuoRuntimePaths = libraries;
4197  }
4198 
4199  // Link the composition.
4200 
4201  {
4202  set<Module *> modules;
4203  set<string> libraries;
4204 
4205  libraries.insert(compiledCompositionPath);
4206  libraries.insert(carriedOverExternalLibraries.begin(), carriedOverExternalLibraries.end());
4207  libraries.insert(addedExternalLibraries.begin(), addedExternalLibraries.end());
4208  libraries.insert(carriedOverNonUnloadableLibraries.begin(), carriedOverNonUnloadableLibraries.end());
4209  libraries.insert(carriedOverUnloadableLibraries.begin(), carriedOverUnloadableLibraries.end());
4210  libraries.insert(builtInLibraries.begin(), builtInLibraries.end());
4211  libraries.insert(userLibraries.begin(), userLibraries.end());
4212  if (! nonUnloadableResourcePath.empty())
4213  libraries.insert(nonUnloadableResourcePath);
4214  if (! unloadableResourcePath.empty())
4215  libraries.insert(unloadableResourcePath);
4216  libraries.insert(vuoRuntimePaths.begin(), vuoRuntimePaths.end());
4217  link(linkedCompositionPath, modules, libraries, allFrameworks, true);
4218  }
4219 
4220  // Now that we're past the point where an exception can be thrown, update the RunningCompositionLibraries.
4221 
4222  if (! nonUnloadableResourcePath.empty())
4223  runningCompositionLibraries->enqueueResourceLibraryToLoad(nonUnloadableResourcePath, nonUnloadableDependencies, false);
4224 
4225  if (! unloadableResourcePath.empty())
4226  runningCompositionLibraries->enqueueResourceLibraryToLoad(unloadableResourcePath, unloadableDependencies, true);
4227 
4228  for (map<string, set<string> >::iterator i = builtInCacheDependencies.begin(); i != builtInCacheDependencies.end(); ++i)
4229  runningCompositionLibraries->enqueueCacheLibraryToLoad(i->first, i->second, false);
4230 
4231  for (map<string, set<string> >::iterator i = userCacheDependencies.begin(); i != userCacheDependencies.end(); ++i)
4232  runningCompositionLibraries->enqueueCacheLibraryToLoad(i->first, i->second, true);
4233 
4234  runningCompositionLibraries->addExternalFrameworks(addedFrameworks);
4235  runningCompositionLibraries->addExternalLibraries(addedExternalLibraries);
4236 }
4237 
4244 set<string> VuoCompiler::getDependenciesForComposition(const string &compiledCompositionPath)
4245 {
4246  VDebugLog("Gathering dependencies for '%s'…", compiledCompositionPath.c_str());
4247 
4248  // Add the node classes in the top-level composition and their dependencies.
4249  __block set<string> directDependencies;
4250  string moduleKey = getModuleKeyForPath(compiledCompositionPath);
4251  Module *module = readModuleFromBitcode(compiledCompositionPath);
4252  dispatch_sync(llvmQueue, ^{
4253  VuoCompilerModule *compilerModule = VuoCompilerModule::newModule(moduleKey, module, "");
4254  directDependencies = compilerModule->getDependencies();
4255  delete compilerModule;
4256  delete module;
4257  });
4258 
4259  try
4260  {
4261  auto deps = getDependenciesForComposition(directDependencies, true);
4262  VDebugLog("Done.");
4263  return deps;
4264  }
4265  catch (VuoCompilerException &e)
4266  {
4267  e.getIssues()->setFilePathIfEmpty(compiledCompositionPath);
4268  throw;
4269  }
4270 }
4271 
4279 {
4280  set<string> directDependencies;
4281 
4282  set<VuoCompilerNode *> nodes = composition->getCachedGraph(this)->getNodes();
4283  for (VuoCompilerNode *node : nodes)
4284  if (node->getBase()->getNodeClass()->hasCompiler())
4285  directDependencies.insert( node->getBase()->getNodeClass()->getCompiler()->getDependencyName() );
4286 
4287  vector<VuoPublishedPort *> publishedInputPorts = composition->getBase()->getPublishedInputPorts();
4288  vector<VuoPublishedPort *> publishedOutputPorts = composition->getBase()->getPublishedOutputPorts();
4289  vector<VuoPublishedPort *> publishedPorts;
4290  publishedPorts.insert(publishedPorts.end(), publishedInputPorts.begin(), publishedInputPorts.end());
4291  publishedPorts.insert(publishedPorts.end(), publishedOutputPorts.begin(), publishedOutputPorts.end());
4292  for (VuoPublishedPort *publishedPort : publishedPorts)
4293  {
4294  if (publishedPort->getClass()->hasCompiler())
4295  {
4296  VuoType *portType = static_cast<VuoCompilerPortClass *>( publishedPort->getClass()->getCompiler() )->getDataVuoType();
4297  if (portType)
4298  {
4299  string dependency;
4300  VuoGenericType *genericType = dynamic_cast<VuoGenericType *>(portType);
4301  if (genericType)
4302  {
4303  VuoGenericType::Compatibility compatibility;
4304  vector<string> compatibleTypeNames = genericType->getCompatibleSpecializedTypes(compatibility);
4305  dependency = VuoCompilerGenericType::chooseBackingTypeName(portType->getModuleKey(), compatibleTypeNames);
4306  }
4307  else
4308  dependency = portType->getModuleKey();
4309 
4310  directDependencies.insert(dependency);
4311  }
4312  }
4313  }
4314 
4315  return directDependencies;
4316 }
4317 
4324 set<string> VuoCompiler::getDependenciesForComposition(VuoCompilerComposition *composition)
4325 {
4326  set<string> directDependencies = getDirectDependenciesForComposition(composition);
4327  return getDependenciesForComposition(directDependencies, false);
4328 }
4329 
4336 {
4337  __block vector<string> librarySearchPaths;
4338  applyToInstalledEnvironments(^void (Environment *env) {
4339  vector<string> result = env->getLibrarySearchPaths();
4340  librarySearchPaths.insert(librarySearchPaths.end(), result.begin(), result.end());
4341  });
4342 
4343  set<string> dylibDeps;
4344  for (string dep : getDependenciesForComposition(composition))
4345  {
4346  string path = getLibraryPath(dep, librarySearchPaths);
4347  if (VuoStringUtilities::endsWith(path, ".dylib"))
4348  dylibDeps.insert(path);
4349  }
4350 
4351  return dylibDeps;
4352 }
4353 
4367 set<string> VuoCompiler::getDependenciesForComposition(const set<string> &directDependencies, bool checkCompatibility)
4368 {
4369  // Make sure that any compiler-generated node classes have been generated and added to the dependency graph.
4370  for (set<string>::const_iterator i = directDependencies.begin(); i != directDependencies.end(); ++i)
4371  getNodeClass(*i);
4372 
4373  set<string> dependencies;
4374  for (set<string>::iterator i = directDependencies.begin(); i != directDependencies.end(); ++i)
4375  {
4376  string moduleKey = *i;
4377 
4378  dependencies.insert(moduleKey);
4379 
4380  // First pass: Find all dependencies of the direct dependency that have been loaded so far.
4381  vector<VuoDirectedAcyclicGraph::Vertex *> firstPassVertices = dependencyGraph->findVertex(moduleKey);
4382  set<VuoDirectedAcyclicGraph::Vertex *> firstPassDependencies(firstPassVertices.begin(), firstPassVertices.end());
4383  for (vector<VuoDirectedAcyclicGraph::Vertex *>::iterator j = firstPassVertices.begin(); j != firstPassVertices.end(); ++j)
4384  {
4385  vector<VuoDirectedAcyclicGraph::Vertex *> downstream = dependencyGraph->getDownstreamVertices(*j);
4386  firstPassDependencies.insert(downstream.begin(), downstream.end());
4387  }
4388 
4389  // Make sure that any compiler-generated node classes in subcompositions have been generated and added to the dependency graph.
4390  for (set<VuoDirectedAcyclicGraph::Vertex *>::iterator j = firstPassDependencies.begin(); j != firstPassDependencies.end(); ++j)
4391  {
4392  DependencyGraphVertex *v = static_cast<DependencyGraphVertex *>(*j);
4393  getNodeClass(v->getDependency());
4394  }
4395 
4396  // Second pass: Find all dependencies of the direct dependency, this time including dependencies of the node classes just generated.
4397  vector<VuoDirectedAcyclicGraph::Vertex *> moduleVertices = dependencyGraph->findVertex(moduleKey);
4398  set<VuoDirectedAcyclicGraph::Vertex *> moduleDependencies(moduleVertices.begin(), moduleVertices.end());
4399  for (vector<VuoDirectedAcyclicGraph::Vertex *>::iterator j = moduleVertices.begin(); j != moduleVertices.end(); ++j)
4400  {
4401  vector<VuoDirectedAcyclicGraph::Vertex *> downstream = dependencyGraph->getDownstreamVertices(*j);
4402  moduleDependencies.insert(downstream.begin(), downstream.end());
4403  }
4404 
4405  // Sort the direct dependency and all of its dependencies into those that are and are not compatible.
4406  set<string> dependenciesToAdd;
4407  set<string> incompatibleDependencies;
4408  for (set<VuoDirectedAcyclicGraph::Vertex *>::iterator j = moduleDependencies.begin(); j != moduleDependencies.end(); ++j)
4409  {
4410  DependencyGraphVertex *v = static_cast<DependencyGraphVertex *>(*j);
4411  if (! checkCompatibility || ! v->getEnvironment() || v->isCompatible())
4412  dependenciesToAdd.insert(v->getDependency());
4413  else
4414  incompatibleDependencies.insert(v->getDependency());
4415  }
4416 
4417  if (! checkCompatibility || incompatibleDependencies.empty())
4418  {
4419  dependencies.insert(dependenciesToAdd.begin(), dependenciesToAdd.end());
4420  }
4421  else
4422  {
4423  VuoCompilerTargetSet compositionTargets;
4424  compositionTargets.restrictToCurrentOperatingSystemVersion();
4425 
4426  string dependencyTargetString;
4427  VuoCompilerModule *module = getModule(moduleKey);
4428  if (module)
4429  {
4430  VuoCompilerTargetSet dependencyTargets = module->getCompatibleTargets();
4431  for (set<string>::iterator i = incompatibleDependencies.begin(); i != incompatibleDependencies.end(); ++i)
4432  {
4433  VuoCompilerModule *subModule = getModule(*i);
4434  if (subModule)
4435  {
4436  VuoCompilerTargetSet subDependencyTargets = subModule->getCompatibleTargets();
4437  dependencyTargets.restrictToBeCompatibleWithAllOf(subDependencyTargets);
4438  }
4439  }
4440  dependencyTargetString = dependencyTargets.toString();
4441  }
4442  else
4443  dependencyTargetString = "(unknown operating systems)";
4444 
4445  string modulePlaceholder = (module ? "%module" : "%moduleKey");
4446  VuoCompilerIssue issue(VuoCompilerIssue::Error, "linking composition", "",
4447  "Node incompatible with operating system",
4448  modulePlaceholder + " is only compatible with " + dependencyTargetString +
4449  ", so this composition can't run on your macOS version (" + compositionTargets.toString() + ").");
4450  issue.setModule(module->getPseudoBase());
4451  issue.setModuleKey(moduleKey);
4452  throw VuoCompilerException(issue);
4453  }
4454  }
4455 
4456  // Add the libraries needed by every linked composition.
4457  vector<string> coreDependencies = getCoreVuoDependencies();
4458  dependencies.insert(coreDependencies.begin(), coreDependencies.end());
4459 
4460  return dependencies;
4461 }
4462 
4467 void VuoCompiler::getLinkerInputs(const set<string> &dependencies, Optimization optimization,
4468  set<Module *> &modules, set<string> &libraries, set<string> &frameworks)
4469 {
4470  set<string> builtInModuleAndLibraryDependencies;
4471  set<string> userModuleAndLibraryDependencies;
4472  map<string, set<string> > builtInCacheDependencies;
4473  map<string, set<string> > userCacheDependencies;
4474  set<Module *> builtInModules;
4475  set<Module *> userModules;
4476  set<string> builtInLibraries;
4477  set<string> userLibraries;
4478  set<string> externalLibraries;
4479 
4480  getLinkerInputs(dependencies, optimization,
4481  builtInModuleAndLibraryDependencies, userModuleAndLibraryDependencies, builtInCacheDependencies, userCacheDependencies,
4482  builtInModules, userModules, builtInLibraries, userLibraries, externalLibraries, frameworks);
4483 
4484  modules.insert(builtInModules.begin(), builtInModules.end());
4485  modules.insert(userModules.begin(), userModules.end());
4486  libraries.insert(builtInLibraries.begin(), builtInLibraries.end());
4487  libraries.insert(userLibraries.begin(), userLibraries.end());
4488  libraries.insert(externalLibraries.begin(), externalLibraries.end());
4489 }
4490 
4504 void VuoCompiler::getLinkerInputs(const set<string> &dependencies, Optimization optimization,
4505  set<string> &builtInModuleAndLibraryDependencies, set<string> &userModuleAndLibraryDependencies,
4506  map<string, set<string> > &builtInCacheDependencies, map<string, set<string> > &userCacheDependencies,
4507  set<Module *> &builtInModules, set<Module *> &userModules,
4508  set<string> &builtInLibraries, set<string> &userLibraries,
4509  set<string> &externalLibraries, set<string> &externalFrameworks)
4510 {
4511  bool shouldUseModuleCache = (optimization == Optimization_FastBuild || optimization == Optimization_FastBuildExistingCache);
4512  if (shouldUseModuleCache)
4513  useModuleCache(true, optimization == Optimization_FastBuildExistingCache);
4514 
4515  __block vector<string> librarySearchPaths;
4516  void (^envGetLibrarySearchPaths)(Environment *) = ^void (Environment *env) {
4517  vector<string> result = env->getLibrarySearchPaths();
4518  librarySearchPaths.insert(librarySearchPaths.end(), result.begin(), result.end());
4519  };
4520  applyToInstalledEnvironments(envGetLibrarySearchPaths);
4521 
4522  for (set<string>::iterator i = dependencies.begin(); i != dependencies.end(); ++i)
4523  {
4524  string dependency = *i;
4525 
4526  bool foundInCache = false;
4527  string moduleCachePath;
4528  bool isInBuiltInModuleCache = false;
4529  if (shouldUseModuleCache)
4530  foundInCache = findInModuleCache(dependency, moduleCachePath, isInBuiltInModuleCache);
4531 
4532  if (foundInCache)
4533  {
4534  if (isInBuiltInModuleCache)
4535  {
4536  builtInLibraries.insert(moduleCachePath);
4537  builtInCacheDependencies[moduleCachePath].insert(dependency);
4538  }
4539  else
4540  {
4541  userLibraries.insert(moduleCachePath);
4542  userCacheDependencies[moduleCachePath].insert(dependency);
4543  }
4544  }
4545  else
4546  {
4547  __block VuoCompilerModule *module = NULL;
4548  void (^envFindModule)(Environment *) = ^void (Environment *env) {
4549  VuoCompilerModule *result = env->findModule(dependency);
4550  if (result)
4551  module = result;
4552  };
4553  applyToAllEnvironments(envFindModule);
4554 
4555  if (module)
4556  {
4557  VuoCompilerNodeClass *nodeClass = dynamic_cast<VuoCompilerNodeClass *>(module);
4558  if (! (nodeClass && VuoCompilerSpecializedNodeClass::hasGenericPortTypes(nodeClass)) ) // Skip not-fully-specialized generic modules
4559  {
4560  string modulePath = module->getModulePath();
4561  if (! modulePath.empty() && dynamic_cast<VuoCompilerType *>(module))
4562  {
4563  if (module->isBuiltIn())
4564  builtInLibraries.insert(modulePath);
4565  else
4566  userLibraries.insert(modulePath);
4567  }
4568  else
4569  {
4570  if (module->isBuiltIn())
4571  builtInModules.insert(module->getModule());
4572  else
4573  userModules.insert(module->getModule());
4574  }
4575 
4576  if (module->isBuiltIn())
4577  builtInModuleAndLibraryDependencies.insert(dependency);
4578  else
4579  userModuleAndLibraryDependencies.insert(dependency);
4580  }
4581  }
4582  else
4583  {
4584  if (VuoStringUtilities::endsWith(dependency, ".framework"))
4585  externalFrameworks.insert(dependency);
4586  else
4587  {
4588  string dependencyPath = getLibraryPath(dependency, librarySearchPaths);
4589  if (! dependencyPath.empty())
4590  externalLibraries.insert(dependencyPath);
4591  else
4592  VUserLog("Warning: Could not locate dependency '%s'.", dependency.c_str());
4593  }
4594  }
4595  }
4596  }
4597 }
4598 
4604 string VuoCompiler::getLibraryPath(const string &dependency, vector<string> librarySearchPaths)
4605 {
4606  if (dependency[0] == '/' && VuoFileUtilities::fileExists(dependency))
4607  return dependency;
4608 
4609  // Put the system library folder last in the list — prefer to use the libraries bundled in Vuo.framework.
4610  // Don't attempt to use OpenSSL from the system library folder, since macOS 10.15 prevents linking to it.
4611  if (dependency != "crypto"
4612  && dependency != "ssl")
4613  librarySearchPaths.push_back("/usr/lib");
4614 
4615  for (auto &path : librarySearchPaths)
4616  {
4617  vector<string> variations;
4618  variations.push_back(path + "/" + dependency);
4619  variations.push_back(path + "/lib" + dependency);
4620  variations.push_back(path + "/lib" + dependency + ".dylib");
4621  variations.push_back(path + "/lib" + dependency + ".a");
4622  for (auto &variation : variations)
4623  if (VuoFileUtilities::fileExists(variation))
4624  return variation;
4625  }
4626 
4627  return "";
4628 }
4629 
4637 void VuoCompiler::useModuleCache(bool shouldUseExistingBuiltInCaches, bool shouldUseExistingOtherCaches)
4638 {
4639  loadModulesIfNeeded();
4640 
4641  // Iterate through the environments in the order that the caches need to be built.
4642 
4643  dispatch_sync(environmentQueue, ^{
4644  set<string> dylibsForCachesOfInstalledModules;
4645  set<string> frameworksForCachesOfInstalledModules;
4646  for (vector< vector<Environment *> >::iterator i = environments.begin(); i != environments.end(); ++i)
4647  {
4648  bool builtin = (i == environments.begin());
4649  set<string> dylibsForCacheOfGeneratedModules;
4650  set<string> frameworksForCacheOfGeneratedModules;
4651 
4652  for (int j = i->size() - 1; j >= 0; --j)
4653  {
4654  Environment *env = i->at(j);
4655  bool installed = (j == 0);
4656 
4657  set<string> cacheableModulesAndDependencies;
4658  set<string> dylibsNeededToLinkToThisCache;
4659  set<string> frameworksNeededToLinkToThisCache;
4660  env->getCacheableModulesAndDependencies(builtin, installed, cacheableModulesAndDependencies,
4661  dylibsNeededToLinkToThisCache, frameworksNeededToLinkToThisCache);
4662 
4663  set<string> accumulatedDylibs;
4664  accumulatedDylibs.insert(dylibsNeededToLinkToThisCache.begin(), dylibsNeededToLinkToThisCache.end());
4665  accumulatedDylibs.insert(dylibsForCachesOfInstalledModules.begin(), dylibsForCachesOfInstalledModules.end());
4666  accumulatedDylibs.insert(dylibsForCacheOfGeneratedModules.begin(), dylibsForCacheOfGeneratedModules.end());
4667 
4668  set<string> accumulatedFrameworks;
4669  accumulatedFrameworks.insert(frameworksNeededToLinkToThisCache.begin(), frameworksNeededToLinkToThisCache.end());
4670  accumulatedFrameworks.insert(frameworksForCachesOfInstalledModules.begin(), frameworksForCachesOfInstalledModules.end());
4671  accumulatedFrameworks.insert(frameworksForCacheOfGeneratedModules.begin(), frameworksForCacheOfGeneratedModules.end());
4672 
4673  bool shouldUseExistingCache = (builtin ? shouldUseExistingBuiltInCaches : shouldUseExistingOtherCaches);
4674  env->useModuleCache(shouldUseExistingCache, this, cacheableModulesAndDependencies,
4675  accumulatedDylibs, accumulatedFrameworks);
4676 
4677  if (installed)
4678  {
4679  dylibsForCachesOfInstalledModules.insert(dylibsNeededToLinkToThisCache.begin(), dylibsNeededToLinkToThisCache.end());
4680  frameworksForCachesOfInstalledModules.insert(frameworksNeededToLinkToThisCache.begin(), frameworksNeededToLinkToThisCache.end());
4681  }
4682  else
4683  {
4684  dylibsForCacheOfGeneratedModules.insert(dylibsNeededToLinkToThisCache.begin(), dylibsNeededToLinkToThisCache.end());
4685  frameworksForCacheOfGeneratedModules.insert(frameworksNeededToLinkToThisCache.begin(), frameworksNeededToLinkToThisCache.end());
4686  }
4687  }
4688  }
4689  });
4690 
4691  Environment::waitForModuleCachesToBuild();
4692 }
4693 
4702 bool VuoCompiler::findInModuleCache(const string &moduleOrDependency, string &cachePath, bool &isBuiltinCache)
4703 {
4704  __block bool found = false;
4705  __block string outPath;
4706  __block bool outBuiltin;
4707  dispatch_sync(environmentQueue, ^{
4708  for (vector< vector<Environment *> >::iterator i = environments.begin(); i != environments.end(); ++i)
4709  {
4710  bool builtin = (i == environments.begin());
4711 
4712  for (int j = i->size() - 1; j >= 0; --j)
4713  {
4714  Environment *env = i->at(j);
4715 
4716  string resultPath;
4717  bool resultFound = env->findInModuleCache(moduleOrDependency, resultPath);
4718  if (resultFound)
4719  {
4720  found = true;
4721  outPath = resultPath;
4722  outBuiltin = builtin;
4723  }
4724  }
4725  }
4726  });
4727 
4728  cachePath = outPath;
4729  isBuiltinCache = outBuiltin;
4730  return found;
4731 }
4732 
4741 {
4742  dispatch_group_async(moduleCacheBuilding, dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{
4743  useModuleCache(true, false);
4744  });
4745 }
4746 
4759 void VuoCompiler::generateBuiltInModuleCaches(const string &vuoFrameworkPath)
4760 {
4761  vuoFrameworkInProgressPath = vuoFrameworkPath;
4762 
4763  VuoCompiler compiler;
4764  compiler.useModuleCache(false, true);
4765 }
4766 
4775 {
4776  unsigned long maxSeconds = 30 * 24 * 60 * 60; // 30 days
4777 
4778  set<VuoFileUtilities::File *> cacheDirs = VuoFileUtilities::findAllFilesInDirectory(VuoFileUtilities::getCachePath());
4779  for (set<VuoFileUtilities::File *>::iterator i = cacheDirs.begin(); i != cacheDirs.end(); ++i)
4780  {
4781  string path = (*i)->path();
4782 
4783  string file = (*i)->basename();
4784  if (file != "Builtin" && file != "System" && file != "User")
4785  {
4786  unsigned long fileSeconds = VuoFileUtilities::getSecondsSinceFileLastAccessed(path);
4787  if (fileSeconds > maxSeconds)
4789  }
4790 
4791  if (VuoStringUtilities::beginsWith(file, Environment::pidCacheDirPrefix))
4792  {
4793  string pidAsString = file.substr(Environment::pidCacheDirPrefix.length());
4794  int pid = atoi(pidAsString.c_str());
4795  if (kill(pid, 0) != 0) // no running process has this pid
4797  }
4798 
4799  delete *i;
4800  }
4801 }
4802 
4807 void VuoCompiler::setLoadAllModules(bool shouldLoadAllModules)
4808 {
4809  dispatch_sync(modulesToLoadQueue, ^{
4810  this->shouldLoadAllModules = shouldLoadAllModules;
4811  });
4812 }
4813 
4825 void VuoCompiler::link(string outputPath, const set<Module *> &modules, const set<string> &libraries, const set<string> &frameworks, bool isDylib, string rPath)
4826 {
4827  VDebugLog("Linking '%s'…", outputPath.c_str());
4828  // https://stackoverflow.com/questions/11657529/how-to-generate-an-executable-from-an-llvmmodule
4829 
4830 
4831  // Write all the modules with renamed symbols to a composite module file (since the linker can't operate on in-memory modules).
4832  string compositeModulePath = VuoFileUtilities::makeTmpFile("composite", "bc");
4833  dispatch_sync(llvmQueue, ^{
4834  double t0 = VuoLogGetTime();
4835  Module *compositeModule = new Module("composite", getGlobalContext());
4836  setTargetForModule(compositeModule);
4837  for (set<Module *>::const_iterator i = modules.begin(); i != modules.end(); ++i)
4838  {
4839  string error;
4840  if (Linker::LinkModules(compositeModule, *i, Linker::PreserveSource, &error))
4841  VUserLog("Error: Failed to link compositeModule: %s", error.c_str());
4842  }
4843  writeModuleToBitcode(compositeModule, compositeModulePath);
4844  delete compositeModule;
4845  VDebugLog("\tLinkModules took %5.2fs", VuoLogGetTime() - t0);
4846  });
4847 
4848 
4849  // llvm-3.1/llvm/tools/clang/tools/driver/driver.cpp
4850 
4851  llvm::sys::Path clangPath = getClangPath();
4852 
4853  vector<const char *> args;
4854  vector<char *> argsToFree;
4855  args.push_back(clangPath.c_str());
4856 
4857  args.push_back(compositeModulePath.c_str());
4858 
4859  vector<string> coreDependencies = getCoreVuoDependencies();
4860  for (set<string>::const_iterator i = libraries.begin(); i != libraries.end(); ++i)
4861  {
4862  string library = *i;
4863 
4864  for (vector<string>::iterator j = coreDependencies.begin(); j != coreDependencies.end(); ++j)
4865  {
4866  string coreDependency = *j;
4867  if (VuoStringUtilities::endsWith(library, "lib" + coreDependency + ".a"))
4868  args.push_back("-force_load"); // Load all symbols of static core dependencies, not just those used in the objects.
4869  }
4870 
4871  // Use the pre-built native object file if it exists (faster than converting .bc to .o every build).
4872  if (VuoStringUtilities::endsWith(library, ".bc"))
4873  {
4874  string libraryObject = VuoStringUtilities::substrBefore(library, ".bc") + ".o";
4875  if (VuoFileUtilities::fileExists(libraryObject))
4876  library = libraryObject;
4877  }
4878 
4879  char *libraryZ = strdup(library.c_str());
4880  args.push_back(libraryZ);
4881  argsToFree.push_back(libraryZ);
4882  }
4883 
4884  // Add framework search paths
4885  vector<string> frameworkArguments;
4886 
4887  __block vector<string> frameworkSearchPaths;
4888  void (^envGetFrameworkSearchPaths)(Environment *) = ^void (Environment *env) {
4889  vector<string> result = env->getFrameworkSearchPaths();
4890  frameworkSearchPaths.insert(frameworkSearchPaths.end(), result.begin(), result.end());
4891  };
4892  applyToInstalledEnvironments(envGetFrameworkSearchPaths);
4893 
4894  for (vector<string>::const_iterator i = frameworkSearchPaths.begin(); i != frameworkSearchPaths.end(); ++i)
4895  {
4896  string a = "-F"+*i;
4897  // Keep these std::strings around until after args is done being used, since the std::string::c_str() is freed when the std::string is deleted.
4898  frameworkArguments.push_back(a);
4899  char *frameworkArgument = strdup(a.c_str());
4900  args.push_back(frameworkArgument);
4901  argsToFree.push_back(frameworkArgument);
4902  }
4903 
4904  for (set<string>::const_iterator i = frameworks.begin(); i != frameworks.end(); ++i)
4905  {
4906  args.push_back("-framework");
4907 
4908  string frameworkName = *i;
4909  frameworkName = frameworkName.substr(0, frameworkName.length() - string(".framework").length());
4910  char *frameworkNameZ = strdup(frameworkName.c_str());
4911  args.push_back(frameworkNameZ);
4912  argsToFree.push_back(frameworkNameZ);
4913  }
4914 
4915  // Check for C Runtime path within Vuo.framework
4916  llvm::sys::Path cRuntimePath;
4917  llvm::sys::Path crt1Path;
4918  string vuoFrameworkPath = getVuoFrameworkPath();
4919  string vuoFrameworkContainingFolder = vuoFrameworkPath + "/..";
4920  if (! vuoFrameworkPath.empty())
4921  {
4922  cRuntimePath = vuoFrameworkPath + "/Modules/";
4923  crt1Path = cRuntimePath;
4924  crt1Path.appendComponent("crt1.o");
4925  }
4926 
4927  // If we have located a bundled version of crt1.o, link it in explicitly rather than relying on
4928  // clang's heuristic to locate a system version.
4929  if (!isDylib && crt1Path.canRead())
4930  {
4931  args.push_back("-nostartfiles");
4932  args.push_back(crt1Path.c_str());
4933  }
4934 
4935  // Linker option necessary for compatibility with our bundled version of ld64:
4936  args.push_back("-Xlinker");
4937  args.push_back("--no-demangle");
4938 
4939  if (isVerbose)
4940  args.push_back("-v");
4941 
4942  if (isDylib)
4943  args.push_back("-dynamiclib");
4944 
4945  args.push_back("-Xlinker");
4946  args.push_back("-headerpad_max_install_names");
4947 
4948  // Tell the built dylib/executable where to find Vuo.framework
4950  args.push_back("-rpath");
4951  string rPathArg = (rPath.empty() ? vuoFrameworkContainingFolder : rPath);
4952  args.push_back(rPathArg.c_str());
4953 
4954 #ifdef COVERAGE
4955  args.push_back("-rpath");
4956  args.push_back(LLVM_ROOT "/lib");
4957 #endif
4958 
4959  args.push_back("-std=c++11");
4960  args.push_back("-stdlib=libc++");
4961 
4962  // Allow clang to print meaningful error messages.
4963  clang::DiagnosticOptions *diagOptions = new clang::DiagnosticOptions();
4964  clang::TextDiagnosticPrinter *diagClient = new clang::TextDiagnosticPrinter(llvm::errs(), diagOptions);
4965  diagClient->setPrefix(clangPath.str());
4966  IntrusiveRefCntPtr<clang::DiagnosticIDs> DiagID(new clang::DiagnosticIDs());
4967  clang::DiagnosticsEngine Diags(DiagID, diagOptions, diagClient);
4968 
4969  clang::driver::Driver TheDriver(args[0], "x86_64-apple-macosx10.10.0", outputPath, Diags);
4970 
4971  TheDriver.CCCIsCXX = true; // clang++ instead of clang
4972 
4973  if (isVerbose)
4974  {
4975  ostringstream s;
4976  for (vector<const char *>::iterator i = args.begin(); i != args.end(); ++i)
4977  s << *i << " ";
4978  VUserLog("\t%s", s.str().c_str());
4979  }
4980 
4981  OwningPtr<clang::driver::Compilation> C(TheDriver.BuildCompilation(args));
4982 
4983  int Res = 0;
4984  if (C)
4985  {
4986  SmallVector<std::pair<int, const clang::driver::Command *>, 4> FailingCommands;
4987  double t0 = VuoLogGetTime();
4988  Res = TheDriver.ExecuteCompilation(*C, FailingCommands);
4989  VDebugLog("\tLinking took %5.2fs", VuoLogGetTime() - t0);
4990  }
4991  for (auto i : argsToFree)
4992  free(i);
4993 
4994  // Clean up composite module file.
4995  remove(compositeModulePath.c_str());
4996 
4997  if (!isDylib)
4998  // Ensure the linked binary has the execute permission set
4999  // (ld64-242 doesn't reliably set it, whereas ld64-133.3 did).
5000  // https://b33p.net/kosada/node/14152
5001  chmod(outputPath.c_str(), 0755);
5002 
5003  if (Res != 0)
5004  {
5005  __block vector<string> thirdPartyNodeClasses;
5006  dispatch_sync(environmentQueue, ^{
5007  for (size_t i = 1; i < environments.size(); ++i)
5008  {
5009  map<string, VuoCompilerNodeClass *> envNodeClasses = environments[i].at(0)->getNodeClasses();
5010  for (map<string, VuoCompilerNodeClass *>::iterator j = envNodeClasses.begin(); j != envNodeClasses.end(); ++j)
5011  thirdPartyNodeClasses.push_back(j->first);
5012  }
5013  });
5014 
5015  string details = "One or more nodes in this composition can't be used by this version of Vuo. ";
5016  if (! thirdPartyNodeClasses.empty())
5017  {
5018  details += "Make sure you're using the latest version of all the extra Vuo nodes you've installed:\n";
5019  sort(thirdPartyNodeClasses.begin(), thirdPartyNodeClasses.end());
5020  for (vector<string>::iterator i = thirdPartyNodeClasses.begin(); i != thirdPartyNodeClasses.end(); ++i)
5021  details += " • " + *i + "\n";
5022  }
5023  details += "Check the macOS Console for more information about the problem.";
5024 
5025  VuoCompilerIssue issue(VuoCompilerIssue::Error, "linking", outputPath,
5026  "Node broken or outdated", details);
5027  throw VuoCompilerException(issue);
5028  }
5029  VDebugLog("Done.");
5030 }
5031 
5037 Module * VuoCompiler::readModuleFromC(string inputPath, const vector<string> &headerSearchPaths, const vector<string> &extraArgs)
5038 {
5039  // llvm-3.1/llvm/tools/clang/examples/clang-interpreter/main.cpp
5040 
5041  vector<const char *> args;
5042  args.push_back(inputPath.c_str());
5043  args.push_back("-DVUO_COMPILER");
5044  args.push_back("-fblocks");
5045 
5046  // Sync with /CMakeLists.txt's `commonFlags`.
5047  args.push_back("-Wall");
5048  args.push_back("-Wextra");
5049  args.push_back("-Wimplicit-fallthrough");
5050  args.push_back("-Wno-unused-parameter");
5051  args.push_back("-Wno-c++11-extensions");
5052  args.push_back("-Wno-sign-compare");
5053  args.push_back("-Werror=implicit");
5054 
5055  if (VuoStringUtilities::endsWith(inputPath, ".cc"))
5056  {
5057  args.push_back("-std=c++11");
5058  args.push_back("-stdlib=libc++");
5059  }
5060 
5061  for (vector<string>::const_iterator i = headerSearchPaths.begin(); i != headerSearchPaths.end(); ++i)
5062  {
5063  args.push_back("-I");
5064  args.push_back(i->c_str());
5065  }
5066 
5067  if (isVerbose)
5068  args.push_back("-v");
5069 
5070  for (vector<string>::const_iterator i = extraArgs.begin(); i != extraArgs.end(); ++i)
5071  args.push_back(i->c_str());
5072 
5073  clang::DiagnosticOptions * diagOptions = new clang::DiagnosticOptions();
5074  IntrusiveRefCntPtr<clang::DiagnosticIDs> DiagID(new clang::DiagnosticIDs());
5075  clang::DiagnosticsEngine Diags(DiagID, diagOptions);
5076 
5077  OwningPtr<clang::CompilerInvocation> CI(new clang::CompilerInvocation);
5078  clang::CompilerInvocation::CreateFromArgs(*CI, &args[0], &args[0] + args.size(), Diags);
5079 
5080  clang::CompilerInstance Clang;
5081  Clang.setInvocation(CI.take());
5082 
5083  Clang.createDiagnostics();
5084  if (!Clang.hasDiagnostics())
5085  return NULL;
5086 
5087  // See CompilerInvocation::GetResourcesPath -- though we're not calling it because we don't have MainAddr.
5088  llvm::sys::Path builtinHeaderSearchPath;
5089  string vuoFrameworkPath = getVuoFrameworkPath();
5090  if (vuoFrameworkPath.empty())
5091  {
5092  llvm::sys::Path clangPath = getClangPath();
5093  builtinHeaderSearchPath = clangPath;
5094  builtinHeaderSearchPath.eraseComponent(); // Remove /clang from foo/bin/clang
5095  builtinHeaderSearchPath.eraseComponent(); // Remove /bin from foo/bin
5096  builtinHeaderSearchPath.appendComponent("lib");
5097  builtinHeaderSearchPath.appendComponent("clang");
5098  builtinHeaderSearchPath.appendComponent(CLANG_VERSION_STRING); // foo/lib/clang/<version>
5099  }
5100  else
5101  {
5102  builtinHeaderSearchPath = vuoFrameworkPath;
5103  builtinHeaderSearchPath.appendComponent("Frameworks");
5104  builtinHeaderSearchPath.appendComponent("llvm.framework");
5105  builtinHeaderSearchPath.appendComponent("Versions");
5106  builtinHeaderSearchPath.appendComponent("A");
5107  builtinHeaderSearchPath.appendComponent("lib");
5108  builtinHeaderSearchPath.appendComponent("clang");
5109  builtinHeaderSearchPath.appendComponent(CLANG_VERSION_STRING); // foo/lib/clang/<version>
5110  }
5111  Clang.getHeaderSearchOpts().ResourceDir = builtinHeaderSearchPath.str();
5112 
5113 // OwningPtr<clang::CodeGenAction> Act(new clang::EmitLLVMOnlyAction()); // @@@ return value of takeModule() is destroyed at the end of this function
5114  clang::CodeGenAction *Act = new clang::EmitLLVMOnlyAction();
5115  if (!Clang.ExecuteAction(*Act))
5116  return NULL;
5117 
5118  return Act->takeModule();
5119 }
5120 
5126 Module * VuoCompiler::readModuleFromBitcode(string inputPath)
5127 {
5128  string dir, file, ext;
5129  VuoFileUtilities::splitPath(inputPath, dir, file, ext);
5130  VuoFileUtilities::File inputFile(dir, file + "." + ext);
5131  return readModuleFromBitcode(&inputFile);
5132 }
5133 
5141 Module * VuoCompiler::readModuleFromBitcode(VuoFileUtilities::File *inputFile)
5142 {
5143  size_t inputDataBytes;
5144  char *inputData = inputFile->getContentsAsRawData(inputDataBytes);
5145 
5146  string error;
5147  VuoLog_status("Loading module \"%s\"", inputFile->getRelativePath().c_str());
5148  Module *module = readModuleFromBitcodeData(inputData, inputDataBytes, error);
5149  VuoLog_status(NULL);
5150  if (! module)
5151  VUserLog("Error: Couldn't parse module '%s': %s.", inputFile->getRelativePath().c_str(), error.c_str());
5152 
5153  free(inputData);
5154 
5155  return module;
5156 }
5157 
5163 Module * VuoCompiler::readModuleFromBitcodeData(char *inputData, size_t inputDataBytes, string &error)
5164 {
5165  __block Module *module;
5166  dispatch_sync(llvmQueue, ^{
5167  StringRef inputDataAsStringRef(inputData, inputDataBytes);
5168  MemoryBuffer *mb = MemoryBuffer::getMemBuffer(inputDataAsStringRef, "", false);
5169  module = ParseBitcodeFile(&(*mb), getGlobalContext(), &error);
5170  delete mb;
5171  });
5172  return module;
5173 }
5174 
5182 bool VuoCompiler::writeModuleToBitcode(Module *module, string outputPath)
5183 {
5184  if (verifyModule(*module, PrintMessageAction))
5185  {
5186  VUserLog("Error: Module verification failed.");
5187  return true;
5188  }
5189 
5190  string err;
5191  raw_fd_ostream out(outputPath.c_str(), err);
5192  if (! err.empty())
5193  {
5194  VUserLog("Error: Couldn't open file '%s' for writing: %s", outputPath.c_str(), err.c_str());
5195  return true;
5196  }
5197  WriteBitcodeToFile(module, out);
5198 
5199  return false;
5200 }
5201 
5207 void VuoCompiler::setTargetForModule(Module *module, string target)
5208 {
5209 /*
5210  string effectiveTarget = target;
5211  if (effectiveTarget.empty())
5212  {
5213  // llvm::sys::getDefaultTargetTriple() finds a target based on the host, but the "default" target is not necessarily the
5214  // same target that results from invoking command-line clang without a -target argument. That is the "effective" target.
5215  // For example, the "default" target could be x86_64-apple-darwin10.10.0 and the "effective" target could be x86_64-apple-macosx10.10.0.
5216 
5217  llvm::sys::Path clangPath = getClangPath();
5218 
5219  vector<const char *> args;
5220  args.push_back(clangPath.c_str());
5221  args.push_back("/bin/sh"); // Driver needs an input file (that exists) or it refuses to give you the correct effective target.
5222 
5223  clang::DiagnosticOptions * diagOptions = new clang::DiagnosticOptions();
5224  IntrusiveRefCntPtr<clang::DiagnosticIDs> DiagID(new clang::DiagnosticIDs());
5225  clang::DiagnosticsEngine Diags(DiagID, diagOptions);
5226 
5227  clang::driver::Driver TheDriver(args[0], llvm::sys::getDefaultTargetTriple(), "a.out", true, Diags);
5228  OwningPtr<clang::driver::Compilation> C(TheDriver.BuildCompilation(args));
5229  effectiveTarget = C->getDefaultToolChain().ComputeEffectiveClangTriple(C->getArgs());
5230  }
5231 
5232  module->setTargetTriple(effectiveTarget);
5233 */
5234  module->setTargetTriple("x86_64-apple-macosx10.10.0");
5235 }
5236 
5244 {
5245  Module *llvmModule = module->getModule();
5246 
5247  // In C++ the return value of a cast may not be the same pointer as the cast arg.
5248  // Because of this, VuoModule::getPseudoBase() returns a different value than VuoNodeClass::getBase().
5249  // Calling delete on VuoModule::getPseudoBase() gives a malloc error: "pointer being freed was not allocated".
5250  // So call it on VuoNodeClass::getBase(). Same idea for VuoType.
5251 
5252  VuoNodeClass *baseNodeClass = NULL;
5253  VuoType *baseType = NULL;
5254  VuoModule *baseModule = NULL;
5255  VuoCompilerNodeClass *nodeClass = dynamic_cast<VuoCompilerNodeClass *>(module);
5256  if (nodeClass)
5257  {
5258  baseNodeClass = dynamic_cast<VuoNodeClass *>(nodeClass->getBase());
5259 
5261  if (dynamic_cast<VuoCompilerSpecializedNodeClass *>(module))
5262  module = NULL;
5263  }
5264  else
5265  {
5266  VuoCompilerType *type = dynamic_cast<VuoCompilerType *>(module);
5267  if (type)
5268  baseType = dynamic_cast<VuoType *>(type->getBase());
5269  else
5270  baseModule = module->getPseudoBase();
5271  }
5272 
5273  delete module;
5274  delete baseNodeClass;
5275  delete baseType;
5276  delete baseModule;
5277  destroyLlvmModule(llvmModule);
5278 }
5279 
5287 {
5288  dispatch_sync(llvmQueue, ^{
5289  delete module;
5290  });
5291 }
5292 
5304 VuoNode * VuoCompiler::createNode(VuoCompilerNodeClass *nodeClass, string title, double x, double y)
5305 {
5307  if (nodeClassForNode)
5308  return nodeClassForNode->newNode(title, x, y);
5309 
5310  return nullptr;
5311 }
5312 
5319 VuoNode * VuoCompiler::createNode(VuoCompilerNodeClass *nodeClass, VuoNode *nodeToCopyMetadataFrom)
5320 {
5322  if (nodeClassForNode)
5323  return nodeClassForNode->newNode(nodeToCopyMetadataFrom);
5324 
5325  return nullptr;
5326 }
5327 
5331 VuoNode * VuoCompiler::createPublishedInputNode(vector<VuoPublishedPort *> publishedInputPorts)
5332 {
5333  string nodeClassName = VuoCompilerPublishedInputNodeClass::buildNodeClassName(publishedInputPorts);
5334  return createPublishedNode(nodeClassName, publishedInputPorts);
5335 }
5336 
5340 VuoNode * VuoCompiler::createPublishedOutputNode(vector<VuoPublishedPort *> publishedOutputPorts)
5341 {
5342  string nodeClassName = VuoCompilerPublishedOutputNodeClass::buildNodeClassName(publishedOutputPorts);
5343  return createPublishedNode(nodeClassName, publishedOutputPorts);
5344 }
5345 
5349 VuoNode * VuoCompiler::createPublishedNode(const string &nodeClassName, const vector<VuoPublishedPort *> &publishedPorts)
5350 {
5351  VuoCompilerNodeClass *nodeClass = getNodeClass(nodeClassName);
5352  VuoNode *node = createNode(nodeClass);
5353 
5354  // Set the generic port types on the node to match those on the published ports set by VuoCompilerComposition::updateGenericPortTypes().
5355  VuoCompilerPublishedInputNodeClass *inputNodeClass = dynamic_cast<VuoCompilerPublishedInputNodeClass *>(nodeClass);
5356  VuoCompilerPublishedOutputNodeClass *outputNodeClass = dynamic_cast<VuoCompilerPublishedOutputNodeClass *>(nodeClass);
5357  for (size_t i = 0; i < publishedPorts.size(); ++i)
5358  {
5359  VuoType *publishedPortType = static_cast<VuoCompilerPort *>(publishedPorts[i]->getCompiler())->getDataVuoType();
5360  if (! dynamic_cast<VuoGenericType *>(publishedPortType))
5361  continue;
5362 
5363  set<VuoPort *> nodePorts;
5364  if (inputNodeClass)
5365  {
5366  VuoPort *inputPort = node->getInputPorts().at( inputNodeClass->getInputPortIndexForPublishedInputPort(i) );
5367  nodePorts.insert(inputPort);
5368 
5369  VuoPort *outputPort = node->getOutputPorts().at( inputNodeClass->getOutputPortIndexForPublishedInputPort(i) );
5370  nodePorts.insert(outputPort);
5371  }
5372  else
5373  {
5374  VuoPort *inputPort = node->getInputPorts().at( outputNodeClass->getInputPortIndexForPublishedOutputPort(i) );
5375  nodePorts.insert(inputPort);
5376  }
5377 
5378  for (VuoPort *port : nodePorts)
5379  {
5380  VuoCompilerPort *compilerPort = static_cast<VuoCompilerPort *>(port->getCompiler());
5381  compilerPort->setDataVuoType(publishedPortType);
5382  }
5383  }
5384 
5385  reifyGenericPortTypes(node);
5386 
5387  return node;
5388 }
5389 
5395 {
5396  dispatch_sync(environmentQueue, ^{
5397  if (environments.size() >= 5)
5398  environments.at(3).at(0)->addExpatriateSourceFile(sourcePath);
5399  });
5400 }
5401 
5407 {
5408  dispatch_sync(environmentQueue, ^{
5409  if (environments.size() >= 5)
5410  {
5411  environments.at(3).at(0)->removeExpatriateSourceFile(sourcePath);
5412 
5413  set<string> sourcesRemoved;
5414  sourcesRemoved.insert(getModuleKeyForPath(sourcePath));
5415  loadModulesAndSources(set<string>(), set<string>(), set<string>(), set<string>(), set<string>(), sourcesRemoved,
5416  false, false, environments.at(3).at(0), nullptr, nullptr, "");
5417  }
5418  });
5419 }
5420 
5437 void VuoCompiler::overrideInstalledNodeClass(const string &sourcePath, const string &sourceCode)
5438 {
5439  string sourcePathCopy = sourcePath;
5440  string sourceCodeCopy = sourceCode;
5441 
5442  dispatch_async(environmentQueue, ^{
5443  string nodeClassName = getModuleKeyForPath(sourcePathCopy);
5444  ModuleInfo *sourceInfo = NULL;
5445 
5446  for (const vector<Environment *> &envs : environments)
5447  {
5448  for (Environment *env : envs)
5449  {
5450  ModuleInfo *potentialSourceInfo = env->listSourceFile(nodeClassName);
5451  if (potentialSourceInfo && VuoFileUtilities::arePathsEqual(potentialSourceInfo->getFile()->path(), sourcePathCopy))
5452  {
5453  sourceInfo = potentialSourceInfo;
5454  break;
5455  }
5456  }
5457  }
5458 
5459  if (! sourceInfo)
5460  return;
5461 
5462  bool shouldRecompileSourcesIfUnchanged;
5463  if (! sourceCodeCopy.empty())
5464  {
5465  sourceInfo->setSourceCode(sourceCodeCopy);
5466  sourceInfo->setSourceCodeOverridden(true);
5467 
5468  shouldRecompileSourcesIfUnchanged = false;
5469  }
5470  else
5471  {
5472  sourceInfo->revertSourceCode();
5473  sourceInfo->setSourceCodeOverridden(false);
5474 
5475  shouldRecompileSourcesIfUnchanged = true;
5476  }
5477  sourceInfo->setAttempted(false);
5478  sourceInfo->setLastModifiedToNow();
5479 
5480  set<string> sourcesModified;
5481  sourcesModified.insert(nodeClassName);
5482 
5483  loadModulesAndSources(set<string>(), set<string>(), set<string>(), set<string>(), sourcesModified, set<string>(),
5484  false, shouldRecompileSourcesIfUnchanged, nullptr, nullptr, nullptr, "");
5485  });
5486 }
5487 
5497 void VuoCompiler::revertOverriddenNodeClass(const string &sourcePath)
5498 {
5499  overrideInstalledNodeClass(sourcePath, "");
5500 }
5501 
5514 VuoCompilerNodeClass * VuoCompiler::getNodeClass(const string &nodeClassName)
5515 {
5516  // For performance, don't bother searching if it's obviously not a node class.
5517 
5518  if (VuoStringUtilities::endsWith(nodeClassName, ".framework"))
5519  return NULL;
5520 
5521  // Attempt to load the node class (if it hasn't already been loaded).
5522 
5523  set<string> nodeClassNameSet;
5524  nodeClassNameSet.insert(nodeClassName);
5525  loadModulesIfNeeded(nodeClassNameSet);
5526 
5527  // If the node class has been loaded, return it.
5528 
5529  __block VuoCompilerNodeClass *nodeClass = NULL;
5530  void (^envGetNodeClass)(Environment *) = ^void (Environment *env) {
5531  VuoCompilerNodeClass *result = env->getNodeClass(nodeClassName);
5532  if (result)
5533  nodeClass = result;
5534  };
5535  applyToAllEnvironments(envGetNodeClass);
5536  return nodeClass;
5537 }
5538 
5544 map<string, VuoCompilerNodeClass *> VuoCompiler::getNodeClasses()
5545 {
5546  loadModulesIfNeeded();
5547 
5548  __block map<string, VuoCompilerNodeClass *> nodeClasses;
5549  void (^envGetNodeClasses)(Environment *) = ^void (Environment *env) {
5550  map<string, VuoCompilerNodeClass *> result = env->getNodeClasses();
5551  nodeClasses.insert(result.begin(), result.end());
5552  };
5553  applyToInstalledEnvironments(envGetNodeClasses);
5554  return nodeClasses;
5555 }
5556 
5562 VuoCompilerType * VuoCompiler::getType(const string &typeName)
5563 {
5564  set<string> typeNameSet;
5565  typeNameSet.insert(typeName);
5566  loadModulesIfNeeded(typeNameSet);
5567 
5568  __block VuoCompilerType *type = NULL;
5569  void (^envGetType)(Environment *) = ^void (Environment *env) {
5570  VuoCompilerType *result = env->getType(typeName);
5571  if (result)
5572  type = result;
5573  };
5574  applyToInstalledEnvironments(envGetType);
5575 
5576  if (! type && VuoGenericType::isGenericTypeName(typeName))
5577  {
5578  VuoCompilerType * (^compilerGetType)(string) = ^VuoCompilerType * (string moduleKey) {
5579  return getType(moduleKey);
5580  };
5581 
5582  VuoGenericType *genericType = new VuoGenericType(typeName, vector<string>());
5583  type = VuoCompilerGenericType::newGenericType(genericType, compilerGetType);
5584  }
5585 
5586  return type;
5587 }
5588 
5594 map<string, VuoCompilerType *> VuoCompiler::getTypes()
5595 {
5596  loadModulesIfNeeded();
5597 
5598  __block map<string, VuoCompilerType *> types;
5599  void (^envGetTypes)(Environment *) = ^void (Environment *env) {
5600  map<string, VuoCompilerType *> result = env->getTypes();
5601  types.insert(result.begin(), result.end());
5602  };
5603  applyToInstalledEnvironments(envGetTypes);
5604  return types;
5605 }
5606 
5612 map<string, VuoCompilerModule *> VuoCompiler::getLibraryModules()
5613 {
5614  loadModulesIfNeeded();
5615 
5616  __block map<string, VuoCompilerModule *> libraryModules;
5617  void (^envGetLibraryModules)(Environment *) = ^void (Environment *env) {
5618  map<string, VuoCompilerModule *> result = env->getLibraryModules();
5619  libraryModules.insert(result.begin(), result.end());
5620  };
5621  applyToInstalledEnvironments(envGetLibraryModules);
5622  return libraryModules;
5623 }
5624 
5630 map<string, VuoNodeSet *> VuoCompiler::getNodeSets()
5631 {
5632  loadModulesIfNeeded();
5633 
5634  __block map<string, VuoNodeSet *> nodeSets;
5635  void (^envGetNodeSets)(Environment *) = ^void (Environment *env) {
5636  map<string, VuoNodeSet *> result = env->getNodeSets();
5637  nodeSets.insert(result.begin(), result.end());
5638  };
5639  applyToInstalledEnvironments(envGetNodeSets);
5640  return nodeSets;
5641 }
5642 
5649 {
5650  loadModulesIfNeeded();
5651 
5652  __block VuoNodeSet *nodeSet = NULL;
5653  void (^envFindNodeSet)(Environment *) = ^void (Environment *env) {
5654  VuoNodeSet *result = env->findNodeSet(name);
5655  if (result)
5656  nodeSet = result;
5657  };
5658  applyToInstalledEnvironments(envFindNodeSet);
5659  return nodeSet;
5660 }
5661 
5665 VuoCompilerModule * VuoCompiler::getModule(const string &moduleKey)
5666 {
5667  __block VuoCompilerModule *module = NULL;
5668  void (^envFindModule)(Environment *) = ^void (Environment *env) {
5669  VuoCompilerModule *result = env->findModule(moduleKey);
5670  if (result)
5671  module = result;
5672  };
5673  applyToAllEnvironments(envFindModule);
5674  return module;
5675 }
5676 
5688 void VuoCompiler::listNodeClasses(const string &format)
5689 {
5690  map<string, VuoCompilerNodeClass *> nodeClasses = getNodeClasses();
5691  for (map<string, VuoCompilerNodeClass *>::const_iterator i = nodeClasses.begin(); i != nodeClasses.end(); ++i)
5692  {
5693  VuoCompilerNodeClass *nodeClass = i->second;
5694  if (format == "")
5695  {
5696  printf("%s\n", nodeClass->getBase()->getClassName().c_str());
5697  }
5698  else if (format == "path")
5699  {
5700  // TODO: If "path", prints the absolute path of each node class, one per line.
5701  }
5702  else if (format == "dot")
5703  {
5704  VuoCompilerNode *node = nodeClass->newNode()->getCompiler();
5705 
5706  printf("%s\n", nodeClass->getDoxygenDocumentation().c_str());
5707  printf("%s\n\n", node->getGraphvizDeclaration().c_str());
5708 
5709  delete node;
5710  }
5711  }
5712 }
5713 
5720 vector<string> VuoCompiler::getCoreVuoDependencies(void)
5721 {
5722  vector<string> dependencies;
5723 
5724  dependencies.push_back("VuoHeap");
5725  dependencies.push_back("VuoApp");
5726 
5727  dependencies.push_back("zmq");
5728  dependencies.push_back("json-c");
5729  dependencies.push_back("objc");
5730  dependencies.push_back("c");
5731  dependencies.push_back("AppKit.framework");
5732 
5733 #ifdef COVERAGE
5734  dependencies.push_back(LLVM_ROOT "/lib/libprofile_rt.dylib");
5735 #endif
5736  return dependencies;
5737 }
5738 
5742 string VuoCompiler::getRuntimeMainDependency(void)
5743 {
5744  return "VuoRuntimeMain.o";
5745 }
5746 
5754 string VuoCompiler::getRuntimeDependency(void)
5755 {
5756  return "VuoRuntime.o";
5757 }
5758 
5765 string VuoCompiler::getVuoFrameworkPath(void)
5766 {
5767  if (! vuoFrameworkInProgressPath.empty())
5768  return vuoFrameworkInProgressPath;
5769 
5771 }
5772 
5776 llvm::sys::Path VuoCompiler::getClangPath(void)
5777 {
5778  return clangPath;
5779 }
5780 
5785 {
5786  // Start with the file name without extension.
5787  string dir, moduleKey, ext;
5788  VuoFileUtilities::splitPath(path, dir, moduleKey, ext);
5789 
5791  {
5792  vector<string> nodeClassNameParts = VuoStringUtilities::split(moduleKey, '.');
5793 
5794  // Remove repeated file extensions.
5795  while (nodeClassNameParts.size() > 1 && nodeClassNameParts.back() == ext)
5796  nodeClassNameParts.pop_back();
5797 
5798  // Convert each part to lowerCamelCase.
5799  for (string &part : nodeClassNameParts)
5800  part = VuoStringUtilities::convertToCamelCase(part, false, true, true);
5801 
5802  // If the node class name has only one part, add a prefix.
5803  if (nodeClassNameParts.size() == 1)
5804  nodeClassNameParts.insert(nodeClassNameParts.begin(), "isf");
5805 
5806  moduleKey = VuoStringUtilities::join(nodeClassNameParts, '.');
5807  }
5808 
5809  return moduleKey;
5810 }
5811 
5817 {
5818  __block bool isLocal = false;
5819 
5820  dispatch_sync(environmentQueue, ^{
5821  if (environments.size() >= 5)
5822  isLocal = environments.at(3).at(0)->findModule(moduleKey);
5823  });
5824 
5825  return isLocal;
5826 }
5827 
5835 {
5836  return lastCompositionBaseDir.empty() ? "" : lastCompositionBaseDir + "/Modules";
5837 }
5838 
5850 {
5851  return lastCompositionBaseDir;
5852 }
5853 
5857 void VuoCompiler::addModuleSearchPath(string path)
5858 {
5859  dispatch_sync(environmentQueue, ^{
5860  environments.back().at(0)->addModuleSearchPath(path);
5861  environments.back().at(0)->addLibrarySearchPath(path);
5862  });
5863 }
5864 
5868 void VuoCompiler::addHeaderSearchPath(const string &path)
5869 {
5870  dispatch_sync(environmentQueue, ^{
5871  environments.back().at(0)->addHeaderSearchPath(path);
5872  });
5873 }
5874 
5878 void VuoCompiler::addLibrarySearchPath(const string &path)
5879 {
5880  dispatch_sync(environmentQueue, ^{
5881  environments.back().at(0)->addLibrarySearchPath(path);
5882  });
5883 }
5884 
5888 void VuoCompiler::addFrameworkSearchPath(const string &path)
5889 {
5890  dispatch_sync(environmentQueue, ^{
5891  environments.back().at(0)->addFrameworkSearchPath(path);
5892  });
5893 }
5894 
5898 void VuoCompiler::setTelemetry(const string &telemetry)
5899 {
5900  this->telemetry = telemetry;
5901 }
5902 
5906 void VuoCompiler::setTarget(const string &target)
5907 {
5908  this->target = target;
5909 }
5910 
5914 void VuoCompiler::setVerbose(bool isVerbose)
5915 {
5916  this->isVerbose = isVerbose;
5917 }
5918 
5924 {
5925 #if VUO_PRO
5926  if (VuoPro::getProAccess())
5927  {
5928  _shouldShowSplashWindow = false;
5929  return;
5930  }
5931 #endif
5932 
5933  _shouldShowSplashWindow = potentiallyShow;
5934 }
5935 
5940 {
5941  return _shouldShowSplashWindow;
5942 }
5943 
5947 void VuoCompiler::setClangPath(const string &clangPath)
5948 {
5949  this->clangPath = llvm::sys::Path(StringRef(clangPath));
5950 }
5951 
5956 {
5957  string vuoFrameworkPath = getVuoFrameworkPath();
5958  return (vuoFrameworkPath.empty() ?
5959  VUO_BUILD_DIR "/bin/VuoCompositionLoader.app/Contents/MacOS/VuoCompositionLoader" :
5960  vuoFrameworkPath + "/Helpers/VuoCompositionLoader.app/Contents/MacOS/VuoCompositionLoader");
5961 }
5962 
5966 string VuoCompiler::getCompositionStubPath(void)
5967 {
5968  string vuoFrameworkPath = getVuoFrameworkPath();
5969  return (vuoFrameworkPath.empty() ?
5970  VUO_BUILD_DIR "/lib/libVuoCompositionStub.dylib" :
5971  vuoFrameworkPath + "/Modules/libVuoCompositionStub.dylib");
5972 }
5973 
5977 string VuoCompiler::getCachePathForComposition(const string compositionDir)
5978 {
5979  string modifierLetterColon("꞉");
5980  string cachedModulesName = compositionDir;
5981  VuoStringUtilities::replaceAll(cachedModulesName, "/", modifierLetterColon);
5982  return VuoFileUtilities::getCachePath() + "/" + cachedModulesName;
5983 }
5984 
5989 {
5990  __block vector<string> moduleSearchPaths;
5991  void (^envGetModuleSearchPaths)(Environment *) = ^void (Environment *env) {
5992  vector<string> result = env->getModuleSearchPaths();
5993  moduleSearchPaths.insert(moduleSearchPaths.end(), result.begin(), result.end());
5994  };
5995  applyToInstalledEnvironments(envGetModuleSearchPaths);
5996 
5997  __block vector<string> headerSearchPaths;
5998  void (^envGetHeaderSearchPaths)(Environment *) = ^void (Environment *env) {
5999  vector<string> result = env->getHeaderSearchPaths();
6000  headerSearchPaths.insert(headerSearchPaths.end(), result.begin(), result.end());
6001  };
6002  applyToInstalledEnvironments(envGetHeaderSearchPaths);
6003 
6004  __block vector<string> librarySearchPaths;
6005  void (^envGetLibrarySearchPaths)(Environment *) = ^void (Environment *env) {
6006  vector<string> result = env->getLibrarySearchPaths();
6007  librarySearchPaths.insert(librarySearchPaths.end(), result.begin(), result.end());
6008  };
6009  applyToInstalledEnvironments(envGetLibrarySearchPaths);
6010 
6011  __block vector<string> frameworkSearchPaths;
6012  void (^envGetFrameworkSearchPaths)(Environment *) = ^void (Environment *env) {
6013  vector<string> result = env->getFrameworkSearchPaths();
6014  frameworkSearchPaths.insert(frameworkSearchPaths.end(), result.begin(), result.end());
6015  };
6016  applyToInstalledEnvironments(envGetFrameworkSearchPaths);
6017 
6018  fprintf(stderr, "Module (node class, type, library) search paths:\n");
6019  for (vector<string>::iterator i = moduleSearchPaths.begin(); i != moduleSearchPaths.end(); ++i)
6020  fprintf(stderr, " %s\n", (*i).c_str());
6021  fprintf(stderr, "Header search paths:\n");
6022  for (vector<string>::iterator i = headerSearchPaths.begin(); i != headerSearchPaths.end(); ++i)
6023  fprintf(stderr, " %s\n", (*i).c_str());
6024  fprintf(stderr, "Other library search paths:\n");
6025  for (vector<string>::iterator i = librarySearchPaths.begin(); i != librarySearchPaths.end(); ++i)
6026  fprintf(stderr, " %s\n", (*i).c_str());
6027  fprintf(stderr, "Other framework search paths:\n");
6028  for (vector<string>::iterator i = frameworkSearchPaths.begin(); i != frameworkSearchPaths.end(); ++i)
6029  fprintf(stderr, " %s\n", (*i).c_str());
6030  fprintf(stderr, "Framework path:\n");
6031  if (! getVuoFrameworkPath().empty())
6032  fprintf(stderr, " %s\n", getVuoFrameworkPath().c_str());
6033  fprintf(stderr, "Clang path:\n");
6034  if (! getClangPath().str().empty())
6035  fprintf(stderr, " %s\n", getClangPath().c_str());
6036 }
6037 
6047 {
6048  try
6049  {
6050  VuoCompiler compiler(compositionFilePath);
6051  string directory, file, extension;
6052  VuoFileUtilities::splitPath(compositionFilePath, directory, file, extension);
6053  string compiledCompositionPath = VuoFileUtilities::makeTmpFile(file, "bc");
6054  string linkedCompositionPath = VuoFileUtilities::makeTmpFile(file + "-linked", "");
6055  compiler.compileComposition(compositionFilePath, compiledCompositionPath, true, issues);
6056  compiler.linkCompositionToCreateExecutable(compiledCompositionPath, linkedCompositionPath, Optimization_FastBuild);
6057  remove(compiledCompositionPath.c_str());
6058  return VuoRunner::newSeparateProcessRunnerFromExecutable(linkedCompositionPath, directory, false, true);
6059  }
6060  catch (VuoCompilerException &e)
6061  {
6062  if (issues != e.getIssues())
6063  issues->append(e.getIssues());
6064  return NULL;
6065  }
6066 }
6067 
6082 VuoRunner * VuoCompiler::newSeparateProcessRunnerFromCompositionString(string composition, string processName, string workingDirectory, VuoCompilerIssues *issues)
6083 {
6084  try
6085  {
6086  string compositionPath = (workingDirectory.empty() ? "." : workingDirectory) + "/" + processName + ".vuo";
6087  VuoCompiler compiler(compositionPath);
6088  string compiledCompositionPath = VuoFileUtilities::makeTmpFile(processName, "bc");
6089  string linkedCompositionPath = VuoFileUtilities::makeTmpFile(processName, "");
6090  compiler.compileCompositionString(composition, compiledCompositionPath, true, issues);
6091  compiler.linkCompositionToCreateExecutable(compiledCompositionPath, linkedCompositionPath, Optimization_FastBuild);
6092  remove(compiledCompositionPath.c_str());
6093  return VuoRunner::newSeparateProcessRunnerFromExecutable(linkedCompositionPath, workingDirectory, false, true);
6094  }
6095  catch (VuoCompilerException &e)
6096  {
6097  if (issues != e.getIssues())
6098  issues->append(e.getIssues());
6099  return NULL;
6100  }
6101 }
6102 
6112 {
6113  try
6114  {
6115  VuoCompiler compiler(compositionFilePath);
6116  string directory, file, extension;
6117  VuoFileUtilities::splitPath(compositionFilePath, directory, file, extension);
6118  string compiledCompositionPath = VuoFileUtilities::makeTmpFile(file, "bc");
6119  compiler.compileComposition(compositionFilePath, compiledCompositionPath, true, issues);
6120  string linkedCompositionPath = VuoFileUtilities::makeTmpFile(file, "dylib");
6121  compiler.linkCompositionToCreateDynamicLibrary(compiledCompositionPath, linkedCompositionPath, Optimization_FastBuild);
6122  remove(compiledCompositionPath.c_str());
6123  return VuoRunner::newCurrentProcessRunnerFromDynamicLibrary(linkedCompositionPath, directory, true);
6124  }
6125  catch (VuoCompilerException &e)
6126  {
6127  if (issues != e.getIssues())
6128  issues->append(e.getIssues());
6129  return NULL;
6130  }
6131 }
6132 
6147 VuoRunner * VuoCompiler::newCurrentProcessRunnerFromCompositionString(string composition, string workingDirectory, VuoCompilerIssues *issues)
6148 {
6149  try
6150  {
6151  string compositionPath = (workingDirectory.empty() ? "." : workingDirectory) + "/UntitledComposition.vuo";
6152  VuoCompiler compiler(compositionPath);
6153  string compiledCompositionPath = VuoFileUtilities::makeTmpFile("VuoRunnerComposition", "bc");
6154  compiler.compileCompositionString(composition, compiledCompositionPath, true, issues);
6155  string linkedCompositionPath = VuoFileUtilities::makeTmpFile("VuoRunnerComposition", "dylib");
6156  compiler.linkCompositionToCreateDynamicLibrary(compiledCompositionPath, linkedCompositionPath, Optimization_FastBuild);
6157  remove(compiledCompositionPath.c_str());
6158  return VuoRunner::newCurrentProcessRunnerFromDynamicLibrary(linkedCompositionPath, workingDirectory, true);
6159  }
6160  catch (VuoCompilerException &e)
6161  {
6162  if (issues != e.getIssues())
6163  issues->append(e.getIssues());
6164  return NULL;
6165  }
6166 }
6167 
6169 {
6170  pendingDataQueue = dispatch_queue_create("org.vuo.compiler.delegate.pending", 0);
6171 }
6172 
6174 {
6175  LoadedModulesData *data = dequeueData();
6176  data->release();
6177 }
6178 
6182 void VuoCompilerDelegate::enqueueData(LoadedModulesData *data)
6183 {
6184  dispatch_sync(pendingDataQueue, ^{
6185  pendingData.push_back(data);
6186  });
6187 }
6188 
6192 VuoCompilerDelegate::LoadedModulesData * VuoCompilerDelegate::dequeueData(void)
6193 {
6194  __block LoadedModulesData *ret;
6195  dispatch_sync(pendingDataQueue, ^{
6196  ret = pendingData.front();
6197  pendingData.pop_front();
6198  });
6199  return ret;
6200 }
6201 
6206 VuoCompilerDelegate::LoadedModulesData::LoadedModulesData(const set< pair<VuoCompilerModule *, VuoCompilerModule *> > &modulesModified,
6207  const set<VuoCompilerModule *> &modulesRemoved, VuoCompilerIssues *issues)
6208 {
6209  referenceCountQueue = dispatch_queue_create("org.vuo.compiler.delegate.reference", 0);
6210  referenceCount = 0;
6211 
6212  this->modulesModified = modulesModified;
6213  this->modulesRemoved = modulesRemoved;
6214  this->issues = issues;
6215 }
6216 
6220 VuoCompilerDelegate::LoadedModulesData::~LoadedModulesData(void)
6221 {
6222  delete issues;
6223 
6224  for (set< pair<VuoCompilerModule *, VuoCompilerModule *> >::iterator i = modulesModified.begin(); i != modulesModified.end(); ++i)
6225  VuoCompiler::destroyModule((*i).first);
6226 
6227  for (set<VuoCompilerModule *>::iterator i = modulesRemoved.begin(); i != modulesRemoved.end(); ++i)
6229 }
6230 
6234 void VuoCompilerDelegate::LoadedModulesData::retain(void)
6235 {
6236  dispatch_sync(referenceCountQueue, ^{
6237  ++referenceCount;
6238  });
6239 }
6240 
6244 void VuoCompilerDelegate::LoadedModulesData::release(void)
6245 {
6246  dispatch_sync(referenceCountQueue, ^{
6247  --referenceCount;
6248  if (referenceCount == 0) {
6249  delete this;
6250  }
6251  });
6252 }