11 #include <sys/param.h>
15 #include <CoreFoundation/CoreFoundation.h>
48 map<string, VuoFileUtilities::File *> VuoCompiler::Environment::moduleCacheFileForLocking;
49 dispatch_queue_t VuoCompiler::Environment::moduleCacheBuildingQueue = dispatch_queue_create(
"org.vuo.compiler.cache", NULL);
50 const string VuoCompiler::Environment::pidCacheDirPrefix =
"pid-";
51 set<VuoCompiler *> VuoCompiler::allCompilers;
52 dispatch_queue_t VuoCompiler::environmentQueue = dispatch_queue_create(
"org.vuo.compiler.environment", NULL);
53 vector< vector<VuoCompiler::Environment *> > VuoCompiler::sharedEnvironments;
54 map<string, vector<VuoCompiler::Environment *> > VuoCompiler::environmentsForCompositionFamily;
55 dispatch_group_t VuoCompiler::moduleSourceCompilersExist = dispatch_group_create();
56 string VuoCompiler::vuoFrameworkInProgressPath;
66 struct rlimit rl{OPEN_MAX, OPEN_MAX};
67 getrlimit(RLIMIT_NOFILE, &rl);
68 rl.rlim_cur =
MIN(OPEN_MAX, rl.rlim_max);
69 if (setrlimit(RLIMIT_NOFILE, &rl))
70 VUserLog(
"Warning: Couldn't set open-files limit: %s", strerror(errno));
73 llvmQueue = dispatch_queue_create(
"org.vuo.compiler.llvm", NULL);
76 llvm::InitializeNativeTarget();
89 Module module(
"", getGlobalContext());
108 VuoCompiler::ModuleInfo::ModuleInfo(Environment *environment,
const string &searchPath,
const string &relativePath,
109 bool isSourceFile,
bool isSubcomposition)
112 initialize(environment, searchPath, file, isSourceFile, isSubcomposition);
118 VuoCompiler::ModuleInfo::ModuleInfo(Environment *environment,
const string &searchPath,
VuoFileUtilities::File *file,
119 bool isSourceFile,
bool isSubcomposition)
121 initialize(environment, searchPath, file, isSourceFile, isSubcomposition);
127 void VuoCompiler::ModuleInfo::initialize(Environment *environment,
const string &searchPath,
VuoFileUtilities::File *file,
128 bool isSourceFile,
bool isSubcomposition)
130 this->environment = environment;
131 this->searchPath = searchPath;
134 sourceCodeOverridden =
false;
136 longestDownstreamPath = 0;
146 loading = dispatch_group_create();
149 if (isSubcomposition)
163 VuoCompiler::ModuleInfo::~ModuleInfo(
void)
172 dispatch_release(loading);
175 VuoCompiler::Environment * VuoCompiler::ModuleInfo::getEnvironment(
void)
const
180 string VuoCompiler::ModuleInfo::getSearchPath(
void)
const
185 string VuoCompiler::ModuleInfo::getModuleKey(
void)
const
195 string VuoCompiler::ModuleInfo::getSourceCode(
void)
const
200 void VuoCompiler::ModuleInfo::setSourceCode(
const string &sourceCode)
202 this->sourceCode = sourceCode;
205 void VuoCompiler::ModuleInfo::revertSourceCode(
void)
210 bool VuoCompiler::ModuleInfo::isSourceCodeOverridden(
void)
const
212 return sourceCodeOverridden;
215 void VuoCompiler::ModuleInfo::setSourceCodeOverridden(
bool overridden)
217 sourceCodeOverridden = overridden;
220 bool VuoCompiler::ModuleInfo::isNewerThan(ModuleInfo *other)
const
222 return lastModified > other->lastModified;
225 bool VuoCompiler::ModuleInfo::isNewerThan(
unsigned long ageInSeconds)
const
227 return lastModified > ageInSeconds;
230 bool VuoCompiler::ModuleInfo::isOlderThan(
unsigned long ageInSeconds)
const
232 return lastModified < ageInSeconds;
235 void VuoCompiler::ModuleInfo::setLastModifiedToNow(
void)
238 gettimeofday(&t, NULL);
239 lastModified = t.tv_sec;
242 set<string> VuoCompiler::ModuleInfo::getContainedNodeClasses(
void)
const
244 return containedNodeClasses;
247 int VuoCompiler::ModuleInfo::getLongestDownstreamPath(
void)
const
249 return longestDownstreamPath;
252 void VuoCompiler::ModuleInfo::setLongestDownstreamPath(
int pathLength)
254 longestDownstreamPath = pathLength;
257 bool VuoCompiler::ModuleInfo::getAttempted(
void)
const
262 void VuoCompiler::ModuleInfo::setAttempted(
bool attempted)
264 this->attempted = attempted;
267 dispatch_group_t VuoCompiler::ModuleInfo::getLoadingGroup(
void)
const
272 void VuoCompiler::ModuleInfo::dump()
const
274 fprintf(stderr,
"module %s:\n"
275 "\tloadingGroup=%p\n"
280 "\tsourceCodeOverridden=%d\n"
282 "\tcontainedNodeClasses:\n",
287 environment->getCompiledModuleCachePath().c_str(),
291 sourceCodeOverridden,
293 for (
auto i: containedNodeClasses)
294 fprintf(stderr,
"\t\t%s\n", i.c_str());
297 VuoCompiler::DependencyGraphVertex * VuoCompiler::DependencyGraphVertex::vertexForDependency(
const string &dependency,
VuoDirectedAcyclicGraph *graph)
301 return dynamic_cast<DependencyGraphVertex *
>(v);
303 DependencyGraphVertex *vv =
new DependencyGraphVertex();
304 vv->dependency = dependency;
305 vv->environment = NULL;
306 vv->compatible =
true;
310 string VuoCompiler::DependencyGraphVertex::getDependency(
void)
315 VuoCompiler::Environment * VuoCompiler::DependencyGraphVertex::getEnvironment(
void)
320 void VuoCompiler::DependencyGraphVertex::setEnvironment(Environment *environment)
322 this->environment = environment;
325 bool VuoCompiler::DependencyGraphVertex::isCompatible(
void)
330 void VuoCompiler::DependencyGraphVertex::setCompatible(
bool compatible)
332 this->compatible = compatible;
335 string VuoCompiler::DependencyGraphVertex::key(
void)
345 VuoCompiler::ModuleInfoIterator::ModuleInfoIterator(map<
string, map<string, ModuleInfo *> > *allModuleInfos)
347 this->allModuleInfos = allModuleInfos;
348 hasSearchModuleKeys =
false;
357 VuoCompiler::ModuleInfoIterator::ModuleInfoIterator(map<
string, map<string, ModuleInfo *> > *allModuleInfos,
const set<string> &searchModuleKeys)
359 this->allModuleInfos = allModuleInfos;
360 this->searchModuleKeys = searchModuleKeys;
361 hasSearchModuleKeys =
true;
370 void VuoCompiler::ModuleInfoIterator::initialize(
void)
372 currSearchPath = allModuleInfos->begin();
373 hasCurrModuleKey =
false;
381 VuoCompiler::ModuleInfo * VuoCompiler::ModuleInfoIterator::next(
void)
383 for ( ; currSearchPath != allModuleInfos->end(); ++currSearchPath)
385 if (! hasCurrModuleKey)
387 currModuleKey = currSearchPath->second.begin();
388 hasCurrModuleKey =
true;
391 for ( ; currModuleKey != currSearchPath->second.end(); ++currModuleKey)
393 if (! hasSearchModuleKeys || searchModuleKeys.find(currModuleKey->first) != searchModuleKeys.end())
395 ModuleInfo *moduleInfo = currModuleKey->second;
401 hasCurrModuleKey =
false;
410 VuoCompiler::Environment::Environment(
void)
412 compilersToNotifyQueue = dispatch_queue_create(
"org.vuo.compiler.notify", 0);
413 moduleSearchPathContentsChangedQueue = dispatch_queue_create(
"org.vuo.compiler.watch", 0);
414 isModuleCacheableDataDirty =
false;
415 isModuleCacheInitialized =
false;
416 isModuleCacheAvailable =
false;
425 VuoCompiler::Environment::~Environment(
void)
427 stopWatchingModuleSearchPaths();
428 dispatch_sync(moduleSearchPathContentsChangedQueue, ^{});
430 dispatch_release(moduleSearchPathContentsChangedQueue);
431 dispatch_release(compilersToNotifyQueue);
433 for (map<string, VuoCompilerNodeClass *>::iterator i = nodeClasses.begin(); i != nodeClasses.end(); ++i)
434 destroyModule(i->second);
436 for (map<string, VuoCompilerType *>::iterator i = types.begin(); i != types.end(); ++i)
437 destroyModule(i->second);
439 for (map<string, VuoCompilerModule *>::iterator i = libraryModules.begin(); i != libraryModules.end(); ++i)
440 destroyModule(i->second);
442 for (set<VuoCompilerGenericType *>::iterator i = genericTypes.begin(); i != genericTypes.end(); ++i)
444 VuoType *base = (*i)->getBase();
449 for (map<string, VuoNodeSet *>::iterator i = nodeSetForName.begin(); i != nodeSetForName.end(); ++i)
452 for (map<
string, map<string, ModuleInfo *> >::iterator i = moduleFilesAtSearchPath.begin(); i != moduleFilesAtSearchPath.end(); ++i)
453 for (map<string, ModuleInfo *>::iterator j = i->second.begin(); j != i->second.end(); ++j)
456 for (map<
string, map<string, ModuleInfo *> >::iterator i = sourceFilesAtSearchPath.begin(); i != sourceFilesAtSearchPath.end(); ++i)
457 for (map<string, ModuleInfo *>::iterator j = i->second.begin(); j != i->second.end(); ++j)
460 delete dependencyGraph;
461 delete compositionDependencyGraph;
467 void VuoCompiler::Environment::addCompilerToNotify(
VuoCompiler *compiler)
469 dispatch_sync(compilersToNotifyQueue, ^{
470 compilersToNotify.insert(compiler);
477 void VuoCompiler::Environment::removeCompilerToNotify(
VuoCompiler *compiler)
479 dispatch_sync(compilersToNotifyQueue, ^{
480 compilersToNotify.erase(compiler);
489 map<string, VuoCompilerNodeClass *> VuoCompiler::Environment::getNodeClasses(
void)
501 map<string, VuoCompilerNodeClass *>::iterator nodeClassIter = nodeClasses.find(moduleKey);
502 if (nodeClassIter != nodeClasses.end())
503 return nodeClassIter->second;
516 map<string, VuoCompilerNodeClass *>::iterator nodeClassIter = nodeClasses.find(moduleKey);
517 if (nodeClassIter != nodeClasses.end())
519 nodeClass = nodeClassIter->second;
521 nodeClasses.erase(nodeClassIter);
522 removeFromDependencyGraph(nodeClass);
534 map<string, VuoCompilerType *> VuoCompiler::Environment::getTypes(
void)
544 VuoCompilerType * VuoCompiler::Environment::getType(
const string &moduleKey)
546 map<string, VuoCompilerType *>::iterator typeIter = types.find(moduleKey);
547 if (typeIter != types.end())
548 return typeIter->second;
558 map<string, VuoNodeSet *> VuoCompiler::Environment::getNodeSets(
void)
560 return nodeSetForName;
568 map<string, VuoCompilerModule *> VuoCompiler::Environment::getLibraryModules(
void)
570 return libraryModules;
579 VuoCompilerModule * VuoCompiler::Environment::findModule(
const string &moduleKey)
581 map<string, VuoCompilerNodeClass *>::iterator nodeClassIter = nodeClasses.find(moduleKey);
582 if (nodeClassIter != nodeClasses.end())
583 return nodeClassIter->second;
585 map<string, VuoCompilerType *>::iterator typeIter = types.find(moduleKey);
586 if (typeIter != types.end())
587 return typeIter->second;
589 map<string, VuoCompilerModule *>::iterator libraryIter = libraryModules.find(moduleKey);
590 if (libraryIter != libraryModules.end())
591 return libraryIter->second;
601 VuoNodeSet * VuoCompiler::Environment::findNodeSet(
const string &name)
604 map<string, VuoNodeSet *>::iterator nodeSetIter = nodeSetForName.find(name);
605 if (nodeSetIter != nodeSetForName.end())
606 return nodeSetIter->second;
617 void VuoCompiler::Environment::addModuleSearchPath(
const string &path,
bool shouldWatch)
619 moduleSearchPaths.push_back(path);
621 updateModulesAtSearchPath(path);
622 updateSourceFilesAtSearchPath(path);
625 startWatchingModuleSearchPath(path);
633 vector<string> VuoCompiler::Environment::getModuleSearchPaths(
void)
635 return moduleSearchPaths;
643 void VuoCompiler::Environment::addHeaderSearchPath(
const string &path)
645 headerSearchPaths.push_back(path);
653 vector<string> VuoCompiler::Environment::getHeaderSearchPaths(
void)
655 return headerSearchPaths;
663 void VuoCompiler::Environment::addLibrarySearchPath(
const string &path)
665 librarySearchPaths.push_back(path);
673 vector<string> VuoCompiler::Environment::getLibrarySearchPaths(
void)
675 return librarySearchPaths;
683 void VuoCompiler::Environment::addFrameworkSearchPath(
const string &path)
685 frameworkSearchPaths.push_back(path);
693 vector<string> VuoCompiler::Environment::getFrameworkSearchPaths(
void)
695 return frameworkSearchPaths;
700 return dependencyGraph;
705 return compositionDependencyGraph;
713 void VuoCompiler::Environment::setModuleCachePath(
const string &path)
715 moduleCachePath = path;
723 string VuoCompiler::Environment::getCompiledModuleCachePath(
void)
725 return (moduleCachePath.empty() ?
"" : moduleCachePath +
"/Modules");
733 string VuoCompiler::Environment::getOverriddenCompiledModuleCachePath(
void)
735 if (moduleCachePath.empty())
741 string dir, moduleCacheDirName, ext;
754 void VuoCompiler::Environment::addExpatriateSourceFile(
const string &sourcePath)
756 expatriateSourceFiles.push_back(sourcePath);
758 if (find(moduleSearchPaths.begin(), moduleSearchPaths.end(),
"") == moduleSearchPaths.end())
759 moduleSearchPaths.push_back(
"");
761 auto iter = sourceFilesAtSearchPath.find(
"");
762 if (iter != sourceFilesAtSearchPath.end())
763 sourceFilesAtSearchPath.erase(iter);
771 void VuoCompiler::Environment::removeExpatriateSourceFile(
const string &sourcePath)
773 for (
auto i = expatriateSourceFiles.begin(); i != expatriateSourceFiles.end(); ++i)
777 expatriateSourceFiles.erase(i);
779 auto iter = sourceFilesAtSearchPath.find(
"");
780 if (iter != sourceFilesAtSearchPath.end())
781 sourceFilesAtSearchPath.erase(iter);
800 void VuoCompiler::Environment::updateModulesAtSearchPath(
const string &path,
bool shouldCleanModuleCache)
802 if (moduleFilesAtSearchPath.find(path) != moduleFilesAtSearchPath.end())
805 set<string> moduleExtensions;
806 moduleExtensions.insert(
"vuonode");
807 moduleExtensions.insert(
"vuonode+");
808 moduleExtensions.insert(
"bc");
809 moduleExtensions.insert(
"bc+");
810 set<string> archiveExtensions;
811 archiveExtensions.insert(
"vuonode");
814 map<string, ModuleInfo *> fileForModuleKey;
815 for (set<VuoFileUtilities::File *>::iterator i = moduleFiles.begin(); i != moduleFiles.end(); ++i)
819 ModuleInfo *m =
new ModuleInfo(
this, path, moduleFile,
false,
false);
820 fileForModuleKey[m->getModuleKey()] = m;
823 if (shouldCleanModuleCache && path == getCompiledModuleCachePath())
825 for (map<string, ModuleInfo *>::iterator i = fileForModuleKey.begin(); i != fileForModuleKey.end(); )
827 ModuleInfo *sourceInfo = listSourceFile(i->first);
828 if (! sourceInfo || sourceInfo->isNewerThan(i->second))
830 ModuleInfo *m = i->second;
833 fileForModuleKey.erase(i++);
840 moduleFilesAtSearchPath[path] = fileForModuleKey;
850 void VuoCompiler::Environment::updateSourceFilesAtSearchPath(
const string &path)
852 map<string, map<string, ModuleInfo *> >::iterator sourceFilesIter = sourceFilesAtSearchPath.find(path);
853 if (sourceFilesIter != sourceFilesAtSearchPath.end())
856 set<VuoFileUtilities::File *> sourceFiles;
859 set<string> sourceExtensions;
860 sourceExtensions.insert(
"vuo");
861 sourceExtensions.insert(
"fs");
863 sourceExtensions.insert(cext.begin(), cext.end());
868 for (
const string &sourcePath : expatriateSourceFiles)
870 string dir, file, ext;
873 sourceFiles.insert(sourceFile);
877 map<string, ModuleInfo *> fileForModuleKey;
878 for (set<VuoFileUtilities::File *>::iterator i = sourceFiles.begin(); i != sourceFiles.end(); ++i)
882 string dir, moduleKey, ext;
884 bool isSubcomposition = (ext ==
"vuo");
887 if (path.empty() && !sourceFile->
exists())
890 ModuleInfo *m =
new ModuleInfo(
this, path, sourceFile,
true, isSubcomposition);
891 if (fileForModuleKey.find(m->getModuleKey()) != fileForModuleKey.end())
892 VUserLog(
"Warning: Conflicting source files for module %s are installed at %s", m->getModuleKey().c_str(), path.c_str());
893 fileForModuleKey[m->getModuleKey()] = m;
895 if (isSubcomposition)
897 DependencyGraphVertex *compositionVertex = DependencyGraphVertex::vertexForDependency(moduleKey, compositionDependencyGraph);
898 compositionDependencyGraph->addVertex(compositionVertex);
900 compositionVertex->setEnvironment(
this);
902 set<string> dependencies = m->getContainedNodeClasses();
903 for (set<string>::iterator j = dependencies.begin(); j != dependencies.end(); ++j)
905 DependencyGraphVertex *dependencyVertex = DependencyGraphVertex::vertexForDependency(*j, compositionDependencyGraph);
906 compositionDependencyGraph->addEdge(compositionVertex, dependencyVertex);
911 sourceFilesAtSearchPath[path] = fileForModuleKey;
919 VuoCompiler::ModuleInfo * VuoCompiler::Environment::listModule(
const string &moduleKey)
921 for (
const auto &moduleFiles : moduleFilesAtSearchPath)
923 map<string, ModuleInfo *>::const_iterator foundIter = moduleFiles.second.find(moduleKey);
924 if (foundIter != moduleFiles.second.end())
925 return foundIter->second;
936 VuoCompiler::ModuleInfoIterator VuoCompiler::Environment::listModules(
const set<string> &moduleKeys)
938 for (
const string &path : moduleSearchPaths)
939 updateModulesAtSearchPath(path);
941 return ModuleInfoIterator(&moduleFilesAtSearchPath, moduleKeys);
949 VuoCompiler::ModuleInfoIterator VuoCompiler::Environment::listAllModules(
void)
951 for (
const string &path : moduleSearchPaths)
952 updateModulesAtSearchPath(path);
954 return ModuleInfoIterator(&moduleFilesAtSearchPath);
962 VuoCompiler::ModuleInfo * VuoCompiler::Environment::listSourceFile(
const string &moduleKey)
964 for (
const auto &sourceFiles : sourceFilesAtSearchPath)
966 map<string, ModuleInfo *>::const_iterator foundIter = sourceFiles.second.find(moduleKey);
967 if (foundIter != sourceFiles.second.end())
968 return foundIter->second;
979 VuoCompiler::ModuleInfoIterator VuoCompiler::Environment::listSourceFiles(
const set<string> &moduleKeys)
981 for (
const string &path : moduleSearchPaths)
982 updateSourceFilesAtSearchPath(path);
984 return ModuleInfoIterator(&sourceFilesAtSearchPath, moduleKeys);
992 VuoCompiler::ModuleInfoIterator VuoCompiler::Environment::listAllSourceFiles(
void)
994 for (
const string &path : moduleSearchPaths)
995 updateSourceFilesAtSearchPath(path);
997 return ModuleInfoIterator(&sourceFilesAtSearchPath);
1003 vector<string> VuoCompiler::Environment::getBuiltInModuleSearchPaths(
void)
1005 vector<string> builtInModuleSearchPaths;
1007 string vuoFrameworkPath = getVuoFrameworkPath();
1008 if (! vuoFrameworkPath.empty())
1010 builtInModuleSearchPaths.push_back(vuoFrameworkPath +
"/Modules");
1014 builtInModuleSearchPaths.push_back(VUO_BUILD_DIR
"/library");
1015 builtInModuleSearchPaths.push_back(VUO_BUILD_DIR
"/node");
1016 builtInModuleSearchPaths.push_back(VUO_BUILD_DIR
"/type");
1017 builtInModuleSearchPaths.push_back(VUO_BUILD_DIR
"/type/list");
1020 return builtInModuleSearchPaths;
1026 vector<string> VuoCompiler::Environment::getBuiltInHeaderSearchPaths(
void)
1028 vector<string> builtInHeaderSearchPaths;
1030 string vuoFrameworkPath = getVuoFrameworkPath();
1031 if (! vuoFrameworkPath.empty())
1033 builtInHeaderSearchPaths.push_back(vuoFrameworkPath +
"/Headers");
1034 builtInHeaderSearchPaths.push_back(vuoFrameworkPath +
"/Headers/macos");
1035 builtInHeaderSearchPaths.push_back(vuoFrameworkPath +
"/Frameworks/llvm.framework/Versions/A/Headers/lib/c++/v1");
1039 builtInHeaderSearchPaths.push_back(VUO_SOURCE_DIR
"/library");
1040 builtInHeaderSearchPaths.push_back(VUO_SOURCE_DIR
"/node");
1041 builtInHeaderSearchPaths.push_back(VUO_SOURCE_DIR
"/type");
1042 builtInHeaderSearchPaths.push_back(VUO_SOURCE_DIR
"/type/list");
1043 builtInHeaderSearchPaths.push_back(VUO_SOURCE_DIR
"/runtime");
1044 builtInHeaderSearchPaths.push_back(JSONC_ROOT
"/include");
1045 builtInHeaderSearchPaths.push_back(LLVM_ROOT
"/include/c++/v1");
1048 return builtInHeaderSearchPaths;
1054 vector<string> VuoCompiler::Environment::getBuiltInLibrarySearchPaths(
void)
1056 vector<string> builtInLibrarySearchPaths;
1058 string vuoFrameworkPath = getVuoFrameworkPath();
1059 if (! vuoFrameworkPath.empty())
1061 builtInLibrarySearchPaths.push_back(vuoFrameworkPath +
"/Modules");
1064 builtInLibrarySearchPaths.push_back(OPENSSL_ROOT
"/lib");
1068 builtInLibrarySearchPaths.push_back(VUO_BUILD_DIR
"/library");
1069 builtInLibrarySearchPaths.push_back(VUO_BUILD_DIR
"/node");
1070 builtInLibrarySearchPaths.push_back(VUO_BUILD_DIR
"/type");
1071 builtInLibrarySearchPaths.push_back(VUO_BUILD_DIR
"/type/list");
1072 builtInLibrarySearchPaths.push_back(VUO_BUILD_DIR
"/runtime");
1074 builtInLibrarySearchPaths.push_back(LEAP_ROOT);
1077 builtInLibrarySearchPaths.insert(builtInLibrarySearchPaths.end(), conanLibDirs.begin(), conanLibDirs.end());
1080 return builtInLibrarySearchPaths;
1086 vector<string> VuoCompiler::Environment::getBuiltInFrameworkSearchPaths(
void)
1088 vector<string> builtInFrameworkSearchPaths;
1090 string vuoFrameworkPath = getVuoFrameworkPath();
1091 if (! vuoFrameworkPath.empty())
1093 builtInFrameworkSearchPaths.push_back(vuoFrameworkPath +
"/Modules/");
1094 builtInFrameworkSearchPaths.push_back(vuoFrameworkPath +
"/Frameworks/");
1098 builtInFrameworkSearchPaths.push_back(SYPHON_ROOT);
1101 return builtInFrameworkSearchPaths;
1110 void VuoCompiler::Environment::startWatchingModuleSearchPath(
const string &moduleSearchPath)
1113 moduleSearchPathWatchers.insert(watcher);
1121 void VuoCompiler::Environment::stopWatchingModuleSearchPaths(
void)
1123 for (set<VuoFileWatcher *>::iterator i = moduleSearchPathWatchers.begin(); i != moduleSearchPathWatchers.end(); ++i)
1126 moduleSearchPathWatchers.clear();
1133 void VuoCompiler::Environment::fileChanged(
const string &moduleSearchPath)
1135 dispatch_sync(moduleSearchPathContentsChangedQueue, ^{
1136 moduleSearchPathContentsChanged(moduleSearchPath);
1147 void VuoCompiler::Environment::moduleSearchPathContentsChanged(
const string &moduleSearchPath,
const string &moduleAddedOrModifiedPath,
1148 const string &moduleAddedOrModifiedSourceCode,
1149 std::function<
void(
void)> moduleLoadedCallback,
1155 __block set<string> modulesAdded;
1156 __block set<string> modulesModifed;
1157 __block set<string> modulesRemoved;
1158 __block set<string> compositionsAdded;
1159 __block set<string> compositionsModifed;
1160 __block set<string> compositionsRemoved;
1162 dispatch_sync(environmentQueue, ^{
1166 map<string, ModuleInfo *> oldModules;
1167 map<string, ModuleInfo *> oldCompositions;
1169 map<string, map<string, ModuleInfo *> >::iterator mf = moduleFilesAtSearchPath.find(moduleSearchPath);
1170 if (mf != moduleFilesAtSearchPath.end())
1172 for (map<string, ModuleInfo *>::iterator j = mf->second.begin(); j != mf->second.end(); ++j)
1174 oldModules[j->first] = j->second;
1177 moduleFilesAtSearchPath.erase(mf);
1180 map<string, map<string, ModuleInfo *> >::iterator cf = sourceFilesAtSearchPath.find(moduleSearchPath);
1181 if (cf != sourceFilesAtSearchPath.end())
1183 for (map<string, ModuleInfo *>::iterator j = cf->second.begin(); j != cf->second.end(); ++j)
1185 oldCompositions[j->first] = j->second;
1187 VuoDirectedAcyclicGraph::Vertex *vertex = compositionDependencyGraph->findVertex(j->first);
1188 compositionDependencyGraph->removeVertex(vertex);
1191 sourceFilesAtSearchPath.erase(cf);
1196 updateModulesAtSearchPath(moduleSearchPath, moduleAddedOrModifiedPath.empty());
1197 updateSourceFilesAtSearchPath(moduleSearchPath);
1199 mf = moduleFilesAtSearchPath.find(moduleSearchPath);
1200 if (mf != moduleFilesAtSearchPath.end())
1202 for (map<string, ModuleInfo *>::iterator n = mf->second.begin(); n != mf->second.end(); ++n)
1204 string moduleKey = n->first;
1206 map<string, ModuleInfo *>::iterator o = oldModules.find(moduleKey);
1207 if (o != oldModules.end())
1209 if (n->second->isNewerThan(o->second) ||
1210 (n->second->getFile() && ! n->second->getFile()->isInArchive() &&
VuoFileUtilities::arePathsEqual(n->second->getFile()->path(), moduleAddedOrModifiedPath)))
1212 modulesModifed.insert(moduleKey);
1216 n->second->setAttempted(o->second->getAttempted());
1220 oldModules.erase(o);
1224 modulesModifed.insert(moduleKey);
1228 modulesAdded.insert(moduleKey);
1233 cf = sourceFilesAtSearchPath.find(moduleSearchPath);
1234 if (cf != sourceFilesAtSearchPath.end())
1236 for (map<string, ModuleInfo *>::iterator n = cf->second.begin(); n != cf->second.end(); ++n)
1238 string moduleKey = n->first;
1240 map<string, ModuleInfo *>::iterator o = oldCompositions.find(moduleKey);
1241 if (o != oldCompositions.end())
1243 if (n->second->isNewerThan(o->second))
1245 compositionsModifed.insert(moduleKey);
1249 n->second->setAttempted(o->second->getAttempted());
1250 n->second->setSourceCode(o->second->getSourceCode());
1254 oldCompositions.erase(o);
1258 compositionsAdded.insert(moduleKey);
1263 for (map<string, ModuleInfo *>::iterator o = oldModules.begin(); o != oldModules.end(); ++o)
1266 modulesRemoved.insert(o->first);
1269 for (map<string, ModuleInfo *>::iterator o = oldCompositions.begin(); o != oldCompositions.end(); ++o)
1272 compositionsRemoved.insert(o->first);
1275 compilerForLoading->loadModulesAndSources(modulesAdded, modulesModifed, modulesRemoved,
1276 compositionsAdded, compositionsModifed, compositionsRemoved,
1277 false,
false,
this, issues, moduleLoadedCallback, moduleAddedOrModifiedSourceCode);
1281 delete compilerForLoading;
1287 void VuoCompiler::Environment::notifyCompilers(
const set<VuoCompilerModule *> &modulesAdded,
1288 const set< pair<VuoCompilerModule *, VuoCompilerModule *> > &modulesModified,
1290 bool oldModulesInvalidated)
1292 if (! (modulesAdded.empty() && modulesModified.empty() && modulesRemoved.empty() && issues->
isEmpty()) )
1294 set< pair<VuoCompilerModule *, VuoCompilerModule *> > invalidatedModulesModified;
1295 set<VuoCompilerModule *> invalidatedModulesRemoved;
1296 if (oldModulesInvalidated)
1298 invalidatedModulesModified = modulesModified;
1299 invalidatedModulesRemoved = modulesRemoved;
1302 VuoCompilerDelegate::LoadedModulesData *delegateData =
new VuoCompilerDelegate::LoadedModulesData(invalidatedModulesModified, invalidatedModulesRemoved, issues);
1304 map<string, VuoCompilerModule *> modulesAddedMap;
1305 map<string, pair<VuoCompilerModule *, VuoCompilerModule *> > modulesModifiedMap;
1306 map<string, VuoCompilerModule *> modulesRemovedMap;
1308 modulesAddedMap[m->getPseudoBase()->getModuleKey()] = m;
1309 for (pair<VuoCompilerModule *, VuoCompilerModule *> m : modulesModified)
1310 modulesModifiedMap[m.first->getPseudoBase()->getModuleKey()] = m;
1312 modulesRemovedMap[m->getPseudoBase()->getModuleKey()] = m;
1314 dispatch_sync(compilersToNotifyQueue, ^{
1315 for (set<VuoCompiler *>::iterator i = compilersToNotify.begin(); i != compilersToNotify.end(); ++i) {
1316 delegateData->retain();
1318 for (set<VuoCompiler *>::iterator i = compilersToNotify.begin(); i != compilersToNotify.end(); ++i) {
1319 (*i)->loadedModules(modulesAddedMap, modulesModifiedMap, modulesRemovedMap, issues, delegateData, this);
1339 set<VuoCompilerModule *> VuoCompiler::Environment::loadCompiledModules(
const set<string> &moduleKeys,
const map<string, string> &sourceCodeForModule)
1341 ModuleInfoIterator modulesToLoadIter = listModules(moduleKeys);
1342 set<VuoCompilerModule *> modulesLoaded;
1344 ModuleInfo *moduleInfo;
1345 while ((moduleInfo = modulesToLoadIter.next()))
1347 string moduleKey = moduleInfo->getModuleKey();
1351 if (moduleInfo->getEnvironment() !=
this || findModule(moduleKey) || moduleInfo->getAttempted())
1354 moduleInfo->setAttempted(
true);
1361 modulesLoaded.insert(module);
1362 addToDependencyGraph(module);
1366 string searchPath = moduleInfo->getSearchPath();
1373 ModuleInfo *sourceInfo = listSourceFile(moduleKey);
1377 auto sourceCodeIter = sourceCodeForModule.find(moduleKey);
1378 if (sourceCodeIter != sourceCodeForModule.end())
1385 return modulesLoaded;
1399 set<dispatch_group_t> VuoCompiler::Environment::loadSpecializedModules(
const set<string> &moduleKeys,
1402 set<dispatch_group_t > loadingGroups;
1404 for (
string moduleKey : moduleKeys)
1413 if (findModule(moduleKey))
1418 map<string, dispatch_group_t>::iterator iter = specializedModulesLoading.find(moduleKey);
1419 if (iter != specializedModulesLoading.end())
1421 loadingGroups.insert(iter->second);
1422 dispatch_retain(iter->second);
1426 dispatch_group_t loadingGroup = dispatch_group_create();
1427 specializedModulesLoading[moduleKey] = loadingGroup;
1428 loadingGroups.insert(loadingGroup);
1434 dispatch_queue_t queue = dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0);
1435 dispatch_group_async(loadingGroup, queue, ^{
1449 compiler->loadNodeClassGeneratedAtRuntime(specNodeClass,
this);
1452 compiler->loadModulesIfNeeded(dependencies);
1455 dispatch_sync(environmentQueue, ^{
1456 specializedModulesLoading.erase(moduleKey);
1461 return loadingGroups;
1476 set<dispatch_group_t> VuoCompiler::Environment::compileModulesFromSourceCode(
const set<string> &moduleKeys,
bool shouldRecompileIfUnchanged)
1478 ModuleInfoIterator modulesToLoadIter = listSourceFiles(moduleKeys);
1480 int environmentIndex = sharedEnvironments.size();
1481 for (
int i = 0; i < sharedEnvironments.size(); ++i)
1482 if (
this == sharedEnvironments.at(i).at(0))
1483 environmentIndex = i;
1485 set<dispatch_group_t> sourcesLoading;
1486 int sourcesEnqueued = 0;
1487 ModuleInfo *sourceInfo;
1488 while ((sourceInfo = modulesToLoadIter.next()))
1490 string moduleKey = sourceInfo->getModuleKey();
1492 dispatch_group_t loadingGroup = sourceInfo->getLoadingGroup();
1493 sourcesLoading.insert(loadingGroup);
1498 if (sourceInfo->getAttempted())
1503 string sourceCode = sourceInfo->getSourceCode();
1505 if (! shouldRecompileIfUnchanged)
1508 if (existingNodeClass && existingNodeClass->
getSourceCode() == sourceCode)
1514 sourceInfo->setAttempted(
true);
1516 dispatch_group_enter(loadingGroup);
1517 dispatch_group_enter(moduleSourceCompilersExist);
1521 queueItem->
sourcePath = sourceInfo->getFile()->path();
1523 queueItem->
sourceFile = sourceInfo->getFile();
1524 queueItem->
cachedModulesPath = sourceInfo->isSourceCodeOverridden() ? getOverriddenCompiledModuleCachePath() : getCompiledModuleCachePath();
1527 queueItem->
priority = { environmentIndex, sourceInfo->getLongestDownstreamPath() };
1528 moduleCompilationQueue->enqueue(queueItem);
1537 dispatch_queue_t queue = dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0);
1538 dispatch_async(queue, ^{
1539 for (
int i = 0; i < sourcesEnqueued; ++i)
1542 VUserLog(
"Compiling %s", queueItem->moduleKey.c_str());
1544 dispatch_async(queue, ^{
1553 notifyCompilers(set<VuoCompilerModule *>(), set< pair<VuoCompilerModule *, VuoCompilerModule *> >(), set<VuoCompilerModule *>(), issues,
false);
1559 auto moduleLoadedCallback = [=](){
1560 dispatch_group_leave(queueItemForCallback->
loadingGroup);
1561 moduleCompilationQueue->completed(queueItemForCallback);
1564 string ext = queueItem->sourceFile->extension();
1575 notifyCompilers(set<VuoCompilerModule *>(), set< pair<VuoCompilerModule *, VuoCompilerModule *> >(), set<VuoCompilerModule *>(), issues,
false);
1582 auto getType = [&compiler] (
const string &moduleKey) {
return compiler->
getType(moduleKey); };
1590 writeModuleToBitcode(module, queueItem->compiledModulePath);
1598 moduleSearchPathContentsChanged(queueItem->cachedModulesPath, queueItem->compiledModulePath, queueItem->sourceCode, moduleLoadedCallback, compiler, issues);
1601 moduleLoadedCallback();
1607 compiler->
compileModule(queueItem->sourcePath, queueItem->compiledModulePath, vector<string>());
1608 moduleSearchPathContentsChanged(queueItem->cachedModulesPath, queueItem->compiledModulePath, queueItem->sourceCode, moduleLoadedCallback, compiler);
1615 notifyCompilers(set<VuoCompilerModule *>(), set< pair<VuoCompilerModule *, VuoCompilerModule *> >(), set<VuoCompilerModule *>(), issues,
false);
1616 moduleLoadedCallback();
1622 compiler->compileSubcompositionString(queueItem->sourceCode, queueItem->compiledModulePath, moduleLoadedCallback,
this, NULL, queueItem->sourcePath);
1626 dispatch_group_leave(moduleSourceCompilersExist);
1631 return sourcesLoading;
1642 set<VuoCompilerModule *> VuoCompiler::Environment::unloadCompiledModules(
const set<string> &moduleKeys)
1644 set<VuoCompilerModule *> modulesUnloaded;
1646 for (set<string>::const_iterator i = moduleKeys.begin(); i != moduleKeys.end(); ++i)
1648 string moduleKey = *i;
1651 map<string, VuoCompilerNodeClass *>::iterator nodeClassIter = nodeClasses.find(moduleKey);
1652 if (nodeClassIter != nodeClasses.end())
1654 module = nodeClassIter->second;
1655 nodeClasses.erase(nodeClassIter);
1659 map<string, VuoCompilerType *>::iterator typeIter = types.find(moduleKey);
1660 if (typeIter != types.end())
1662 module = typeIter->second;
1663 types.erase(typeIter);
1667 map<string, VuoCompilerModule *>::iterator libraryModuleIter = libraryModules.find(moduleKey);
1668 if (libraryModuleIter != libraryModules.end())
1670 module = libraryModuleIter->second;
1671 libraryModules.erase(libraryModuleIter);
1678 modulesUnloaded.insert(module);
1679 removeFromDependencyGraph(module);
1684 return modulesUnloaded;
1692 void VuoCompiler::Environment::deleteModulesCompiledFromSourceCode(
const set<string> &moduleKeys)
1694 for (set<string>::const_iterator i = moduleKeys.begin(); i != moduleKeys.end(); ++i)
1696 string moduleKey = *i;
1698 map<string, ModuleInfo *>::iterator iter = moduleFilesAtSearchPath[getCompiledModuleCachePath()].find(moduleKey);
1699 if (iter != moduleFilesAtSearchPath[getCompiledModuleCachePath()].end())
1711 VuoCompilerModule * VuoCompiler::Environment::loadModule(ModuleInfo *moduleInfo)
1713 string moduleKey = moduleInfo->getModuleKey();
1720 __block
size_t inputDataBytes;
1721 __block
char *rawInputData;
1725 rawInputData = moduleInfo->getFile()->getContentsAsRawData(inputDataBytes);
1729 rawInputData = NULL;
1730 VUserLog(
"Warning: Couldn't load module '%s'. Its file may have been deleted. (%s)", moduleKey.c_str(), e.
what());
1736 char *processedInputData;
1738 processedInputData = loadModule_Pro0(moduleInfo, moduleKey, inputDataBytes, rawInputData);
1740 processedInputData = rawInputData;
1743 Module *module = NULL;
1744 bool moduleParseError = !processedInputData;
1745 if (!moduleParseError)
1747 string moduleReadError;
1749 module = readModuleFromBitcodeData(processedInputData, inputDataBytes, moduleReadError);
1751 free(processedInputData);
1755 moduleParseError =
true;
1757 string dir, file, ext;
1758 VuoFileUtilities::splitPath(moduleInfo->getFile()->isInArchive() ? moduleInfo->getFile()->getRelativePath() : moduleInfo->getFile()->path(), dir, file, ext);
1760 if (dir == getCompiledModuleCachePath())
1763 VUserLog(
"Error: Couldn't parse module '%s': %s.", moduleInfo->getFile()->getRelativePath().c_str(), moduleReadError.c_str());
1767 if (!moduleParseError)
1770 if (! moduleInfo->getFile()->isInArchive())
1771 modulePath = moduleInfo->getFile()->path();
1785 libraryModules[moduleKey] = compilerModule;
1790 map<string, VuoNodeSet *>::iterator nodeSetIter = nodeSetForName.find(nodeSet->
getName());
1791 if (nodeSetIter == nodeSetForName.end())
1793 nodeSetForName[nodeSet->
getName()] = nodeSet;
1798 nodeSet = nodeSetIter->second;
1800 compilerModule->getPseudoBase()->setNodeSet(nodeSet);
1804 loadModule_Pro1(rawInputData, processedInputData, compilerModule);
1807 compilerModule->setBuiltIn( isBuiltIn() );
1809 return compilerModule;
1813 destroyLlvmModule(module);
1832 removeFromDependencyGraph(oldNodeClass);
1833 destroyModule(oldNodeClass);
1836 nodeClasses[moduleKey] = newNodeClass;
1837 addToDependencyGraph(newNodeClass);
1843 DependencyGraphVertex *vertex = DependencyGraphVertex::vertexForDependency(module->
getPseudoBase()->
getModuleKey(), dependencyGraph);
1844 dependencyGraph->addVertex(vertex);
1846 vertex->setEnvironment(
this);
1853 for (set<string>::iterator i = dependencies.begin(); i != dependencies.end(); ++i)
1855 DependencyGraphVertex *depVertex = DependencyGraphVertex::vertexForDependency(*i, dependencyGraph);
1856 dependencyGraph->addEdge(vertex, depVertex);
1860 void VuoCompiler::Environment::removeFromDependencyGraph(
VuoCompilerModule *module)
1862 DependencyGraphVertex *vertex = DependencyGraphVertex::vertexForDependency(module->
getPseudoBase()->
getModuleKey(), dependencyGraph);
1863 dependencyGraph->removeVertex(vertex);
1876 void VuoCompiler::Environment::reifyPortTypes(
const map<string, VuoCompilerType *> &inheritedTypes)
1878 map<string, VuoCompilerType *> availableTypes = inheritedTypes;
1879 availableTypes.insert(types.begin(), types.end());
1881 for (map<string, VuoCompilerNodeClass *>::const_iterator i = nodeClasses.begin(); i != nodeClasses.end(); ++i)
1887 vector<VuoPortClass *> portClasses;
1888 portClasses.insert(portClasses.end(), inputPortClasses.begin(), inputPortClasses.end());
1889 portClasses.insert(portClasses.end(), outputPortClasses.begin(), outputPortClasses.end());
1891 for (vector<VuoPortClass *>::iterator j = portClasses.begin(); j != portClasses.end(); ++j)
1911 map<string, VuoCompilerType *>::iterator reifiedTypeIter = availableTypes.find(typeName);
1912 if (reifiedTypeIter != availableTypes.end())
1915 reifiedType = reifiedTypeIter->second;
1926 for (vector<VuoCompilerTriggerDescription *>::iterator j = triggers.begin(); j != triggers.end(); ++j)
1934 map<string, VuoCompilerType *>::iterator reifiedTypeIter = availableTypes.find(typeName);
1935 if (reifiedTypeIter != availableTypes.end())
1958 void VuoCompiler::Environment::getCacheableModulesAndDependencies(
bool builtIn,
bool installed, set<string> &cacheableModulesAndDependencies,
1959 set<string> &dylibsNeededToLinkToThisCache,
1960 set<string> &frameworksNeededToLinkToThisCache)
1962 if (isModuleCacheInitialized && ! isModuleCacheableDataDirty)
1964 cacheableModulesAndDependencies = moduleCacheContents;
1965 dylibsNeededToLinkToThisCache = moduleCacheDylibs;
1966 frameworksNeededToLinkToThisCache = moduleCacheFrameworks;
1974 map<string, VuoCompilerModule *> allModules;
1975 allModules.insert(nodeClasses.begin(), nodeClasses.end());
1976 allModules.insert(types.begin(), types.end());
1977 allModules.insert(libraryModules.begin(), libraryModules.end());
1979 set<string> dependencies;
1980 for (map<string, VuoCompilerModule *>::iterator i = allModules.begin(); i != allModules.end(); ++i)
1982 string moduleKey = i->first;
1987 if (module->requiresPro())
1995 cacheableModulesAndDependencies.insert(moduleKey);
1998 dependencies.insert(moduleDependencies.begin(), moduleDependencies.end());
2002 if (builtIn && installed)
2004 vector<string> coreDependencies = getCoreVuoDependencies();
2005 dependencies.insert(coreDependencies.begin(), coreDependencies.end());
2010 for (set<string>::iterator i = dependencies.begin(); i != dependencies.end(); ++i)
2012 string dependency = *i;
2013 if (allModules.find(dependency) == allModules.end())
2016 frameworksNeededToLinkToThisCache.insert(dependency);
2019 string dependencyPath = VuoCompiler::getLibraryPath(dependency, librarySearchPaths);
2020 if (! dependencyPath.empty())
2023 dylibsNeededToLinkToThisCache.insert(dependencyPath);
2025 cacheableModulesAndDependencies.insert(dependency);
2031 moduleCacheSuffix = (installed ?
"installed" :
"generated");
2032 dylibsNeededToLinkToThisCache.insert(moduleCachePath +
"/libVuoModuleCache-" + moduleCacheSuffix +
".dylib");
2034 moduleCacheDylibs = dylibsNeededToLinkToThisCache;
2035 moduleCacheFrameworks = frameworksNeededToLinkToThisCache;
2057 void VuoCompiler::Environment::useModuleCache(
bool shouldUseExistingCache,
VuoCompiler *compiler, set<string> cacheableModulesAndDependencies,
2058 set<string> dylibsNeededToLinkToCaches, set<string> frameworksNeededToLinkToCaches)
2062 static dispatch_once_t checked = 0;
2063 static bool prelinkCache =
true;
2064 dispatch_once(&checked, ^{
2066 bool result = CFPreferencesGetAppBooleanValue(CFSTR(
"prelinkCache"), CFSTR(
"org.vuo.Editor"), &valid);
2068 prelinkCache = result;
2072 VDebugLog(
"Ignoring the module cache since the 'prelinkCache' preference is false.");
2078 if (moduleCachePath.empty())
2083 string cacheDescription = string() +
"the cache of " + moduleCacheSuffix +
" modules at '" + moduleCachePath +
"'";
2084 if (isModuleCacheInitialized && ! isModuleCacheableDataDirty)
2086 VDebugLog(
"No need to recheck %s.", cacheDescription.c_str());
2092 VDebugLog(
"Checking if %s is up-to-date…", cacheDescription.c_str());
2094 bool isCacheUpToDate =
true;
2096 if (isModuleCacheInitialized && isModuleCacheableDataDirty)
2097 isCacheUpToDate =
false;
2099 isModuleCacheInitialized =
true;
2100 isModuleCacheableDataDirty =
false;
2102 const string dylibFileName =
"libVuoModuleCache-" + moduleCacheSuffix +
".dylib";
2103 const string indexFileName =
"moduleCache-" + moduleCacheSuffix +
".txt";
2104 string dylibPath = moduleCachePath +
"/" + dylibFileName;
2105 string indexPath = moduleCachePath +
"/" + indexFileName;
2109 bool dylibFileExists =
false;
2110 bool indexFileExists =
false;
2119 if (! (dirExists && dylibFileExists && indexFileExists))
2121 if (shouldUseExistingCache)
2122 throw VuoException(
"Trying to use the existing cache, but the cache doesn't exist.",
false);
2127 if (! indexFileExists)
2129 if (! dylibFileExists)
2132 isCacheUpToDate =
false;
2140 fileForLocking = moduleCacheFileForLocking[dylibPath];
2141 if (! fileForLocking)
2144 moduleCacheFileForLocking[dylibPath] = fileForLocking;
2148 VDebugLog(
"\tWarning: Couldn't lock for reading.");
2153 if (isCacheUpToDate)
2158 if (shouldUseExistingCache)
2159 throw VuoException(
"Trying to use the existing cache, but the cache doesn't contain readable data.",
false);
2161 isCacheUpToDate =
false;
2167 const char separator =
'\n';
2168 if (isCacheUpToDate || shouldUseExistingCache)
2171 string index = indexFile.getContentsAsString();
2174 moduleCacheContents.clear();
2175 moduleCacheContents.insert(actualIndex.begin(), actualIndex.end());
2177 if (shouldUseExistingCache)
2179 isModuleCacheAvailable =
true;
2186 if (isCacheUpToDate)
2188 if (moduleCacheContents.size() != cacheableModulesAndDependencies.size())
2189 isCacheUpToDate =
false;
2192 set<string> contentsInBoth;
2193 std::set_intersection(moduleCacheContents.begin(), moduleCacheContents.end(),
2194 cacheableModulesAndDependencies.begin(), cacheableModulesAndDependencies.end(),
2195 std::inserter(contentsInBoth, contentsInBoth.begin()));
2197 if (contentsInBoth.size() != cacheableModulesAndDependencies.size())
2198 isCacheUpToDate =
false;
2204 if (isCacheUpToDate)
2208 for (map<
string, map<string, ModuleInfo *> >::iterator i = moduleFilesAtSearchPath.begin(); i != moduleFilesAtSearchPath.end(); ++i)
2210 for (map<string, ModuleInfo *>::iterator j = i->second.begin(); j != i->second.end(); ++j)
2212 if (! j->second->isOlderThan(cacheLastModified))
2214 isCacheUpToDate =
false;
2223 if (isCacheUpToDate)
2227 isModuleCacheAvailable =
true;
2233 dispatch_async(moduleCacheBuildingQueue, ^{
2234 VDebugLog(
"Rebuilding %s…", cacheDescription.c_str());
2236 set<Module *> modulesToLink;
2237 set<string> librariesToLink;
2238 set<string> frameworksToLink;
2240 compiler->getLinkerInputs(cacheableModulesAndDependencies, Optimization_SmallBinary, modulesToLink, librariesToLink, frameworksToLink);
2242 librariesToLink.insert(dylibsNeededToLinkToCaches.begin(), dylibsNeededToLinkToCaches.end());
2243 set<string>::iterator iter = librariesToLink.find(dylibPath);
2244 if (iter != librariesToLink.end())
2245 librariesToLink.erase(iter);
2247 frameworksToLink.insert(frameworksNeededToLinkToCaches.begin(), frameworksNeededToLinkToCaches.end());
2250 bool gotLockForWriting =
false;
2255 if (! gotLockForWriting)
2256 throw VuoException(
"The cache file is being used by another process. "
2257 "If any composition windows are open from previous Vuo sessions, quit them. "
2258 "If any processes whose names start with \"VuoComposition\" or one of your composition file names appear in Activity Monitor, force-quit them.",
2262 string dir, file, ext;
2265 compiler->link(tmpPath, modulesToLink, librariesToLink, frameworksToLink,
true);
2273 getVuoFrameworkPath() +
"/Helpers/install_name_tool",
2280 vector<string> expectedContents(cacheableModulesAndDependencies.begin(), cacheableModulesAndDependencies.end());
2286 VDebugLog(
"\tWarning: Couldn't downgrade the lock back to reading.");
2288 dispatch_sync(environmentQueue, ^{
2289 moduleCacheContents = cacheableModulesAndDependencies;
2290 isModuleCacheAvailable =
true;
2296 if (gotLockForWriting)
2298 VDebugLog(
"\tWarning: Couldn't downgrade the lock back to reading.");
2300 VUserLog(
"Warning: Couldn't rebuild %s for the \"faster build\" optimization: %s", cacheDescription.c_str(), e.
what());
2308 VUserLog(
"Warning: Couldn't use %s for the \"faster build\" optimization: %s", cacheDescription.c_str(), e.
what());
2315 void VuoCompiler::Environment::waitForModuleCachesToBuild(
void)
2317 dispatch_sync(moduleCacheBuildingQueue, ^{});
2329 bool VuoCompiler::Environment::findInModuleCache(
const string &moduleOrDependency,
string &cachePath)
2331 if (isModuleCacheAvailable && moduleCacheContents.find(moduleOrDependency) != moduleCacheContents.end())
2333 cachePath = moduleCachePath +
"/libVuoModuleCache-" + moduleCacheSuffix +
".dylib";
2346 void VuoCompiler::Environment::modulesChanged(
void)
2348 isModuleCacheableDataDirty =
true;
2349 isModuleCacheAvailable =
false;
2355 bool VuoCompiler::Environment::isBuiltIn()
2357 return this == sharedEnvironments[0][0];
2363 string VuoCompiler::Environment::getName()
2367 else if (
this == sharedEnvironments[0][1])
2368 return "builtin (generated)";
2369 else if (
this == sharedEnvironments[1][0])
2371 else if (
this == sharedEnvironments[1][1])
2372 return "system (generated)";
2373 else if (
this == sharedEnvironments[2][0])
2375 else if (
this == sharedEnvironments[2][1])
2376 return "user (generated)";
2377 return "composition-local";
2383 void VuoCompiler::applyToInstalledEnvironments(
void (^doForEnvironment)(Environment *))
2385 dispatch_sync(environmentQueue, ^{
2386 for (vector< vector<Environment *> >::iterator i = environments.begin(); i != environments.end(); ++i) {
2387 doForEnvironment((*i)[0]);
2395 void VuoCompiler::applyToInstalledEnvironments(
void (^doForEnvironment)(Environment *,
bool *,
string),
bool *result,
string arg)
2397 dispatch_sync(environmentQueue, ^{
2398 for (vector< vector<Environment *> >::iterator i = environments.begin(); i != environments.end(); ++i) {
2399 doForEnvironment((*i)[0], result, arg);
2407 void VuoCompiler::applyToAllEnvironments(
void (^doForEnvironment)(Environment *environment))
2409 dispatch_sync(environmentQueue, ^{
2410 for (vector< vector<Environment *> >::iterator i = environments.begin(); i != environments.end(); ++i) {
2411 for (vector<Environment *>::iterator j = i->begin(); j != i->end(); ++j) {
2412 doForEnvironment(*j);
2432 shouldLoadAllModules =
true;
2433 hasLoadedAllModules =
false;
2434 modulesToLoadQueue = dispatch_queue_create(
"org.vuo.compiler.modules", NULL);
2435 moduleCacheBuilding = dispatch_group_create();
2436 dependencyGraph = NULL;
2437 compositionDependencyGraph = NULL;
2439 _shouldShowSplashWindow =
false;
2441 delegateQueue = dispatch_queue_create(
"org.vuo.compiler.delegate", NULL);
2443 string vuoFrameworkPath = getVuoFrameworkPath();
2444 if (! vuoFrameworkPath.empty())
2445 clangPath = vuoFrameworkPath +
"/Helpers/clang";
2447 clangPath = llvm::sys::Path(StringRef(LLVM_ROOT
"/bin/clang"));
2449 dispatch_sync(environmentQueue, ^{
2450 allCompilers.insert(
this);
2452 if (sharedEnvironments.empty())
2454 sharedEnvironments = vector< vector<Environment *> >(3, vector<Environment *>(2, NULL));
2455 for (vector< vector<Environment *> >::iterator i = sharedEnvironments.begin(); i != sharedEnvironments.end(); ++i) {
2456 for (vector<Environment *>::iterator j = i->begin(); j != i->end(); ++j) {
2457 *j = new Environment();
2461 vector<string> builtInModuleSearchPaths = Environment::getBuiltInModuleSearchPaths();
2462 for (vector<string>::iterator i = builtInModuleSearchPaths.begin(); i != builtInModuleSearchPaths.end(); ++i) {
2463 sharedEnvironments[0][0]->addModuleSearchPath(*i, false);
2466 vector<string> builtInHeaderSearchPaths = Environment::getBuiltInHeaderSearchPaths();
2467 for (vector<string>::iterator i = builtInHeaderSearchPaths.begin(); i != builtInHeaderSearchPaths.end(); ++i) {
2468 sharedEnvironments[0][0]->addHeaderSearchPath(*i);
2471 vector<string> builtInLibrarySearchPaths = Environment::getBuiltInLibrarySearchPaths();
2472 for (vector<string>::iterator i = builtInLibrarySearchPaths.begin(); i != builtInLibrarySearchPaths.end(); ++i) {
2473 sharedEnvironments[0][0]->addLibrarySearchPath(*i);
2476 vector<string> builtInFrameworkSearchPaths = Environment::getBuiltInFrameworkSearchPaths();
2477 for (vector<string>::iterator i = builtInFrameworkSearchPaths.begin(); i != builtInFrameworkSearchPaths.end(); ++i) {
2478 sharedEnvironments[0][0]->addFrameworkSearchPath(*i);
2492 string vuoFrameworkPath = getVuoFrameworkPath();
2493 if (! vuoFrameworkPath.empty())
2495 vector<string> moduleCachePaths(3);
2496 moduleCachePaths[0] = vuoFrameworkPath +
"/Modules/Builtin";
2497 moduleCachePaths[1] = VuoFileUtilities::getCachePath() +
"/System";
2498 moduleCachePaths[2] = VuoFileUtilities::getCachePath() +
"/User";
2500 for (size_t i = 0; i < moduleCachePaths.size(); ++i)
2502 string moduleCachePath = moduleCachePaths[i];
2504 sharedEnvironments[i][0]->setModuleCachePath(moduleCachePath);
2505 sharedEnvironments[i][0]->addModuleSearchPath(moduleCachePath +
"/Modules", false);
2507 sharedEnvironments[i][1]->setModuleCachePath(moduleCachePath);
2513 setCompositionPath(compositionPath);
2525 dispatch_sync(environmentQueue, ^{
2526 allCompilers.erase(
this);
2529 dispatch_group_wait(moduleCacheBuilding, DISPATCH_TIME_FOREVER);
2530 dispatch_release(moduleCacheBuilding);
2532 for (vector< vector<Environment *> >::iterator i = environments.begin(); i != environments.end(); ++i)
2533 (*i)[0]->removeCompilerToNotify(
this);
2535 dispatch_sync(delegateQueue, ^{});
2537 dispatch_release(modulesToLoadQueue);
2538 dispatch_release(delegateQueue);
2540 delete dependencyGraph;
2541 delete compositionDependencyGraph;
2547 void VuoCompiler::reset(
void)
2549 dispatch_group_wait(moduleSourceCompilersExist, DISPATCH_TIME_FOREVER);
2551 for (vector< vector<Environment *> >::iterator i = sharedEnvironments.begin(); i != sharedEnvironments.end(); ++i)
2553 (*i)[0]->stopWatchingModuleSearchPaths();
2554 dispatch_sync((*i)[0]->moduleSearchPathContentsChangedQueue, ^{});
2557 for (map<
string, vector<Environment *> >::iterator i = environmentsForCompositionFamily.begin(); i != environmentsForCompositionFamily.end(); ++i)
2559 (i->second)[0]->stopWatchingModuleSearchPaths();
2560 dispatch_sync((i->second)[0]->moduleSearchPathContentsChangedQueue, ^{});
2563 for (vector< vector<Environment *> >::iterator i = sharedEnvironments.begin(); i != sharedEnvironments.end(); ++i)
2564 for (vector<Environment *>::iterator j = i->begin(); j != i->end(); ++j)
2567 for (map<
string, vector<Environment *> >::iterator i = environmentsForCompositionFamily.begin(); i != environmentsForCompositionFamily.end(); ++i)
2568 for (vector<Environment *>::iterator j = i->second.begin(); j != i->second.end(); ++j)
2571 allCompilers.clear();
2572 sharedEnvironments.clear();
2573 environmentsForCompositionFamily.clear();
2583 dispatch_async(delegateQueue, ^{
2584 this->delegate = delegate;
2599 string compositionModulesDir;
2600 string compositionBaseDir;
2601 lastCompositionIsSubcomposition =
false;
2602 if (! compositionPath.empty())
2610 string compositionDir;
2613 lastCompositionIsSubcomposition = (compositionDir == compositionModulesDir);
2618 dispatch_sync(environmentQueue, ^{
2619 if (! environments.empty() && compositionBaseDir == lastCompositionBaseDir) {
2622 lastCompositionBaseDir = compositionBaseDir;
2626 Environment *oldCompositionFamilyInstalledEnvironment =
nullptr;
2627 vector<Environment *> compositionEnvironments;
2628 if (! environments.empty())
2630 for (vector< vector<Environment *> >::iterator i = environments.begin(); i != environments.end(); ++i) {
2631 (*i)[0]->removeCompilerToNotify(this);
2634 if (environments.size() >= 5) {
2635 oldCompositionFamilyInstalledEnvironment = environments[3][0];
2638 compositionEnvironments = environments.back();
2640 environments.clear();
2647 bool isCompositionInSharedEnvironment =
false;
2648 for (vector< vector<Environment *> >::iterator i = sharedEnvironments.begin(); i != sharedEnvironments.end(); ++i)
2650 environments.push_back(*i);
2652 vector<string> moduleSearchPaths = (*i)[0]->getModuleSearchPaths();
2653 for (vector<string>::iterator j = moduleSearchPaths.begin(); j != moduleSearchPaths.end(); ++j)
2655 string moduleSearchPath = *j;
2656 VuoFileUtilities::canonicalizePath(moduleSearchPath);
2657 if (moduleSearchPath == compositionModulesDir)
2659 isCompositionInSharedEnvironment = true;
2664 if (isCompositionInSharedEnvironment) {
2671 Environment *newCompositionFamilyInstalledEnvironment =
nullptr;
2672 if (! isCompositionInSharedEnvironment && ! compositionPath.empty())
2674 vector<Environment *> compositionFamilyEnvironments = environmentsForCompositionFamily[compositionBaseDir];
2675 if (compositionFamilyEnvironments.empty())
2677 compositionFamilyEnvironments = vector<Environment *>(2, NULL);
2678 compositionFamilyEnvironments[0] = new Environment();
2679 compositionFamilyEnvironments[1] = new Environment();
2680 environmentsForCompositionFamily[compositionBaseDir] = compositionFamilyEnvironments;
2684 compositionFamilyEnvironments[0]->addModuleSearchPath(compositionModulesDir);
2688 string moduleCachePath = getCachePathForComposition(compositionBaseDir);
2690 compositionFamilyEnvironments[0]->setModuleCachePath(moduleCachePath);
2691 compositionFamilyEnvironments[0]->addModuleSearchPath(moduleCachePath +
"/Modules", false);
2693 compositionFamilyEnvironments[1]->setModuleCachePath(moduleCachePath);
2695 environments.push_back(compositionFamilyEnvironments);
2697 newCompositionFamilyInstalledEnvironment = compositionFamilyEnvironments[0];
2702 if (compositionEnvironments.empty())
2704 compositionEnvironments = vector<Environment *>(2, NULL);
2705 compositionEnvironments[0] = new Environment();
2706 compositionEnvironments[1] = new Environment();
2708 environments.push_back(compositionEnvironments);
2710 for (vector< vector<Environment *> >::iterator i = environments.begin(); i != environments.end(); ++i) {
2711 (*i)[0]->addCompilerToNotify(this);
2714 delete dependencyGraph;
2715 delete compositionDependencyGraph;
2716 dependencyGraph = makeDependencyNetwork(environments, ^
VuoDirectedAcyclicGraph * (Environment *env) {
return env->getDependencyGraph(); });
2717 compositionDependencyGraph = makeDependencyNetwork(environments, ^
VuoDirectedAcyclicGraph * (Environment *env) {
return env->getCompositionDependencyGraph(); });
2722 if (oldCompositionFamilyInstalledEnvironment != newCompositionFamilyInstalledEnvironment)
2724 auto getModules = [] (Environment *env)
2726 map<string, VuoCompilerModule *> modules;
2729 for (
auto i : env->getNodeClasses()) {
2732 for (
auto i : env->getTypes()) {
2735 for (
auto i : env->getLibraryModules()) {
2742 map<string, VuoCompilerModule *> modulesAdded = getModules(newCompositionFamilyInstalledEnvironment);
2743 map<string, VuoCompilerModule *> modulesRemoved = getModules(oldCompositionFamilyInstalledEnvironment);
2745 map<string, pair<VuoCompilerModule *, VuoCompilerModule *> > modulesModified;
2746 for (map<string, VuoCompilerModule *>::iterator
add = modulesAdded.begin();
add != modulesAdded.end(); )
2748 map<string, VuoCompilerModule *>::iterator rem = modulesRemoved.find(
add->first);
2749 if (rem != modulesRemoved.end())
2751 modulesModified[
add->first] = make_pair(rem->second,
add->second);
2752 modulesAdded.erase(
add++);
2753 modulesRemoved.erase(rem);
2761 if (! (modulesAdded.empty() && modulesModified.empty() && modulesRemoved.empty()) )
2764 VuoCompilerDelegate::LoadedModulesData *delegateData =
new VuoCompilerDelegate::LoadedModulesData(set< pair<VuoCompilerModule *, VuoCompilerModule *> >(), set<VuoCompilerModule *>(), issues);
2765 delegateData->retain();
2767 Environment *scopeEnvironment = newCompositionFamilyInstalledEnvironment;
2768 if (! scopeEnvironment) {
2769 scopeEnvironment = compositionEnvironments.at(0);
2772 loadedModules(modulesAdded, modulesModified, modulesRemoved, issues, delegateData, scopeEnvironment);
2786 if (!graphForEnvironment)
2791 for (vector< vector<Environment *> >::const_iterator i = environments.begin(); i != environments.end(); ++i)
2795 network->
addEdge(graphForEnvironment(i->at(0)), graphForEnvironment(i->at(1)));
2799 for (vector< vector<Environment *> >::const_iterator ii = environments.begin(); ii != i; ++ii)
2801 network->
addEdge(graphForEnvironment(i->at(0)), graphForEnvironment(ii->at(0)));
2802 network->
addEdge(graphForEnvironment(i->at(1)), graphForEnvironment(ii->at(0)));
2829 void VuoCompiler::loadModulesIfNeeded(
const set<string> &moduleKeys)
2831 __block
bool willLoadAllModules =
false;
2832 if (moduleKeys.empty())
2834 dispatch_sync(modulesToLoadQueue, ^{
2835 if (shouldLoadAllModules && ! hasLoadedAllModules) {
2836 willLoadAllModules =
true;
2837 hasLoadedAllModules =
true;
2842 if (! willLoadAllModules && moduleKeys.empty())
2847 __block set<dispatch_group_t> sourcesLoading;
2848 dispatch_sync(environmentQueue, ^{
2849 sourcesLoading = loadModulesAndSources(moduleKeys, set<string>(), set<string>(),
2850 moduleKeys, set<string>(), set<string>(),
2851 willLoadAllModules,
false,
nullptr,
nullptr,
nullptr,
"");
2857 for (set<dispatch_group_t>::iterator i = sourcesLoading.begin(); i != sourcesLoading.end(); ++i)
2859 dispatch_group_wait(*i, DISPATCH_TIME_FOREVER);
2860 dispatch_release(*i);
2872 set<dispatch_group_t> VuoCompiler::loadModulesAndSources(
const set<string> &modulesAddedKeys,
const set<string> &modulesModifiedKeys,
const set<string> &modulesRemovedKeys,
2873 const set<string> &sourcesAddedKeys,
const set<string> &sourcesModifiedKeys,
const set<string> &sourcesRemovedKeys,
2874 bool willLoadAllModules,
bool shouldRecompileSourcesIfUnchanged,
2875 Environment *currentEnvironment,
VuoCompilerIssues *issuesForCurrentEnvironment,
2876 std::function<
void(
void)> moduleLoadedCallback,
const string &moduleAddedOrModifiedSourceCode)
2888 map<Environment *, set<string> > modulesAdded;
2889 map<Environment *, set<string> > modulesModified;
2890 map<Environment *, set<string> > modulesRemoved;
2891 map<Environment *, set<string> > sourcesAdded;
2892 map<Environment *, set<string> > sourcesModified;
2893 map<Environment *, set<string> > sourcesRemoved;
2894 map<Environment *, set<string> > potentialSpecializedModules;
2896 if (currentEnvironment)
2898 modulesAdded[currentEnvironment] = modulesAddedKeys;
2899 modulesModified[currentEnvironment] = modulesModifiedKeys;
2900 modulesRemoved[currentEnvironment] = modulesRemovedKeys;
2901 sourcesAdded[currentEnvironment] = sourcesAddedKeys;
2902 sourcesModified[currentEnvironment] = sourcesModifiedKeys;
2903 sourcesRemoved[currentEnvironment] = sourcesRemovedKeys;
2907 Environment *genEnv =
nullptr;
2908 if (! willLoadAllModules)
2914 int scope = environments.size() - (lastCompositionIsSubcomposition ? 2 : 1);
2915 genEnv = environments.at(scope).at(1);
2916 potentialSpecializedModules[genEnv] = modulesAddedKeys;
2919 for (vector< vector<Environment *> >::iterator i = environments.begin(); i != environments.end(); ++i)
2921 Environment *env = (*i).at(0);
2923 ModuleInfoIterator modulesAddedIter = (willLoadAllModules ? env->listAllModules() : env->listModules(modulesAddedKeys));
2924 ModuleInfoIterator sourcesAddedIter = (willLoadAllModules ? env->listAllSourceFiles() : env->listSourceFiles(sourcesAddedKeys));
2925 ModuleInfoIterator sourcesModifiedIter = (willLoadAllModules ? env->listAllSourceFiles() : env->listSourceFiles(sourcesModifiedKeys));
2927 ModuleInfo *moduleInfo;
2928 while ((moduleInfo = modulesAddedIter.next()))
2930 string moduleKey = moduleInfo->getModuleKey();
2932 modulesAdded[env].insert(moduleKey);
2934 if (! willLoadAllModules)
2936 auto foundIter = potentialSpecializedModules[genEnv].find(moduleKey);
2937 if (foundIter != potentialSpecializedModules[genEnv].end())
2938 potentialSpecializedModules[genEnv].erase(foundIter);
2944 auto isCompiledModuleAtSameSearchPath = [&env] (ModuleInfo *sourceInfo)
2946 ModuleInfo *compiledModuleInfo = env->listModule(sourceInfo->getModuleKey());
2947 return (compiledModuleInfo && compiledModuleInfo->getSearchPath() == sourceInfo->getSearchPath());
2950 while ((moduleInfo = sourcesAddedIter.next()))
2952 if (isCompiledModuleAtSameSearchPath(moduleInfo))
2955 sourcesAdded[env].insert( moduleInfo->getModuleKey() );
2958 while ((moduleInfo = sourcesModifiedIter.next()))
2960 if (isCompiledModuleAtSameSearchPath(moduleInfo))
2963 sourcesModified[env].insert( moduleInfo->getModuleKey() );
2968 map<Environment *, VuoCompilerIssues *> issues;
2969 for (vector< vector<Environment *> >::iterator i = environments.begin(); i != environments.end(); ++i)
2971 Environment *env = (*i).at(0);
2972 issues[env] = (env == currentEnvironment && issuesForCurrentEnvironment ? issuesForCurrentEnvironment :
new VuoCompilerIssues());
2977 for (vector< vector<Environment *> >::iterator i = environments.begin(); i != environments.end(); ++i)
2979 Environment *env = (*i).at(0);
2984 set<VuoDirectedAcyclicGraph::Vertex *> circularDependencies = env->getCompositionDependencyGraph()->getCycleVertices();
2986 set<string> sourcesAddedModified;
2987 sourcesAddedModified.insert(sourcesAdded[env].begin(), sourcesAdded[env].end());
2988 sourcesAddedModified.insert(sourcesModified[env].begin(), sourcesModified[env].end());
2990 for (set<string>::iterator j = sourcesAddedModified.begin(); j != sourcesAddedModified.end(); ++j)
2992 string moduleKey = *j;
2995 for (set<VuoDirectedAcyclicGraph::Vertex *>::iterator k = circularDependencies.begin(); k != circularDependencies.end(); ++k)
2997 DependencyGraphVertex *vertex =
static_cast<DependencyGraphVertex *
>(*k);
2998 if (vertex->getDependency() == moduleKey)
3007 sourcesAdded[env].erase(moduleKey);
3008 sourcesModified[env].erase(moduleKey);
3011 "Subcomposition contains itself",
3012 "%moduleKey contains an instance of itself, "
3013 "or contains another subcomposition that contains an instance of %moduleKey.");
3014 issue.setModuleKey(moduleKey);
3016 ModuleInfo *sourceInfo = env->listSourceFile(moduleKey);
3018 issue.setFilePath(sourceInfo->getFile()->path());
3020 issues[env]->append(issue);
3027 for (
const vector<Environment *> &envs : environments)
3029 Environment *env = envs.at(0);
3031 set<string> sourcesAddedModified;
3032 sourcesAddedModified.insert(sourcesAdded[env].begin(), sourcesAdded[env].end());
3033 sourcesAddedModified.insert(sourcesModified[env].begin(), sourcesModified[env].end());
3035 for (
const string &moduleKey : sourcesAddedModified)
3038 int pathLength = env->getCompositionDependencyGraph()->getLongestDownstreamPath(vertex);
3040 ModuleInfo *sourceInfo = env->listSourceFile(moduleKey);
3041 sourceInfo->setLongestDownstreamPath(pathLength);
3049 map<Environment *, set<string> > modulesDepOnModulesModified;
3050 map<Environment *, set<string> > sourcesDepOnModulesModified;
3051 map<Environment *, set<string> > modulesDepOnModulesRemoved;
3052 map<Environment *, set<string> > sourcesDepOnModulesRemoved;
3055 __block map<Environment *, set<string> > modulesDepOnModulesModified_otherCompiler;
3056 __block map<Environment *, set<string> > sourcesDepOnModulesModified_otherCompiler;
3057 __block map<Environment *, set<string> > modulesDepOnModulesRemoved_otherCompiler;
3058 __block map<Environment *, set<string> > sourcesDepOnModulesRemoved_otherCompiler;
3060 vector<VuoDirectedAcyclicNetwork *> searchDependencyGraphs;
3062 searchDependencyGraphs.push_back(compiler->dependencyGraph);
3064 VuoDirectedAcyclicGraph *currentEnvironmentDependencyGraph = (currentEnvironment ? currentEnvironment->getDependencyGraph() :
nullptr);
3066 findDependentModulesAndSources(modulesModified, searchDependencyGraphs, currentEnvironmentDependencyGraph,
3067 modulesDepOnModulesModified, modulesDepOnModulesModified_otherCompiler,
3068 sourcesDepOnModulesModified, sourcesDepOnModulesModified_otherCompiler);
3070 findDependentModulesAndSources(modulesRemoved, searchDependencyGraphs, currentEnvironmentDependencyGraph,
3071 modulesDepOnModulesRemoved, modulesDepOnModulesRemoved_otherCompiler,
3072 sourcesDepOnModulesRemoved, sourcesDepOnModulesRemoved_otherCompiler);
3074 set<Environment *> otherEnvironments;
3075 for (map<Environment *, set<string> >::iterator i = modulesDepOnModulesModified_otherCompiler.begin(); i != modulesDepOnModulesModified_otherCompiler.end(); ++i)
3076 otherEnvironments.insert(i->first);
3077 for (map<Environment *, set<string> >::iterator i = sourcesDepOnModulesModified_otherCompiler.begin(); i != sourcesDepOnModulesModified_otherCompiler.end(); ++i)
3078 otherEnvironments.insert(i->first);
3079 for (map<Environment *, set<string> >::iterator i = modulesDepOnModulesRemoved_otherCompiler.begin(); i != modulesDepOnModulesRemoved_otherCompiler.end(); ++i)
3080 otherEnvironments.insert(i->first);
3081 for (map<Environment *, set<string> >::iterator i = sourcesDepOnModulesRemoved_otherCompiler.begin(); i != sourcesDepOnModulesRemoved_otherCompiler.end(); ++i)
3082 otherEnvironments.insert(i->first);
3084 for (Environment *env : otherEnvironments)
3088 for (
const vector<Environment *> &ee : c->environments)
3089 for (Environment *e : ee)
3093 goto foundOtherCompiler;
3097 dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{
3098 dispatch_sync(environmentQueue, ^{
3099 otherCompiler->loadModulesAndSources(set<string>(), modulesDepOnModulesModified_otherCompiler[env], modulesDepOnModulesRemoved_otherCompiler[env],
3100 set<string>(), sourcesDepOnModulesModified_otherCompiler[env], sourcesDepOnModulesRemoved_otherCompiler[env],
3101 false,
true, env,
nullptr,
nullptr,
"");
3111 map<Environment *, set<VuoCompilerModule *> > actualModulesRemoved;
3112 for (
const vector<Environment *> &envs : environments)
3114 for (Environment *env : envs)
3116 set<string> modulesToUnload;
3117 modulesToUnload.insert(modulesRemoved[env].begin(), modulesRemoved[env].end());
3118 modulesToUnload.insert(modulesModified[env].begin(), modulesModified[env].end());
3119 modulesToUnload.insert(modulesDepOnModulesRemoved[env].begin(), modulesDepOnModulesRemoved[env].end());
3120 modulesToUnload.insert(modulesDepOnModulesModified[env].begin(), modulesDepOnModulesModified[env].end());
3122 actualModulesRemoved[env] = env->unloadCompiledModules(modulesToUnload);
3124 if (!env->isBuiltIn() && !actualModulesRemoved[env].empty())
3126 set<string> actualModulesRemovedKeys;
3127 for (
auto m : actualModulesRemoved[env])
3128 actualModulesRemovedKeys.insert(m->getPseudoBase()->getModuleKey());
3141 map<Environment *, set<string> > modulesToLoad;
3142 map<Environment *, map<string, string> > modulesToLoadSourceCode;
3143 for (vector< vector<Environment *> >::iterator i = environments.begin(); i != environments.end(); ++i)
3145 Environment *env = (*i).at(0);
3147 if (! modulesAdded[env].empty())
3148 modulesToLoad[env].insert(modulesAdded[env].begin(), modulesAdded[env].end());
3149 if (! modulesModified[env].empty())
3150 modulesToLoad[env].insert(modulesModified[env].begin(), modulesModified[env].end());
3151 if (! modulesDepOnModulesModified[env].empty())
3152 modulesToLoad[env].insert(modulesDepOnModulesModified[env].begin(), modulesDepOnModulesModified[env].end());
3154 if (env == currentEnvironment && moduleLoadedCallback)
3156 if (modulesAdded[env].size() == 1)
3157 modulesToLoadSourceCode[env][*modulesAdded[env].begin()] = moduleAddedOrModifiedSourceCode;
3158 else if (modulesModified[env].size() == 1)
3159 modulesToLoadSourceCode[env][*modulesModified[env].begin()] = moduleAddedOrModifiedSourceCode;
3163 map<Environment *, set<VuoCompilerModule *> > actualModulesAdded;
3164 while (! modulesToLoad.empty())
3166 set<string> dependenciesToLoad;
3167 map<Environment *, set<string> > potentialSpecializedDependencies;
3168 for (vector< vector<Environment *> >::iterator i = environments.begin(); i != environments.end(); ++i)
3170 Environment *env = (*i).at(0);
3171 Environment *genEnv = (*i).at(1);
3173 set<VuoCompilerModule *> actualModulesLoaded = env->loadCompiledModules(modulesToLoad[env], modulesToLoadSourceCode[env]);
3175 actualModulesAdded[env].insert(actualModulesLoaded.begin(), actualModulesLoaded.end());
3176 modulesToLoad.erase(env);
3178 set<string> actualModulesLoadedKeys;
3179 for (set<VuoCompilerModule *>::iterator j = actualModulesLoaded.begin(); j != actualModulesLoaded.end(); ++j)
3181 set<string> dependencies = (*j)->getDependencies();
3182 dependenciesToLoad.insert(dependencies.begin(), dependencies.end());
3183 potentialSpecializedDependencies[genEnv].insert(dependencies.begin(), dependencies.end());
3184 actualModulesLoadedKeys.insert((*j)->getPseudoBase()->getModuleKey());
3187 if (!env->isBuiltIn() && !actualModulesLoadedKeys.empty())
3191 for (vector< vector<Environment *> >::iterator i = environments.begin(); i != environments.end(); ++i)
3193 Environment *env = (*i).at(0);
3195 ModuleInfoIterator dependenciesInEnv = env->listModules(dependenciesToLoad);
3196 ModuleInfo *moduleInfo;
3197 while ((moduleInfo = dependenciesInEnv.next()))
3199 modulesToLoad[env].insert( moduleInfo->getModuleKey() );
3201 for (map<Environment *, set<string> >::iterator j = potentialSpecializedDependencies.begin(); j != potentialSpecializedDependencies.end(); ++j)
3203 auto foundIter = j->second.find( moduleInfo->getModuleKey() );
3204 if (foundIter != j->second.end())
3205 j->second.erase(foundIter);
3210 for (vector< vector<Environment *> >::iterator i = environments.begin(); i != environments.end(); ++i)
3212 Environment *genEnv = (*i).at(1);
3213 potentialSpecializedModules[genEnv].insert(potentialSpecializedDependencies[genEnv].begin(), potentialSpecializedDependencies[genEnv].end());
3220 set<dispatch_group_t> specializedModulesLoading;
3221 for (vector< vector<Environment *> >::iterator i = environments.begin(); i != environments.end(); ++i)
3223 Environment *genEnv = (*i).at(1);
3224 set<dispatch_group_t> s = genEnv->loadSpecializedModules(potentialSpecializedModules[genEnv],
this,
llvmQueue);
3225 specializedModulesLoading.insert(s.begin(), s.end());
3230 if (moduleLoadedCallback)
3231 moduleLoadedCallback();
3235 map<Environment *, set< pair<VuoCompilerModule *, VuoCompilerModule *> > > actualModulesModified;
3236 for (vector< vector<Environment *> >::iterator i = environments.begin(); i != environments.end(); ++i)
3238 Environment *env = (*i).at(0);
3240 for (set<VuoCompilerModule *>::iterator
add = actualModulesAdded[env].begin();
add != actualModulesAdded[env].end(); )
3242 set<VuoCompilerModule *>::iterator rem;
3243 for (rem = actualModulesRemoved[env].begin(); rem != actualModulesRemoved[env].end(); ++rem)
3244 if ((*rem)->getPseudoBase()->getModuleKey() == (*add)->getPseudoBase()->getModuleKey())
3247 if (rem != actualModulesRemoved[env].end())
3249 actualModulesModified[env].insert( make_pair(*rem, *
add) );
3250 actualModulesRemoved[env].erase(rem);
3251 actualModulesAdded[env].erase(
add++);
3260 bool wereModulesAddedOrModified =
false;
3261 for (vector< vector<Environment *> >::iterator i = environments.begin(); i != environments.end(); ++i)
3263 Environment *env = (*i).at(0);
3264 if (! (actualModulesAdded[env].empty() && actualModulesModified[env].empty()) )
3266 wereModulesAddedOrModified =
true;
3271 if (wereModulesAddedOrModified)
3273 map<string, VuoCompilerType *> inheritedTypes;
3274 for (
const vector<Environment *> &envs : environments)
3276 for (Environment *env : envs)
3278 env->reifyPortTypes(inheritedTypes);
3279 map<string, VuoCompilerType *> envTypes = env->getTypes();
3280 inheritedTypes.insert(envTypes.begin(), envTypes.end());
3289 for (vector< vector<Environment *> >::iterator i = environments.begin(); i != environments.end(); ++i)
3291 Environment *env = (*i).at(0);
3293 set<string> sourcesToUnload;
3294 sourcesToUnload.insert(sourcesRemoved[env].begin(), sourcesRemoved[env].end());
3295 sourcesToUnload.insert(sourcesDepOnModulesRemoved[env].begin(), sourcesDepOnModulesRemoved[env].end());
3296 if (! sourcesToUnload.empty())
3298 string moduleSearchPath = env->getModuleSearchPaths().front();
3300 dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{
3303 dispatch_sync(environmentQueue, ^{
3304 otherCompiler->loadModulesAndSources(set<string>(), set<string>(), sourcesToUnload,
3305 set<string>(), set<string>(), set<string>(),
3306 false,
false, env,
nullptr,
nullptr,
"");
3309 delete otherCompiler;
3313 if (!env->isBuiltIn() && !sourcesToUnload.empty())
3316 env->deleteModulesCompiledFromSourceCode(sourcesToUnload);
3324 map<Environment *, set<string> > sourcesDepOnModulesAdded;
3326 map<Environment *, set<string> > modulesDepOnModulesAdded;
3327 __block map<Environment *, set<string> > modulesDepOnModulesAdded_otherCompiler;
3328 __block map<Environment *, set<string> > sourcesDepOnModulesAdded_otherCompiler;
3330 map<Environment *, set<string> > actualModuleKeysAdded;
3331 for (
const vector<Environment *> &envs : environments)
3333 Environment *env = envs.at(0);
3338 vector<VuoDirectedAcyclicNetwork *> searchDependencyGraphs;
3339 searchDependencyGraphs.push_back(compositionDependencyGraph);
3340 for (map<
string, vector<Environment *> >::iterator ii = environmentsForCompositionFamily.begin(); ii != environmentsForCompositionFamily.end(); ++ii)
3342 vector< vector<Environment *> > otherEnvs = sharedEnvironments;
3343 otherEnvs.push_back(ii->second);
3345 searchDependencyGraphs.push_back(other);
3348 VuoDirectedAcyclicGraph *currentEnvironmentDependencyGraph = (currentEnvironment ? currentEnvironment->getCompositionDependencyGraph() :
nullptr);
3350 findDependentModulesAndSources(actualModuleKeysAdded, searchDependencyGraphs, currentEnvironmentDependencyGraph,
3351 modulesDepOnModulesAdded, modulesDepOnModulesAdded_otherCompiler,
3352 sourcesDepOnModulesAdded, sourcesDepOnModulesAdded_otherCompiler);
3354 set<Environment *> otherEnvironments;
3355 for (map<Environment *, set<string> >::iterator i = sourcesDepOnModulesAdded_otherCompiler.begin(); i != sourcesDepOnModulesAdded_otherCompiler.end(); ++i)
3356 otherEnvironments.insert(i->first);
3358 for (Environment *env : otherEnvironments)
3360 dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{
3361 string moduleSearchPath = env->getModuleSearchPaths().front();
3364 dispatch_sync(environmentQueue, ^{
3365 otherCompiler->loadModulesAndSources(set<string>(), set<string>(), set<string>(),
3366 sourcesDepOnModulesAdded_otherCompiler[env], set<string>(), set<string>(),
3367 false,
true, env,
nullptr,
nullptr,
"");
3370 delete otherCompiler;
3375 set<dispatch_group_t> sourcesLoading;
3376 for (vector< vector<Environment *> >::iterator i = environments.begin(); i != environments.end(); ++i)
3378 Environment *env = (*i).at(0);
3380 set<string> sourcesToCompile;
3381 sourcesToCompile.insert(sourcesAdded[env].begin(), sourcesAdded[env].end());
3382 sourcesToCompile.insert(sourcesModified[env].begin(), sourcesModified[env].end());
3384 if (sourcesToCompile.size() == 0)
3387 set<dispatch_group_t> s = env->compileModulesFromSourceCode(sourcesToCompile, shouldRecompileSourcesIfUnchanged);
3388 sourcesLoading.insert(s.begin(), s.end());
3391 for (vector< vector<Environment *> >::iterator i = environments.begin(); i != environments.end(); ++i)
3393 Environment *env = (*i).at(0);
3395 set<string> sourcesToCompile;
3396 sourcesToCompile.insert(sourcesDepOnModulesAdded[env].begin(), sourcesDepOnModulesAdded[env].end());
3397 sourcesToCompile.insert(sourcesDepOnModulesModified[env].begin(), sourcesDepOnModulesModified[env].end());
3399 if (sourcesToCompile.size() == 0)
3402 env->compileModulesFromSourceCode(sourcesToCompile,
true);
3407 for (vector< vector<Environment *> >::iterator i = environments.begin(); i != environments.end(); ++i)
3409 Environment *env = (*i).at(0);
3410 env->notifyCompilers(actualModulesAdded[env], actualModulesModified[env], actualModulesRemoved[env], issues[env]);
3417 for (
const dispatch_group_t &group : sourcesLoading)
3418 dispatch_retain(group);
3420 set<dispatch_group_t> loadingGroups;
3421 loadingGroups.insert(specializedModulesLoading.begin(), specializedModulesLoading.end());
3422 loadingGroups.insert(sourcesLoading.begin(), sourcesLoading.end());
3423 return loadingGroups;
3437 void VuoCompiler::findDependentModulesAndSources(map<Environment *, set<string> > &changedModules,
3438 const vector<VuoDirectedAcyclicNetwork *> &searchDependencyGraphs,
3440 map<Environment *, set<string> > &modulesDepOnChangedModules_this,
3441 map<Environment *, set<string> > &modulesDepOnChangedModules_other,
3442 map<Environment *, set<string> > &sourcesDepOnChangedModules_this,
3443 map<Environment *, set<string> > &sourcesDepOnChangedModules_other)
3445 for (
const vector<Environment *> &envs : environments)
3447 Environment *env = envs.at(0);
3449 for (
const string &module : changedModules[env])
3451 set<VuoDirectedAcyclicGraph::Vertex *> dependents;
3454 vector<VuoDirectedAcyclicGraph::Vertex *> moduleVertices;
3455 if (currentEnvironmentDependencyGraph)
3461 moduleVertices.push_back(mv);
3464 moduleVertices = searchDependencyGraph->findVertex(module);
3468 DependencyGraphVertex *moduleVertex =
static_cast<DependencyGraphVertex *
>(moduleVertexRaw);
3469 if (moduleVertex->getEnvironment())
3471 vector<VuoDirectedAcyclicGraph::Vertex *> upstream = searchDependencyGraph->getUpstreamVertices(moduleVertex);
3472 dependents.insert(upstream.begin(), upstream.end());
3477 set< pair<Environment *, string> > dependentsMap;
3480 DependencyGraphVertex *v =
static_cast<DependencyGraphVertex *
>(dependentVertexRaw);
3481 Environment *dependentEnv = v->getEnvironment();
3485 string dependent = v->getDependency();
3487 dependentsMap.insert({dependentEnv, dependent});
3493 for (
auto i : envs.at(1)->getNodeClasses())
3497 dependentsMap.insert({envs.at(1), i.first});
3500 for (
auto i : dependentsMap)
3502 Environment *dependentEnv = i.first;
3503 string dependent = i.second;
3507 if (changedModules[dependentEnv].find(dependent) != changedModules[dependentEnv].end())
3510 ModuleInfo *foundSourceInfo = dependentEnv->listSourceFile(dependent);
3511 ModuleInfo *foundModuleInfo = dependentEnv->listModule(dependent);
3513 bool belongsToCurrentCompiler =
false;
3514 for (
const vector<Environment *> &envs2 : environments)
3516 if (find(envs2.begin(), envs2.end(), dependentEnv) != envs2.end())
3518 belongsToCurrentCompiler =
true;
3523 map<Environment *, set<string> > *whicheverDependents =
nullptr;
3524 ModuleInfo *moduleInfo =
nullptr;
3525 if (foundSourceInfo)
3527 moduleInfo = foundSourceInfo;
3528 whicheverDependents = (belongsToCurrentCompiler ? &sourcesDepOnChangedModules_this : &sourcesDepOnChangedModules_other);
3530 else if (foundModuleInfo)
3532 moduleInfo = foundModuleInfo;
3533 whicheverDependents = (belongsToCurrentCompiler ? &modulesDepOnChangedModules_this : &modulesDepOnChangedModules_other);
3537 whicheverDependents = (belongsToCurrentCompiler ? &modulesDepOnChangedModules_this : &modulesDepOnChangedModules_other);
3540 (*whicheverDependents)[dependentEnv].insert(dependent);
3542 moduleInfo->setAttempted(
false);
3551 void VuoCompiler::loadedModules(map<string, VuoCompilerModule *> modulesAdded,
3552 map<
string, pair<VuoCompilerModule *, VuoCompilerModule *> > modulesModified,
3553 map<string, VuoCompilerModule *> modulesRemoved,
3554 VuoCompilerIssues *issues,
void *delegateDataV, Environment *currentEnvironment)
3566 auto findVersionsOfModule = [
this, currentEnvironment] (
const string &moduleKey)
3568 vector< pair<Environment *, VuoCompilerModule *> > moduleVersions;
3569 for (
const vector<Environment *> &envs : environments)
3571 Environment *env = envs.at(0);
3573 if (module || env == currentEnvironment)
3574 moduleVersions.push_back( make_pair(env, module) );
3576 return moduleVersions;
3579 for (map<string, VuoCompilerModule *>::iterator i = modulesAdded.begin(); i != modulesAdded.end(); )
3581 string moduleKey = i->first;
3584 vector< pair<Environment *, VuoCompilerModule *> > moduleVersions = findVersionsOfModule(moduleKey);
3586 if (moduleVersions.size() > 1)
3588 modulesAdded.erase(i++);
3590 if (moduleVersions.back().second == moduleAdded)
3592 VuoCompilerModule *moduleSuperseded = moduleVersions.at(moduleVersions.size()-2).second;
3593 modulesModified[moduleKey] = make_pair(moduleSuperseded, moduleAdded);
3600 for (map<
string, pair<VuoCompilerModule *, VuoCompilerModule *> >::iterator i = modulesModified.begin(); i != modulesModified.end(); )
3602 string moduleKey = i->first;
3605 vector< pair<Environment *, VuoCompilerModule *> > moduleVersions = findVersionsOfModule(moduleKey);
3607 if (moduleVersions.size() > 1 && moduleVersions.back().second != moduleModified)
3608 modulesModified.erase(i++);
3613 for (map<string, VuoCompilerModule *>::iterator i = modulesRemoved.begin(); i != modulesRemoved.end(); )
3615 string moduleKey = i->first;
3618 vector< pair<Environment *, VuoCompilerModule *> > moduleVersions = findVersionsOfModule(moduleKey);
3620 if (moduleVersions.size() > 1)
3622 modulesRemoved.erase(i++);
3624 if (moduleVersions.back().first == currentEnvironment)
3626 VuoCompilerModule *moduleUnsuperseded = moduleVersions.at(moduleVersions.size()-2).second;
3627 modulesModified[moduleKey] = make_pair(moduleRemoved, moduleUnsuperseded);
3634 dispatch_async(delegateQueue, ^{
3635 VuoCompilerDelegate::LoadedModulesData *delegateData =
static_cast<VuoCompilerDelegate::LoadedModulesData *
>(delegateDataV);
3637 if (delegate && ! (modulesAdded.empty() && modulesModified.empty() && modulesRemoved.empty() && issues->
isEmpty()))
3639 delegate->enqueueData(delegateData);
3640 delegate->
loadedModules(modulesAdded, modulesModified, modulesRemoved, issues);
3644 delegateData->release();
3654 void VuoCompiler::loadNodeClassGeneratedAtRuntime(
VuoCompilerNodeClass *nodeClass, Environment *env)
3656 Module *module = nodeClass->
getModule();
3660 setTargetForModule(nodeClass->
getModule());
3664 dispatch_sync(environmentQueue, ^{
3665 env->replaceNodeClass(nodeClass);
3668 __block map<string, VuoCompilerType *> inheritedTypes;
3669 void (^envReifyPortTypes)(Environment *) = ^
void (Environment *env) {
3670 env->reifyPortTypes(inheritedTypes);
3671 map<string, VuoCompilerType *> currentTypes = env->getTypes();
3672 inheritedTypes.insert(currentTypes.begin(), currentTypes.end());
3674 applyToAllEnvironments(envReifyPortTypes);
3683 reifyGenericPortTypes(node->getBase());
3691 void VuoCompiler::reifyGenericPortTypes(
VuoNode *node)
3701 vector<VuoPort *> ports;
3702 ports.insert(ports.end(), inputPorts.begin(), inputPorts.end());
3703 ports.insert(ports.end(), outputPorts.begin(), outputPorts.end());
3705 for (vector<VuoPort *>::iterator j = ports.begin(); j != ports.end(); ++j)
3752 vector<string> allIncludePaths = includePaths;
3753 string preprocessedInputPath = inputPath;
3755 string tmpPreprocessedInputDir;
3756 string dir, file, ext;
3761 string preprocessedInputContents = inputContents;
3763 if (inputContents != preprocessedInputContents)
3766 allIncludePaths.push_back(dir.empty() ?
"." : dir);
3768 preprocessedInputPath = tmpPreprocessedInputDir +
"/" + file +
"." + ext;
3773 else if (ext ==
"fs")
3779 auto getType = [
this] (
const string &moduleKey) {
return this->
getType(moduleKey); };
3784 writeModuleToBitcode(module, outputPath);
3794 vector<string> extraArgs;
3795 for (vector<string>::iterator i = allIncludePaths.begin(); i != allIncludePaths.end(); ++i)
3797 extraArgs.push_back(
"-I");
3798 extraArgs.push_back(*i);
3801 string macosxSdkFolder =
"/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/";
3804 extraArgs.push_back(
"-isysroot");
3805 extraArgs.push_back(macosxSdkFolder +
"MacOSX10.10.sdk");
3808 __block vector<string> headerSearchPaths;
3809 void (^envGetHeaderSearchPaths)(Environment *) = ^
void (Environment *env) {
3810 vector<string> result = env->getHeaderSearchPaths();
3811 headerSearchPaths.insert(headerSearchPaths.end(), result.begin(), result.end());
3813 applyToInstalledEnvironments(envGetHeaderSearchPaths);
3815 __block Module *module;
3817 module = readModuleFromC(preprocessedInputPath, headerSearchPaths, extraArgs);
3820 if (! tmpPreprocessedInputDir.empty())
3821 remove(tmpPreprocessedInputDir.c_str());
3825 "",
"%moduleKey couldn't be compiled as a node class, type, or library. Check the macOS Console for details.");
3832 if (! compilerModule)
3834 VUserLog(
"Error: Didn't recognize '%s' as a node class, type, or library.", inputPath.c_str());
3838 setTargetForModule(module, target);
3839 writeModuleToBitcode(module, outputPath);
3848 Module * VuoCompiler::compileCompositionToModule(
VuoCompilerComposition *composition,
const string &moduleKey,
bool isTopLevelComposition,
3851 composition->
check(issues);
3853 reifyGenericPortTypes(composition);
3856 isTopLevelComposition,
3858 if (telemetry ==
"console")
3861 __block Module *module;
3864 setTargetForModule(module, target);
3886 Module *module = compileCompositionToModule(composition, moduleKey, isTopLevelComposition, issues);
3889 writeModuleToBitcode(module, outputPath);
3906 VDebugLog(
"Compiling '%s'…", inputPath.c_str());
3913 "",
"The composition file couldn't be read or was empty.");
3949 delete baseComposition;
3955 void VuoCompiler::compileSubcompositionString(
const string &compositionString,
const string &outputPath,
3956 std::function<
void(
void)> moduleLoadedCallback, Environment *environment,
3962 bool compilationSucceeded =
false;
3966 compilationSucceeded =
true;
3974 if (! compilationSucceeded)
3980 string outputDir, file, ext;
3984 environment->moduleSearchPathContentsChanged(outputDir, outputPath, compositionString, moduleLoadedCallback,
this, issues);
4002 linkCompositionToCreateExecutableOrDynamicLibrary(inputPath, outputPath, optimization,
false, rPath);
4023 linkCompositionToCreateExecutableOrDynamicLibrary(inputPath, outputPath, optimization,
true);
4040 void VuoCompiler::linkCompositionToCreateExecutableOrDynamicLibrary(
string compiledCompositionPath,
string linkedCompositionPath,
4041 Optimization optimization,
bool isDylib,
string rPath)
4047 shouldLoadAllModules =
false;
4049 set<string> dependencies = getDependenciesForComposition(compiledCompositionPath);
4050 dependencies.insert(getRuntimeDependency());
4052 dependencies.insert(getRuntimeMainDependency());
4054 set<Module *> modules;
4055 set<string> libraries;
4056 set<string> frameworks;
4057 getLinkerInputs(dependencies, optimization, modules, libraries, frameworks);
4059 libraries.insert(compiledCompositionPath);
4061 link(linkedCompositionPath, modules, libraries, frameworks, isDylib, rPath);
4087 set<string> allDependencies = getDependenciesForComposition(compiledCompositionPath);
4088 set<string> addedDependencies;
4089 std::set_difference(allDependencies.begin(), allDependencies.end(),
4090 carriedOverDependencies.begin(), carriedOverDependencies.end(),
4091 std::inserter(addedDependencies, addedDependencies.end()));
4097 set<string> carriedOverExternalLibraries = runningCompositionLibraries->
getExternalLibraries();
4102 string nonUnloadableResourcePath;
4103 string unloadableResourcePath;
4104 set<string> nonUnloadableDependencies;
4105 set<string> unloadableDependencies;
4106 map<string, set<string> > builtInCacheDependencies;
4107 map<string, set<string> > userCacheDependencies;
4108 set<string> builtInLibraries;
4109 set<string> userLibraries;
4110 set<string> addedExternalLibraries;
4111 set<string> addedFrameworks;
4112 set<string> allFrameworks;
4113 if (! addedDependencies.empty())
4115 set<string> builtInModuleAndLibraryDependencies;
4116 set<string> userModuleAndLibraryDependencies;
4117 set<Module *> builtInModules;
4118 set<Module *> userModules;
4121 builtInModuleAndLibraryDependencies, userModuleAndLibraryDependencies, builtInCacheDependencies, userCacheDependencies,
4122 builtInModules, userModules, builtInLibraries, userLibraries, addedExternalLibraries, addedFrameworks);
4124 allFrameworks.insert(carriedOverFrameworks.begin(), carriedOverFrameworks.end());
4125 allFrameworks.insert(addedFrameworks.begin(), addedFrameworks.end());
4127 string dir, linkedCompositionFile, ext;
4130 if (! builtInModules.empty() || builtInLibraries.size() > builtInCacheDependencies.size())
4133 nonUnloadableDependencies = builtInModuleAndLibraryDependencies;
4135 set<string> librariesForNonUnloadableResource = builtInLibraries;
4136 librariesForNonUnloadableResource.insert(carriedOverNonUnloadableLibraries.begin(), carriedOverNonUnloadableLibraries.end());
4137 librariesForNonUnloadableResource.insert(carriedOverExternalLibraries.begin(), carriedOverExternalLibraries.end());
4138 librariesForNonUnloadableResource.insert(addedExternalLibraries.begin(), addedExternalLibraries.end());
4140 link(nonUnloadableResourcePath, builtInModules, librariesForNonUnloadableResource, allFrameworks,
true);\
4142 for (set<string>::iterator i = builtInLibraries.begin(); i != builtInLibraries.end(); )
4145 builtInLibraries.erase(i++);
4150 for (set<string>::iterator i = addedExternalLibraries.begin(); i != addedExternalLibraries.end(); )
4153 addedExternalLibraries.erase(i++);
4159 if (! userModules.empty() || userLibraries.size() > userCacheDependencies.size())
4162 unloadableDependencies = userModuleAndLibraryDependencies;
4164 set<string> librariesForUnloadableResource = userLibraries;
4165 librariesForUnloadableResource.insert(builtInLibraries.begin(), builtInLibraries.end());
4166 librariesForUnloadableResource.insert(carriedOverUnloadableLibraries.begin(), carriedOverUnloadableLibraries.end());
4167 librariesForUnloadableResource.insert(carriedOverNonUnloadableLibraries.begin(), carriedOverNonUnloadableLibraries.end());
4168 librariesForUnloadableResource.insert(carriedOverExternalLibraries.begin(), carriedOverExternalLibraries.end());
4169 librariesForUnloadableResource.insert(addedExternalLibraries.begin(), addedExternalLibraries.end());
4170 if (! nonUnloadableResourcePath.empty())
4171 librariesForUnloadableResource.insert(nonUnloadableResourcePath);
4173 link(unloadableResourcePath, userModules, librariesForUnloadableResource, allFrameworks,
true);
4175 for (set<string>::iterator i = userLibraries.begin(); i != userLibraries.end(); )
4178 userLibraries.erase(i++);
4187 set<string> vuoRuntimePaths;
4189 set<Module *> modules;
4190 set<string> libraries;
4191 set<string> frameworks;
4193 set<string> dependencies;
4194 dependencies.insert(getRuntimeDependency());
4196 vuoRuntimePaths = libraries;
4202 set<Module *> modules;
4203 set<string> libraries;
4205 libraries.insert(compiledCompositionPath);
4206 libraries.insert(carriedOverExternalLibraries.begin(), carriedOverExternalLibraries.end());
4207 libraries.insert(addedExternalLibraries.begin(), addedExternalLibraries.end());
4208 libraries.insert(carriedOverNonUnloadableLibraries.begin(), carriedOverNonUnloadableLibraries.end());
4209 libraries.insert(carriedOverUnloadableLibraries.begin(), carriedOverUnloadableLibraries.end());
4210 libraries.insert(builtInLibraries.begin(), builtInLibraries.end());
4211 libraries.insert(userLibraries.begin(), userLibraries.end());
4212 if (! nonUnloadableResourcePath.empty())
4213 libraries.insert(nonUnloadableResourcePath);
4214 if (! unloadableResourcePath.empty())
4215 libraries.insert(unloadableResourcePath);
4216 libraries.insert(vuoRuntimePaths.begin(), vuoRuntimePaths.end());
4217 link(linkedCompositionPath, modules, libraries, allFrameworks,
true);
4222 if (! nonUnloadableResourcePath.empty())
4225 if (! unloadableResourcePath.empty())
4228 for (map<
string, set<string> >::iterator i = builtInCacheDependencies.begin(); i != builtInCacheDependencies.end(); ++i)
4231 for (map<
string, set<string> >::iterator i = userCacheDependencies.begin(); i != userCacheDependencies.end(); ++i)
4244 set<string> VuoCompiler::getDependenciesForComposition(
const string &compiledCompositionPath)
4246 VDebugLog(
"Gathering dependencies for '%s'…", compiledCompositionPath.c_str());
4249 __block set<string> directDependencies;
4251 Module *module = readModuleFromBitcode(compiledCompositionPath);
4255 delete compilerModule;
4261 auto deps = getDependenciesForComposition(directDependencies,
true);
4280 set<string> directDependencies;
4289 vector<VuoPublishedPort *> publishedPorts;
4290 publishedPorts.insert(publishedPorts.end(), publishedInputPorts.begin(), publishedInputPorts.end());
4291 publishedPorts.insert(publishedPorts.end(), publishedOutputPorts.begin(), publishedOutputPorts.end());
4294 if (publishedPort->getClass()->hasCompiler())
4310 directDependencies.insert(dependency);
4315 return directDependencies;
4327 return getDependenciesForComposition(directDependencies,
false);
4337 __block vector<string> librarySearchPaths;
4338 applyToInstalledEnvironments(^
void (Environment *env) {
4339 vector<string> result = env->getLibrarySearchPaths();
4340 librarySearchPaths.insert(librarySearchPaths.end(), result.begin(), result.end());
4343 set<string> dylibDeps;
4344 for (
string dep : getDependenciesForComposition(composition))
4346 string path = getLibraryPath(dep, librarySearchPaths);
4348 dylibDeps.insert(path);
4367 set<string> VuoCompiler::getDependenciesForComposition(
const set<string> &directDependencies,
bool checkCompatibility)
4370 for (set<string>::const_iterator i = directDependencies.begin(); i != directDependencies.end(); ++i)
4373 set<string> dependencies;
4374 for (set<string>::iterator i = directDependencies.begin(); i != directDependencies.end(); ++i)
4376 string moduleKey = *i;
4378 dependencies.insert(moduleKey);
4381 vector<VuoDirectedAcyclicGraph::Vertex *> firstPassVertices = dependencyGraph->
findVertex(moduleKey);
4382 set<VuoDirectedAcyclicGraph::Vertex *> firstPassDependencies(firstPassVertices.begin(), firstPassVertices.end());
4383 for (vector<VuoDirectedAcyclicGraph::Vertex *>::iterator j = firstPassVertices.begin(); j != firstPassVertices.end(); ++j)
4386 firstPassDependencies.insert(downstream.begin(), downstream.end());
4390 for (set<VuoDirectedAcyclicGraph::Vertex *>::iterator j = firstPassDependencies.begin(); j != firstPassDependencies.end(); ++j)
4392 DependencyGraphVertex *v =
static_cast<DependencyGraphVertex *
>(*j);
4397 vector<VuoDirectedAcyclicGraph::Vertex *> moduleVertices = dependencyGraph->
findVertex(moduleKey);
4398 set<VuoDirectedAcyclicGraph::Vertex *> moduleDependencies(moduleVertices.begin(), moduleVertices.end());
4399 for (vector<VuoDirectedAcyclicGraph::Vertex *>::iterator j = moduleVertices.begin(); j != moduleVertices.end(); ++j)
4402 moduleDependencies.insert(downstream.begin(), downstream.end());
4406 set<string> dependenciesToAdd;
4407 set<string> incompatibleDependencies;
4408 for (set<VuoDirectedAcyclicGraph::Vertex *>::iterator j = moduleDependencies.begin(); j != moduleDependencies.end(); ++j)
4410 DependencyGraphVertex *v =
static_cast<DependencyGraphVertex *
>(*j);
4411 if (! checkCompatibility || ! v->getEnvironment() || v->isCompatible())
4412 dependenciesToAdd.insert(v->getDependency());
4414 incompatibleDependencies.insert(v->getDependency());
4417 if (! checkCompatibility || incompatibleDependencies.empty())
4419 dependencies.insert(dependenciesToAdd.begin(), dependenciesToAdd.end());
4426 string dependencyTargetString;
4431 for (set<string>::iterator i = incompatibleDependencies.begin(); i != incompatibleDependencies.end(); ++i)
4440 dependencyTargetString = dependencyTargets.
toString();
4443 dependencyTargetString =
"(unknown operating systems)";
4445 string modulePlaceholder = (module ?
"%module" :
"%moduleKey");
4447 "Node incompatible with operating system",
4448 modulePlaceholder +
" is only compatible with " + dependencyTargetString +
4449 ", so this composition can't run on your macOS version (" + compositionTargets.
toString() +
").");
4451 issue.setModuleKey(moduleKey);
4457 vector<string> coreDependencies = getCoreVuoDependencies();
4458 dependencies.insert(coreDependencies.begin(), coreDependencies.end());
4460 return dependencies;
4467 void VuoCompiler::getLinkerInputs(
const set<string> &dependencies, Optimization optimization,
4468 set<Module *> &modules, set<string> &libraries, set<string> &frameworks)
4470 set<string> builtInModuleAndLibraryDependencies;
4471 set<string> userModuleAndLibraryDependencies;
4472 map<string, set<string> > builtInCacheDependencies;
4473 map<string, set<string> > userCacheDependencies;
4474 set<Module *> builtInModules;
4475 set<Module *> userModules;
4476 set<string> builtInLibraries;
4477 set<string> userLibraries;
4478 set<string> externalLibraries;
4480 getLinkerInputs(dependencies, optimization,
4481 builtInModuleAndLibraryDependencies, userModuleAndLibraryDependencies, builtInCacheDependencies, userCacheDependencies,
4482 builtInModules, userModules, builtInLibraries, userLibraries, externalLibraries, frameworks);
4484 modules.insert(builtInModules.begin(), builtInModules.end());
4485 modules.insert(userModules.begin(), userModules.end());
4486 libraries.insert(builtInLibraries.begin(), builtInLibraries.end());
4487 libraries.insert(userLibraries.begin(), userLibraries.end());
4488 libraries.insert(externalLibraries.begin(), externalLibraries.end());
4504 void VuoCompiler::getLinkerInputs(
const set<string> &dependencies, Optimization optimization,
4505 set<string> &builtInModuleAndLibraryDependencies, set<string> &userModuleAndLibraryDependencies,
4506 map<
string, set<string> > &builtInCacheDependencies, map<
string, set<string> > &userCacheDependencies,
4507 set<Module *> &builtInModules, set<Module *> &userModules,
4508 set<string> &builtInLibraries, set<string> &userLibraries,
4509 set<string> &externalLibraries, set<string> &externalFrameworks)
4512 if (shouldUseModuleCache)
4515 __block vector<string> librarySearchPaths;
4516 void (^envGetLibrarySearchPaths)(Environment *) = ^
void (Environment *env) {
4517 vector<string> result = env->getLibrarySearchPaths();
4518 librarySearchPaths.insert(librarySearchPaths.end(), result.begin(), result.end());
4520 applyToInstalledEnvironments(envGetLibrarySearchPaths);
4522 for (set<string>::iterator i = dependencies.begin(); i != dependencies.end(); ++i)
4524 string dependency = *i;
4526 bool foundInCache =
false;
4527 string moduleCachePath;
4528 bool isInBuiltInModuleCache =
false;
4529 if (shouldUseModuleCache)
4530 foundInCache = findInModuleCache(dependency, moduleCachePath, isInBuiltInModuleCache);
4534 if (isInBuiltInModuleCache)
4536 builtInLibraries.insert(moduleCachePath);
4537 builtInCacheDependencies[moduleCachePath].insert(dependency);
4541 userLibraries.insert(moduleCachePath);
4542 userCacheDependencies[moduleCachePath].insert(dependency);
4548 void (^envFindModule)(Environment *) = ^
void (Environment *env) {
4553 applyToAllEnvironments(envFindModule);
4563 if (module->isBuiltIn())
4564 builtInLibraries.insert(modulePath);
4566 userLibraries.insert(modulePath);
4570 if (module->isBuiltIn())
4571 builtInModules.insert(module->getModule());
4573 userModules.insert(module->getModule());
4576 if (module->isBuiltIn())
4577 builtInModuleAndLibraryDependencies.insert(dependency);
4579 userModuleAndLibraryDependencies.insert(dependency);
4585 externalFrameworks.insert(dependency);
4588 string dependencyPath = getLibraryPath(dependency, librarySearchPaths);
4589 if (! dependencyPath.empty())
4590 externalLibraries.insert(dependencyPath);
4592 VUserLog(
"Warning: Could not locate dependency '%s'.", dependency.c_str());
4604 string VuoCompiler::getLibraryPath(
const string &dependency, vector<string> librarySearchPaths)
4611 if (dependency !=
"crypto"
4612 && dependency !=
"ssl")
4613 librarySearchPaths.push_back(
"/usr/lib");
4615 for (
auto &path : librarySearchPaths)
4617 vector<string> variations;
4618 variations.push_back(path +
"/" + dependency);
4619 variations.push_back(path +
"/lib" + dependency);
4620 variations.push_back(path +
"/lib" + dependency +
".dylib");
4621 variations.push_back(path +
"/lib" + dependency +
".a");
4622 for (
auto &variation : variations)
4637 void VuoCompiler::useModuleCache(
bool shouldUseExistingBuiltInCaches,
bool shouldUseExistingOtherCaches)
4639 loadModulesIfNeeded();
4643 dispatch_sync(environmentQueue, ^{
4644 set<string> dylibsForCachesOfInstalledModules;
4645 set<string> frameworksForCachesOfInstalledModules;
4646 for (vector< vector<Environment *> >::iterator i = environments.begin(); i != environments.end(); ++i)
4648 bool builtin = (i == environments.begin());
4649 set<string> dylibsForCacheOfGeneratedModules;
4650 set<string> frameworksForCacheOfGeneratedModules;
4652 for (int j = i->size() - 1; j >= 0; --j)
4654 Environment *env = i->at(j);
4655 bool installed = (j == 0);
4657 set<string> cacheableModulesAndDependencies;
4658 set<string> dylibsNeededToLinkToThisCache;
4659 set<string> frameworksNeededToLinkToThisCache;
4660 env->getCacheableModulesAndDependencies(builtin, installed, cacheableModulesAndDependencies,
4661 dylibsNeededToLinkToThisCache, frameworksNeededToLinkToThisCache);
4663 set<string> accumulatedDylibs;
4664 accumulatedDylibs.insert(dylibsNeededToLinkToThisCache.begin(), dylibsNeededToLinkToThisCache.end());
4665 accumulatedDylibs.insert(dylibsForCachesOfInstalledModules.begin(), dylibsForCachesOfInstalledModules.end());
4666 accumulatedDylibs.insert(dylibsForCacheOfGeneratedModules.begin(), dylibsForCacheOfGeneratedModules.end());
4668 set<string> accumulatedFrameworks;
4669 accumulatedFrameworks.insert(frameworksNeededToLinkToThisCache.begin(), frameworksNeededToLinkToThisCache.end());
4670 accumulatedFrameworks.insert(frameworksForCachesOfInstalledModules.begin(), frameworksForCachesOfInstalledModules.end());
4671 accumulatedFrameworks.insert(frameworksForCacheOfGeneratedModules.begin(), frameworksForCacheOfGeneratedModules.end());
4673 bool shouldUseExistingCache = (builtin ? shouldUseExistingBuiltInCaches : shouldUseExistingOtherCaches);
4674 env->useModuleCache(shouldUseExistingCache, this, cacheableModulesAndDependencies,
4675 accumulatedDylibs, accumulatedFrameworks);
4679 dylibsForCachesOfInstalledModules.insert(dylibsNeededToLinkToThisCache.begin(), dylibsNeededToLinkToThisCache.end());
4680 frameworksForCachesOfInstalledModules.insert(frameworksNeededToLinkToThisCache.begin(), frameworksNeededToLinkToThisCache.end());
4684 dylibsForCacheOfGeneratedModules.insert(dylibsNeededToLinkToThisCache.begin(), dylibsNeededToLinkToThisCache.end());
4685 frameworksForCacheOfGeneratedModules.insert(frameworksNeededToLinkToThisCache.begin(), frameworksNeededToLinkToThisCache.end());
4691 Environment::waitForModuleCachesToBuild();
4702 bool VuoCompiler::findInModuleCache(
const string &moduleOrDependency,
string &cachePath,
bool &isBuiltinCache)
4704 __block
bool found =
false;
4705 __block
string outPath;
4706 __block
bool outBuiltin;
4707 dispatch_sync(environmentQueue, ^{
4708 for (vector< vector<Environment *> >::iterator i = environments.begin(); i != environments.end(); ++i)
4710 bool builtin = (i == environments.begin());
4712 for (int j = i->size() - 1; j >= 0; --j)
4714 Environment *env = i->at(j);
4717 bool resultFound = env->findInModuleCache(moduleOrDependency, resultPath);
4721 outPath = resultPath;
4722 outBuiltin = builtin;
4728 cachePath = outPath;
4729 isBuiltinCache = outBuiltin;
4742 dispatch_group_async(moduleCacheBuilding, dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{
4743 useModuleCache(
true,
false);
4761 vuoFrameworkInProgressPath = vuoFrameworkPath;
4764 compiler.useModuleCache(
false,
true);
4776 unsigned long maxSeconds = 30 * 24 * 60 * 60;
4779 for (set<VuoFileUtilities::File *>::iterator i = cacheDirs.begin(); i != cacheDirs.end(); ++i)
4781 string path = (*i)->path();
4783 string file = (*i)->basename();
4784 if (file !=
"Builtin" && file !=
"System" && file !=
"User")
4787 if (fileSeconds > maxSeconds)
4793 string pidAsString = file.substr(Environment::pidCacheDirPrefix.length());
4794 int pid = atoi(pidAsString.c_str());
4795 if (kill(pid, 0) != 0)
4809 dispatch_sync(modulesToLoadQueue, ^{
4810 this->shouldLoadAllModules = shouldLoadAllModules;
4825 void VuoCompiler::link(
string outputPath,
const set<Module *> &modules,
const set<string> &libraries,
const set<string> &frameworks,
bool isDylib,
string rPath)
4827 VDebugLog(
"Linking '%s'…", outputPath.c_str());
4835 Module *compositeModule =
new Module(
"composite", getGlobalContext());
4836 setTargetForModule(compositeModule);
4837 for (set<Module *>::const_iterator i = modules.begin(); i != modules.end(); ++i)
4840 if (Linker::LinkModules(compositeModule, *i, Linker::PreserveSource, &error))
4841 VUserLog(
"Error: Failed to link compositeModule: %s", error.c_str());
4843 writeModuleToBitcode(compositeModule, compositeModulePath);
4844 delete compositeModule;
4851 llvm::sys::Path clangPath = getClangPath();
4853 vector<const char *> args;
4854 vector<char *> argsToFree;
4855 args.push_back(clangPath.c_str());
4857 args.push_back(compositeModulePath.c_str());
4859 vector<string> coreDependencies = getCoreVuoDependencies();
4860 for (set<string>::const_iterator i = libraries.begin(); i != libraries.end(); ++i)
4862 string library = *i;
4864 for (vector<string>::iterator j = coreDependencies.begin(); j != coreDependencies.end(); ++j)
4866 string coreDependency = *j;
4868 args.push_back(
"-force_load");
4876 library = libraryObject;
4879 char *libraryZ = strdup(library.c_str());
4880 args.push_back(libraryZ);
4881 argsToFree.push_back(libraryZ);
4885 vector<string> frameworkArguments;
4887 __block vector<string> frameworkSearchPaths;
4888 void (^envGetFrameworkSearchPaths)(Environment *) = ^
void (Environment *env) {
4889 vector<string> result = env->getFrameworkSearchPaths();
4890 frameworkSearchPaths.insert(frameworkSearchPaths.end(), result.begin(), result.end());
4892 applyToInstalledEnvironments(envGetFrameworkSearchPaths);
4894 for (vector<string>::const_iterator i = frameworkSearchPaths.begin(); i != frameworkSearchPaths.end(); ++i)
4898 frameworkArguments.push_back(a);
4899 char *frameworkArgument = strdup(a.c_str());
4900 args.push_back(frameworkArgument);
4901 argsToFree.push_back(frameworkArgument);
4904 for (set<string>::const_iterator i = frameworks.begin(); i != frameworks.end(); ++i)
4906 args.push_back(
"-framework");
4908 string frameworkName = *i;
4909 frameworkName = frameworkName.substr(0, frameworkName.length() -
string(
".framework").length());
4910 char *frameworkNameZ = strdup(frameworkName.c_str());
4911 args.push_back(frameworkNameZ);
4912 argsToFree.push_back(frameworkNameZ);
4916 llvm::sys::Path cRuntimePath;
4917 llvm::sys::Path crt1Path;
4918 string vuoFrameworkPath = getVuoFrameworkPath();
4919 string vuoFrameworkContainingFolder = vuoFrameworkPath +
"/..";
4920 if (! vuoFrameworkPath.empty())
4922 cRuntimePath = vuoFrameworkPath +
"/Modules/";
4923 crt1Path = cRuntimePath;
4924 crt1Path.appendComponent(
"crt1.o");
4929 if (!isDylib && crt1Path.canRead())
4931 args.push_back(
"-nostartfiles");
4932 args.push_back(crt1Path.c_str());
4936 args.push_back(
"-Xlinker");
4937 args.push_back(
"--no-demangle");
4940 args.push_back(
"-v");
4943 args.push_back(
"-dynamiclib");
4945 args.push_back(
"-Xlinker");
4946 args.push_back(
"-headerpad_max_install_names");
4950 args.push_back(
"-rpath");
4951 string rPathArg = (rPath.empty() ? vuoFrameworkContainingFolder : rPath);
4952 args.push_back(rPathArg.c_str());
4955 args.push_back(
"-rpath");
4956 args.push_back(LLVM_ROOT
"/lib");
4959 args.push_back(
"-std=c++11");
4960 args.push_back(
"-stdlib=libc++");
4963 clang::DiagnosticOptions *diagOptions =
new clang::DiagnosticOptions();
4964 clang::TextDiagnosticPrinter *diagClient =
new clang::TextDiagnosticPrinter(llvm::errs(), diagOptions);
4965 diagClient->setPrefix(clangPath.str());
4966 IntrusiveRefCntPtr<clang::DiagnosticIDs> DiagID(
new clang::DiagnosticIDs());
4967 clang::DiagnosticsEngine Diags(DiagID, diagOptions, diagClient);
4969 clang::driver::Driver TheDriver(args[0],
"x86_64-apple-macosx10.10.0", outputPath, Diags);
4971 TheDriver.CCCIsCXX =
true;
4976 for (vector<const char *>::iterator i = args.begin(); i != args.end(); ++i)
4981 OwningPtr<clang::driver::Compilation> C(TheDriver.BuildCompilation(args));
4986 SmallVector<std::pair<int, const clang::driver::Command *>, 4> FailingCommands;
4988 Res = TheDriver.ExecuteCompilation(*C, FailingCommands);
4991 for (
auto i : argsToFree)
4995 remove(compositeModulePath.c_str());
5001 chmod(outputPath.c_str(), 0755);
5005 __block vector<string> thirdPartyNodeClasses;
5006 dispatch_sync(environmentQueue, ^{
5007 for (
size_t i = 1; i < environments.size(); ++i)
5009 map<string, VuoCompilerNodeClass *> envNodeClasses = environments[i].at(0)->
getNodeClasses();
5010 for (map<string, VuoCompilerNodeClass *>::iterator j = envNodeClasses.begin(); j != envNodeClasses.end(); ++j)
5011 thirdPartyNodeClasses.push_back(j->first);
5015 string details =
"One or more nodes in this composition can't be used by this version of Vuo. ";
5016 if (! thirdPartyNodeClasses.empty())
5018 details +=
"Make sure you're using the latest version of all the extra Vuo nodes you've installed:\n";
5019 sort(thirdPartyNodeClasses.begin(), thirdPartyNodeClasses.end());
5020 for (vector<string>::iterator i = thirdPartyNodeClasses.begin(); i != thirdPartyNodeClasses.end(); ++i)
5021 details +=
" • " + *i +
"\n";
5023 details +=
"Check the macOS Console for more information about the problem.";
5026 "Node broken or outdated", details);
5037 Module * VuoCompiler::readModuleFromC(
string inputPath,
const vector<string> &headerSearchPaths,
const vector<string> &extraArgs)
5041 vector<const char *> args;
5042 args.push_back(inputPath.c_str());
5043 args.push_back(
"-DVUO_COMPILER");
5044 args.push_back(
"-fblocks");
5047 args.push_back(
"-Wall");
5048 args.push_back(
"-Wextra");
5049 args.push_back(
"-Wimplicit-fallthrough");
5050 args.push_back(
"-Wno-unused-parameter");
5051 args.push_back(
"-Wno-c++11-extensions");
5052 args.push_back(
"-Wno-sign-compare");
5053 args.push_back(
"-Werror=implicit");
5057 args.push_back(
"-std=c++11");
5058 args.push_back(
"-stdlib=libc++");
5061 for (vector<string>::const_iterator i = headerSearchPaths.begin(); i != headerSearchPaths.end(); ++i)
5063 args.push_back(
"-I");
5064 args.push_back(i->c_str());
5068 args.push_back(
"-v");
5070 for (vector<string>::const_iterator i = extraArgs.begin(); i != extraArgs.end(); ++i)
5071 args.push_back(i->c_str());
5073 clang::DiagnosticOptions * diagOptions =
new clang::DiagnosticOptions();
5074 IntrusiveRefCntPtr<clang::DiagnosticIDs> DiagID(
new clang::DiagnosticIDs());
5075 clang::DiagnosticsEngine Diags(DiagID, diagOptions);
5077 OwningPtr<clang::CompilerInvocation> CI(
new clang::CompilerInvocation);
5078 clang::CompilerInvocation::CreateFromArgs(*CI, &args[0], &args[0] + args.size(), Diags);
5080 clang::CompilerInstance Clang;
5081 Clang.setInvocation(CI.take());
5083 Clang.createDiagnostics();
5084 if (!Clang.hasDiagnostics())
5088 llvm::sys::Path builtinHeaderSearchPath;
5089 string vuoFrameworkPath = getVuoFrameworkPath();
5090 if (vuoFrameworkPath.empty())
5092 llvm::sys::Path clangPath = getClangPath();
5093 builtinHeaderSearchPath = clangPath;
5094 builtinHeaderSearchPath.eraseComponent();
5095 builtinHeaderSearchPath.eraseComponent();
5096 builtinHeaderSearchPath.appendComponent(
"lib");
5097 builtinHeaderSearchPath.appendComponent(
"clang");
5098 builtinHeaderSearchPath.appendComponent(CLANG_VERSION_STRING);
5102 builtinHeaderSearchPath = vuoFrameworkPath;
5103 builtinHeaderSearchPath.appendComponent(
"Frameworks");
5104 builtinHeaderSearchPath.appendComponent(
"llvm.framework");
5105 builtinHeaderSearchPath.appendComponent(
"Versions");
5106 builtinHeaderSearchPath.appendComponent(
"A");
5107 builtinHeaderSearchPath.appendComponent(
"lib");
5108 builtinHeaderSearchPath.appendComponent(
"clang");
5109 builtinHeaderSearchPath.appendComponent(CLANG_VERSION_STRING);
5111 Clang.getHeaderSearchOpts().ResourceDir = builtinHeaderSearchPath.str();
5114 clang::CodeGenAction *Act =
new clang::EmitLLVMOnlyAction();
5115 if (!Clang.ExecuteAction(*Act))
5118 return Act->takeModule();
5126 Module * VuoCompiler::readModuleFromBitcode(
string inputPath)
5128 string dir, file, ext;
5131 return readModuleFromBitcode(&inputFile);
5143 size_t inputDataBytes;
5148 Module *module = readModuleFromBitcodeData(inputData, inputDataBytes, error);
5163 Module * VuoCompiler::readModuleFromBitcodeData(
char *inputData,
size_t inputDataBytes,
string &error)
5165 __block Module *module;
5167 StringRef inputDataAsStringRef(inputData, inputDataBytes);
5168 MemoryBuffer *mb = MemoryBuffer::getMemBuffer(inputDataAsStringRef,
"",
false);
5169 module = ParseBitcodeFile(&(*mb), getGlobalContext(), &error);
5182 bool VuoCompiler::writeModuleToBitcode(Module *module,
string outputPath)
5184 if (verifyModule(*module, PrintMessageAction))
5186 VUserLog(
"Error: Module verification failed.");
5191 raw_fd_ostream out(outputPath.c_str(), err);
5194 VUserLog(
"Error: Couldn't open file '%s' for writing: %s", outputPath.c_str(), err.c_str());
5197 WriteBitcodeToFile(module, out);
5207 void VuoCompiler::setTargetForModule(Module *module,
string target)
5234 module->setTargetTriple(
"x86_64-apple-macosx10.10.0");
5245 Module *llvmModule = module->
getModule();
5274 delete baseNodeClass;
5307 if (nodeClassForNode)
5308 return nodeClassForNode->
newNode(title, x, y);
5322 if (nodeClassForNode)
5323 return nodeClassForNode->
newNode(nodeToCopyMetadataFrom);
5334 return createPublishedNode(nodeClassName, publishedInputPorts);
5343 return createPublishedNode(nodeClassName, publishedOutputPorts);
5349 VuoNode * VuoCompiler::createPublishedNode(
const string &nodeClassName,
const vector<VuoPublishedPort *> &publishedPorts)
5357 for (
size_t i = 0; i < publishedPorts.size(); ++i)
5359 VuoType *publishedPortType =
static_cast<VuoCompilerPort *
>(publishedPorts[i]->getCompiler())->getDataVuoType();
5363 set<VuoPort *> nodePorts;
5367 nodePorts.insert(inputPort);
5370 nodePorts.insert(outputPort);
5375 nodePorts.insert(inputPort);
5378 for (
VuoPort *port : nodePorts)
5385 reifyGenericPortTypes(node);
5396 dispatch_sync(environmentQueue, ^{
5397 if (environments.size() >= 5)
5398 environments.at(3).at(0)->addExpatriateSourceFile(sourcePath);
5408 dispatch_sync(environmentQueue, ^{
5409 if (environments.size() >= 5)
5411 environments.at(3).at(0)->removeExpatriateSourceFile(sourcePath);
5413 set<string> sourcesRemoved;
5414 sourcesRemoved.insert(getModuleKeyForPath(sourcePath));
5415 loadModulesAndSources(set<string>(), set<string>(), set<string>(), set<string>(), set<string>(), sourcesRemoved,
5416 false, false, environments.at(3).at(0), nullptr, nullptr,
"");
5439 string sourcePathCopy = sourcePath;
5440 string sourceCodeCopy = sourceCode;
5442 dispatch_async(environmentQueue, ^{
5444 ModuleInfo *sourceInfo = NULL;
5446 for (
const vector<Environment *> &envs : environments)
5448 for (Environment *env : envs)
5450 ModuleInfo *potentialSourceInfo = env->listSourceFile(nodeClassName);
5453 sourceInfo = potentialSourceInfo;
5462 bool shouldRecompileSourcesIfUnchanged;
5463 if (! sourceCodeCopy.empty())
5465 sourceInfo->setSourceCode(sourceCodeCopy);
5466 sourceInfo->setSourceCodeOverridden(
true);
5468 shouldRecompileSourcesIfUnchanged =
false;
5472 sourceInfo->revertSourceCode();
5473 sourceInfo->setSourceCodeOverridden(
false);
5475 shouldRecompileSourcesIfUnchanged =
true;
5477 sourceInfo->setAttempted(
false);
5478 sourceInfo->setLastModifiedToNow();
5480 set<string> sourcesModified;
5481 sourcesModified.insert(nodeClassName);
5483 loadModulesAndSources(set<string>(), set<string>(), set<string>(), set<string>(), sourcesModified, set<string>(),
5484 false, shouldRecompileSourcesIfUnchanged,
nullptr,
nullptr,
nullptr,
"");
5523 set<string> nodeClassNameSet;
5524 nodeClassNameSet.insert(nodeClassName);
5525 loadModulesIfNeeded(nodeClassNameSet);
5530 void (^envGetNodeClass)(Environment *) = ^
void (Environment *env) {
5535 applyToAllEnvironments(envGetNodeClass);
5546 loadModulesIfNeeded();
5548 __block map<string, VuoCompilerNodeClass *> nodeClasses;
5549 void (^envGetNodeClasses)(Environment *) = ^
void (Environment *env) {
5550 map<string, VuoCompilerNodeClass *> result = env->getNodeClasses();
5551 nodeClasses.insert(result.begin(), result.end());
5553 applyToInstalledEnvironments(envGetNodeClasses);
5564 set<string> typeNameSet;
5565 typeNameSet.insert(typeName);
5566 loadModulesIfNeeded(typeNameSet);
5569 void (^envGetType)(Environment *) = ^
void (Environment *env) {
5574 applyToInstalledEnvironments(envGetType);
5596 loadModulesIfNeeded();
5598 __block map<string, VuoCompilerType *> types;
5599 void (^envGetTypes)(Environment *) = ^
void (Environment *env) {
5600 map<string, VuoCompilerType *> result = env->getTypes();
5601 types.insert(result.begin(), result.end());
5603 applyToInstalledEnvironments(envGetTypes);
5614 loadModulesIfNeeded();
5616 __block map<string, VuoCompilerModule *> libraryModules;
5617 void (^envGetLibraryModules)(Environment *) = ^
void (Environment *env) {
5618 map<string, VuoCompilerModule *> result = env->getLibraryModules();
5619 libraryModules.insert(result.begin(), result.end());
5621 applyToInstalledEnvironments(envGetLibraryModules);
5622 return libraryModules;
5632 loadModulesIfNeeded();
5634 __block map<string, VuoNodeSet *> nodeSets;
5635 void (^envGetNodeSets)(Environment *) = ^
void (Environment *env) {
5636 map<string, VuoNodeSet *> result = env->getNodeSets();
5637 nodeSets.insert(result.begin(), result.end());
5639 applyToInstalledEnvironments(envGetNodeSets);
5650 loadModulesIfNeeded();
5653 void (^envFindNodeSet)(Environment *) = ^
void (Environment *env) {
5658 applyToInstalledEnvironments(envFindNodeSet);
5668 void (^envFindModule)(Environment *) = ^
void (Environment *env) {
5673 applyToAllEnvironments(envFindModule);
5690 map<string, VuoCompilerNodeClass *> nodeClasses =
getNodeClasses();
5691 for (map<string, VuoCompilerNodeClass *>::const_iterator i = nodeClasses.begin(); i != nodeClasses.end(); ++i)
5698 else if (format ==
"path")
5702 else if (format ==
"dot")
5720 vector<string> VuoCompiler::getCoreVuoDependencies(
void)
5722 vector<string> dependencies;
5724 dependencies.push_back(
"VuoHeap");
5725 dependencies.push_back(
"VuoApp");
5727 dependencies.push_back(
"zmq");
5728 dependencies.push_back(
"json-c");
5729 dependencies.push_back(
"objc");
5730 dependencies.push_back(
"c");
5731 dependencies.push_back(
"AppKit.framework");
5734 dependencies.push_back(LLVM_ROOT
"/lib/libprofile_rt.dylib");
5736 return dependencies;
5742 string VuoCompiler::getRuntimeMainDependency(
void)
5744 return "VuoRuntimeMain.o";
5754 string VuoCompiler::getRuntimeDependency(
void)
5756 return "VuoRuntime.o";
5765 string VuoCompiler::getVuoFrameworkPath(
void)
5767 if (! vuoFrameworkInProgressPath.empty())
5768 return vuoFrameworkInProgressPath;
5776 llvm::sys::Path VuoCompiler::getClangPath(
void)
5787 string dir, moduleKey, ext;
5795 while (nodeClassNameParts.size() > 1 && nodeClassNameParts.back() == ext)
5796 nodeClassNameParts.pop_back();
5799 for (
string &part : nodeClassNameParts)
5803 if (nodeClassNameParts.size() == 1)
5804 nodeClassNameParts.insert(nodeClassNameParts.begin(),
"isf");
5818 __block
bool isLocal =
false;
5820 dispatch_sync(environmentQueue, ^{
5821 if (environments.size() >= 5)
5822 isLocal = environments.at(3).at(0)->findModule(moduleKey);
5836 return lastCompositionBaseDir.empty() ?
"" : lastCompositionBaseDir +
"/Modules";
5851 return lastCompositionBaseDir;
5857 void VuoCompiler::addModuleSearchPath(
string path)
5859 dispatch_sync(environmentQueue, ^{
5860 environments.back().at(0)->addModuleSearchPath(path);
5870 dispatch_sync(environmentQueue, ^{
5880 dispatch_sync(environmentQueue, ^{
5890 dispatch_sync(environmentQueue, ^{
5900 this->telemetry = telemetry;
5908 this->target = target;
5916 this->isVerbose = isVerbose;
5926 if (VuoPro::getProAccess())
5928 _shouldShowSplashWindow =
false;
5933 _shouldShowSplashWindow = potentiallyShow;
5941 return _shouldShowSplashWindow;
5947 void VuoCompiler::setClangPath(
const string &clangPath)
5949 this->clangPath = llvm::sys::Path(StringRef(clangPath));
5957 string vuoFrameworkPath = getVuoFrameworkPath();
5958 return (vuoFrameworkPath.empty() ?
5959 VUO_BUILD_DIR
"/bin/VuoCompositionLoader.app/Contents/MacOS/VuoCompositionLoader" :
5960 vuoFrameworkPath +
"/Helpers/VuoCompositionLoader.app/Contents/MacOS/VuoCompositionLoader");
5966 string VuoCompiler::getCompositionStubPath(
void)
5968 string vuoFrameworkPath = getVuoFrameworkPath();
5969 return (vuoFrameworkPath.empty() ?
5970 VUO_BUILD_DIR
"/lib/libVuoCompositionStub.dylib" :
5971 vuoFrameworkPath +
"/Modules/libVuoCompositionStub.dylib");
5977 string VuoCompiler::getCachePathForComposition(
const string compositionDir)
5979 string modifierLetterColon(
"꞉");
5980 string cachedModulesName = compositionDir;
5990 __block vector<string> moduleSearchPaths;
5991 void (^envGetModuleSearchPaths)(Environment *) = ^
void (Environment *env) {
5992 vector<string> result = env->getModuleSearchPaths();
5993 moduleSearchPaths.insert(moduleSearchPaths.end(), result.begin(), result.end());
5995 applyToInstalledEnvironments(envGetModuleSearchPaths);
5997 __block vector<string> headerSearchPaths;
5998 void (^envGetHeaderSearchPaths)(Environment *) = ^
void (Environment *env) {
5999 vector<string> result = env->getHeaderSearchPaths();
6000 headerSearchPaths.insert(headerSearchPaths.end(), result.begin(), result.end());
6002 applyToInstalledEnvironments(envGetHeaderSearchPaths);
6004 __block vector<string> librarySearchPaths;
6005 void (^envGetLibrarySearchPaths)(Environment *) = ^
void (Environment *env) {
6006 vector<string> result = env->getLibrarySearchPaths();
6007 librarySearchPaths.insert(librarySearchPaths.end(), result.begin(), result.end());
6009 applyToInstalledEnvironments(envGetLibrarySearchPaths);
6011 __block vector<string> frameworkSearchPaths;
6012 void (^envGetFrameworkSearchPaths)(Environment *) = ^
void (Environment *env) {
6013 vector<string> result = env->getFrameworkSearchPaths();
6014 frameworkSearchPaths.insert(frameworkSearchPaths.end(), result.begin(), result.end());
6016 applyToInstalledEnvironments(envGetFrameworkSearchPaths);
6018 fprintf(stderr,
"Module (node class, type, library) search paths:\n");
6019 for (vector<string>::iterator i = moduleSearchPaths.begin(); i != moduleSearchPaths.end(); ++i)
6020 fprintf(stderr,
" %s\n", (*i).c_str());
6021 fprintf(stderr,
"Header search paths:\n");
6022 for (vector<string>::iterator i = headerSearchPaths.begin(); i != headerSearchPaths.end(); ++i)
6023 fprintf(stderr,
" %s\n", (*i).c_str());
6024 fprintf(stderr,
"Other library search paths:\n");
6025 for (vector<string>::iterator i = librarySearchPaths.begin(); i != librarySearchPaths.end(); ++i)
6026 fprintf(stderr,
" %s\n", (*i).c_str());
6027 fprintf(stderr,
"Other framework search paths:\n");
6028 for (vector<string>::iterator i = frameworkSearchPaths.begin(); i != frameworkSearchPaths.end(); ++i)
6029 fprintf(stderr,
" %s\n", (*i).c_str());
6030 fprintf(stderr,
"Framework path:\n");
6031 if (! getVuoFrameworkPath().empty())
6032 fprintf(stderr,
" %s\n", getVuoFrameworkPath().c_str());
6033 fprintf(stderr,
"Clang path:\n");
6034 if (! getClangPath().str().empty())
6035 fprintf(stderr,
" %s\n", getClangPath().c_str());
6051 string directory, file, extension;
6055 compiler.
compileComposition(compositionFilePath, compiledCompositionPath,
true, issues);
6057 remove(compiledCompositionPath.c_str());
6086 string compositionPath = (workingDirectory.empty() ?
"." : workingDirectory) +
"/" + processName +
".vuo";
6092 remove(compiledCompositionPath.c_str());
6116 string directory, file, extension;
6119 compiler.
compileComposition(compositionFilePath, compiledCompositionPath,
true, issues);
6122 remove(compiledCompositionPath.c_str());
6151 string compositionPath = (workingDirectory.empty() ?
"." : workingDirectory) +
"/UntitledComposition.vuo";
6157 remove(compiledCompositionPath.c_str());
6170 pendingDataQueue = dispatch_queue_create(
"org.vuo.compiler.delegate.pending", 0);
6175 LoadedModulesData *data = dequeueData();
6182 void VuoCompilerDelegate::enqueueData(LoadedModulesData *data)
6184 dispatch_sync(pendingDataQueue, ^{
6185 pendingData.push_back(data);
6192 VuoCompilerDelegate::LoadedModulesData * VuoCompilerDelegate::dequeueData(
void)
6194 __block LoadedModulesData *ret;
6195 dispatch_sync(pendingDataQueue, ^{
6196 ret = pendingData.front();
6197 pendingData.pop_front();
6206 VuoCompilerDelegate::LoadedModulesData::LoadedModulesData(
const set< pair<VuoCompilerModule *, VuoCompilerModule *> > &modulesModified,
6209 referenceCountQueue = dispatch_queue_create(
"org.vuo.compiler.delegate.reference", 0);
6212 this->modulesModified = modulesModified;
6213 this->modulesRemoved = modulesRemoved;
6214 this->issues = issues;
6220 VuoCompilerDelegate::LoadedModulesData::~LoadedModulesData(
void)
6224 for (set< pair<VuoCompilerModule *, VuoCompilerModule *> >::iterator i = modulesModified.begin(); i != modulesModified.end(); ++i)
6227 for (set<VuoCompilerModule *>::iterator i = modulesRemoved.begin(); i != modulesRemoved.end(); ++i)
6234 void VuoCompilerDelegate::LoadedModulesData::retain(
void)
6236 dispatch_sync(referenceCountQueue, ^{
6244 void VuoCompilerDelegate::LoadedModulesData::release(
void)
6246 dispatch_sync(referenceCountQueue, ^{
6248 if (referenceCount == 0) {