11 #include <sys/param.h>
15 #include <CoreFoundation/CoreFoundation.h>
49 map<string, VuoFileUtilities::File *> VuoCompiler::Environment::moduleCacheFileForLocking;
50 dispatch_queue_t VuoCompiler::Environment::moduleCacheBuildingQueue = dispatch_queue_create(
"org.vuo.compiler.cache", NULL);
51 const string VuoCompiler::Environment::pidCacheDirPrefix =
"pid-";
52 set<VuoCompiler *> VuoCompiler::allCompilers;
53 dispatch_queue_t VuoCompiler::environmentQueue = dispatch_queue_create(
"org.vuo.compiler.environment", NULL);
54 vector< vector<VuoCompiler::Environment *> > VuoCompiler::sharedEnvironments;
55 map<string, vector<VuoCompiler::Environment *> > VuoCompiler::environmentsForCompositionFamily;
56 dispatch_group_t VuoCompiler::moduleSourceCompilersExist = dispatch_group_create();
57 string VuoCompiler::vuoFrameworkInProgressPath;
64 static void __attribute__((constructor)) VuoCompiler_init(
void)
67 struct rlimit rl{OPEN_MAX, OPEN_MAX};
68 getrlimit(RLIMIT_NOFILE, &rl);
69 rl.rlim_cur =
MIN(OPEN_MAX, rl.rlim_max);
70 if (setrlimit(RLIMIT_NOFILE, &rl))
71 VUserLog(
"Warning: Couldn't set open-files limit: %s", strerror(errno));
74 llvmQueue = dispatch_queue_create(
"org.vuo.compiler.llvm", NULL);
77 llvm::InitializeNativeTarget();
90 Module module(
"", getGlobalContext());
109 VuoCompiler::ModuleInfo::ModuleInfo(Environment *environment,
const string &searchPath,
const string &relativePath,
110 bool isSourceFile,
bool isSubcomposition)
113 initialize(environment, searchPath, file, isSourceFile, isSubcomposition);
119 VuoCompiler::ModuleInfo::ModuleInfo(Environment *environment,
const string &searchPath,
VuoFileUtilities::File *file,
120 bool isSourceFile,
bool isSubcomposition)
122 initialize(environment, searchPath, file, isSourceFile, isSubcomposition);
128 void VuoCompiler::ModuleInfo::initialize(Environment *environment,
const string &searchPath,
VuoFileUtilities::File *file,
129 bool isSourceFile,
bool isSubcomposition)
131 this->environment = environment;
132 this->searchPath = searchPath;
135 sourceCodeOverridden =
false;
137 longestDownstreamPath = 0;
147 loading = dispatch_group_create();
150 if (isSubcomposition)
164 VuoCompiler::ModuleInfo::~ModuleInfo(
void)
173 dispatch_release(loading);
176 VuoCompiler::Environment * VuoCompiler::ModuleInfo::getEnvironment(
void)
const
181 string VuoCompiler::ModuleInfo::getSearchPath(
void)
const
186 string VuoCompiler::ModuleInfo::getModuleKey(
void)
const
196 string VuoCompiler::ModuleInfo::getSourceCode(
void)
const
201 void VuoCompiler::ModuleInfo::setSourceCode(
const string &sourceCode)
203 this->sourceCode = sourceCode;
206 void VuoCompiler::ModuleInfo::revertSourceCode(
void)
211 bool VuoCompiler::ModuleInfo::isSourceCodeOverridden(
void)
const
213 return sourceCodeOverridden;
216 void VuoCompiler::ModuleInfo::setSourceCodeOverridden(
bool overridden)
218 sourceCodeOverridden = overridden;
221 bool VuoCompiler::ModuleInfo::isNewerThan(ModuleInfo *other)
const
223 return lastModified > other->lastModified;
226 bool VuoCompiler::ModuleInfo::isNewerThan(
unsigned long ageInSeconds)
const
228 return lastModified > ageInSeconds;
231 bool VuoCompiler::ModuleInfo::isOlderThan(
unsigned long ageInSeconds)
const
233 return lastModified < ageInSeconds;
236 void VuoCompiler::ModuleInfo::setLastModifiedToNow(
void)
239 gettimeofday(&t, NULL);
240 lastModified = t.tv_sec;
243 set<string> VuoCompiler::ModuleInfo::getContainedNodeClasses(
void)
const
245 return containedNodeClasses;
248 int VuoCompiler::ModuleInfo::getLongestDownstreamPath(
void)
const
250 return longestDownstreamPath;
253 void VuoCompiler::ModuleInfo::setLongestDownstreamPath(
int pathLength)
255 longestDownstreamPath = pathLength;
258 bool VuoCompiler::ModuleInfo::getAttempted(
void)
const
263 void VuoCompiler::ModuleInfo::setAttempted(
bool attempted)
265 this->attempted = attempted;
268 dispatch_group_t VuoCompiler::ModuleInfo::getLoadingGroup(
void)
const
273 void VuoCompiler::ModuleInfo::dump()
const
275 fprintf(stderr,
"module %s:\n"
276 "\tloadingGroup=%p\n"
281 "\tsourceCodeOverridden=%d\n"
283 "\tcontainedNodeClasses:\n",
288 environment->getCompiledModuleCachePath().c_str(),
292 sourceCodeOverridden,
294 for (
auto i: containedNodeClasses)
295 fprintf(stderr,
"\t\t%s\n", i.c_str());
298 VuoCompiler::DependencyGraphVertex * VuoCompiler::DependencyGraphVertex::vertexForDependency(
const string &dependency,
VuoDirectedAcyclicGraph *graph)
302 return dynamic_cast<DependencyGraphVertex *
>(v);
304 DependencyGraphVertex *vv =
new DependencyGraphVertex();
305 vv->dependency = dependency;
306 vv->environment = NULL;
307 vv->compatible =
true;
311 string VuoCompiler::DependencyGraphVertex::getDependency(
void)
316 VuoCompiler::Environment * VuoCompiler::DependencyGraphVertex::getEnvironment(
void)
321 void VuoCompiler::DependencyGraphVertex::setEnvironment(Environment *environment)
323 this->environment = environment;
326 bool VuoCompiler::DependencyGraphVertex::isCompatible(
void)
331 void VuoCompiler::DependencyGraphVertex::setCompatible(
bool compatible)
333 this->compatible = compatible;
336 string VuoCompiler::DependencyGraphVertex::key(
void)
346 VuoCompiler::ModuleInfoIterator::ModuleInfoIterator(map<
string, map<string, ModuleInfo *> > *allModuleInfos)
348 this->allModuleInfos = allModuleInfos;
349 hasSearchModuleKeys =
false;
358 VuoCompiler::ModuleInfoIterator::ModuleInfoIterator(map<
string, map<string, ModuleInfo *> > *allModuleInfos,
const set<string> &searchModuleKeys)
360 this->allModuleInfos = allModuleInfos;
361 this->searchModuleKeys = searchModuleKeys;
362 hasSearchModuleKeys =
true;
371 void VuoCompiler::ModuleInfoIterator::initialize(
void)
373 currSearchPath = allModuleInfos->begin();
374 hasCurrModuleKey =
false;
382 VuoCompiler::ModuleInfo * VuoCompiler::ModuleInfoIterator::next(
void)
384 for ( ; currSearchPath != allModuleInfos->end(); ++currSearchPath)
386 if (! hasCurrModuleKey)
388 currModuleKey = currSearchPath->second.begin();
389 hasCurrModuleKey =
true;
392 for ( ; currModuleKey != currSearchPath->second.end(); ++currModuleKey)
394 if (! hasSearchModuleKeys || searchModuleKeys.find(currModuleKey->first) != searchModuleKeys.end())
396 ModuleInfo *moduleInfo = currModuleKey->second;
402 hasCurrModuleKey =
false;
411 VuoCompiler::Environment::Environment(
void)
413 compilersToNotifyQueue = dispatch_queue_create(
"org.vuo.compiler.notify", 0);
414 moduleSearchPathContentsChangedQueue = dispatch_queue_create(
"org.vuo.compiler.watch", 0);
415 isModuleCacheableDataDirty =
false;
416 isModuleCacheInitialized =
false;
417 isModuleCacheAvailable =
false;
426 VuoCompiler::Environment::~Environment(
void)
428 stopWatchingModuleSearchPaths();
429 dispatch_sync(moduleSearchPathContentsChangedQueue, ^{});
431 dispatch_release(moduleSearchPathContentsChangedQueue);
432 dispatch_release(compilersToNotifyQueue);
434 for (map<string, VuoCompilerNodeClass *>::iterator i = nodeClasses.begin(); i != nodeClasses.end(); ++i)
435 destroyModule(i->second);
437 for (map<string, VuoCompilerType *>::iterator i = types.begin(); i != types.end(); ++i)
438 destroyModule(i->second);
440 for (map<string, VuoCompilerModule *>::iterator i = libraryModules.begin(); i != libraryModules.end(); ++i)
441 destroyModule(i->second);
443 for (set<VuoCompilerGenericType *>::iterator i = genericTypes.begin(); i != genericTypes.end(); ++i)
445 VuoType *base = (*i)->getBase();
450 for (map<string, VuoNodeSet *>::iterator i = nodeSetForName.begin(); i != nodeSetForName.end(); ++i)
453 for (map<
string, map<string, ModuleInfo *> >::iterator i = moduleFilesAtSearchPath.begin(); i != moduleFilesAtSearchPath.end(); ++i)
454 for (map<string, ModuleInfo *>::iterator j = i->second.begin(); j != i->second.end(); ++j)
457 for (map<
string, map<string, ModuleInfo *> >::iterator i = sourceFilesAtSearchPath.begin(); i != sourceFilesAtSearchPath.end(); ++i)
458 for (map<string, ModuleInfo *>::iterator j = i->second.begin(); j != i->second.end(); ++j)
461 delete dependencyGraph;
462 delete compositionDependencyGraph;
468 void VuoCompiler::Environment::addCompilerToNotify(
VuoCompiler *compiler)
470 dispatch_sync(compilersToNotifyQueue, ^{
471 compilersToNotify.insert(compiler);
478 void VuoCompiler::Environment::removeCompilerToNotify(
VuoCompiler *compiler)
480 dispatch_sync(compilersToNotifyQueue, ^{
481 compilersToNotify.erase(compiler);
490 map<string, VuoCompilerNodeClass *> VuoCompiler::Environment::getNodeClasses(
void)
502 map<string, VuoCompilerNodeClass *>::iterator nodeClassIter = nodeClasses.find(moduleKey);
503 if (nodeClassIter != nodeClasses.end())
504 return nodeClassIter->second;
517 map<string, VuoCompilerNodeClass *>::iterator nodeClassIter = nodeClasses.find(moduleKey);
518 if (nodeClassIter != nodeClasses.end())
520 nodeClass = nodeClassIter->second;
522 nodeClasses.erase(nodeClassIter);
523 removeFromDependencyGraph(nodeClass);
535 map<string, VuoCompilerType *> VuoCompiler::Environment::getTypes(
void)
545 VuoCompilerType * VuoCompiler::Environment::getType(
const string &moduleKey)
547 map<string, VuoCompilerType *>::iterator typeIter = types.find(moduleKey);
548 if (typeIter != types.end())
549 return typeIter->second;
559 map<string, VuoNodeSet *> VuoCompiler::Environment::getNodeSets(
void)
561 return nodeSetForName;
569 map<string, VuoCompilerModule *> VuoCompiler::Environment::getLibraryModules(
void)
571 return libraryModules;
580 VuoCompilerModule * VuoCompiler::Environment::findModule(
const string &moduleKey)
582 map<string, VuoCompilerNodeClass *>::iterator nodeClassIter = nodeClasses.find(moduleKey);
583 if (nodeClassIter != nodeClasses.end())
584 return nodeClassIter->second;
586 map<string, VuoCompilerType *>::iterator typeIter = types.find(moduleKey);
587 if (typeIter != types.end())
588 return typeIter->second;
590 map<string, VuoCompilerModule *>::iterator libraryIter = libraryModules.find(moduleKey);
591 if (libraryIter != libraryModules.end())
592 return libraryIter->second;
602 VuoNodeSet * VuoCompiler::Environment::findNodeSet(
const string &name)
605 map<string, VuoNodeSet *>::iterator nodeSetIter = nodeSetForName.find(name);
606 if (nodeSetIter != nodeSetForName.end())
607 return nodeSetIter->second;
618 void VuoCompiler::Environment::addModuleSearchPath(
const string &path,
bool shouldWatch)
620 moduleSearchPaths.push_back(path);
622 updateModulesAtSearchPath(path);
623 updateSourceFilesAtSearchPath(path);
626 startWatchingModuleSearchPath(path);
634 vector<string> VuoCompiler::Environment::getModuleSearchPaths(
void)
636 return moduleSearchPaths;
644 void VuoCompiler::Environment::addHeaderSearchPath(
const string &path)
646 headerSearchPaths.push_back(path);
654 vector<string> VuoCompiler::Environment::getHeaderSearchPaths(
void)
656 return headerSearchPaths;
664 void VuoCompiler::Environment::addLibrarySearchPath(
const string &path)
666 librarySearchPaths.push_back(path);
674 vector<string> VuoCompiler::Environment::getLibrarySearchPaths(
void)
676 return librarySearchPaths;
684 void VuoCompiler::Environment::addFrameworkSearchPath(
const string &path)
686 frameworkSearchPaths.push_back(path);
694 vector<string> VuoCompiler::Environment::getFrameworkSearchPaths(
void)
696 return frameworkSearchPaths;
701 return dependencyGraph;
706 return compositionDependencyGraph;
714 void VuoCompiler::Environment::setModuleCachePath(
const string &path)
716 moduleCachePath = path;
724 string VuoCompiler::Environment::getCompiledModuleCachePath(
void)
726 return (moduleCachePath.empty() ?
"" : moduleCachePath +
"/Modules");
734 string VuoCompiler::Environment::getOverriddenCompiledModuleCachePath(
void)
736 if (moduleCachePath.empty())
742 string dir, moduleCacheDirName, ext;
755 void VuoCompiler::Environment::addExpatriateSourceFile(
const string &sourcePath)
757 expatriateSourceFiles.push_back(sourcePath);
759 if (find(moduleSearchPaths.begin(), moduleSearchPaths.end(),
"") == moduleSearchPaths.end())
760 moduleSearchPaths.push_back(
"");
762 auto iter = sourceFilesAtSearchPath.find(
"");
763 if (iter != sourceFilesAtSearchPath.end())
764 sourceFilesAtSearchPath.erase(iter);
772 void VuoCompiler::Environment::removeExpatriateSourceFile(
const string &sourcePath)
774 for (
auto i = expatriateSourceFiles.begin(); i != expatriateSourceFiles.end(); ++i)
778 expatriateSourceFiles.erase(i);
780 auto iter = sourceFilesAtSearchPath.find(
"");
781 if (iter != sourceFilesAtSearchPath.end())
782 sourceFilesAtSearchPath.erase(iter);
801 void VuoCompiler::Environment::updateModulesAtSearchPath(
const string &path)
803 if (moduleFilesAtSearchPath.find(path) != moduleFilesAtSearchPath.end())
806 set<string> moduleExtensions;
807 moduleExtensions.insert(
"vuonode");
808 moduleExtensions.insert(
"vuonode+");
809 moduleExtensions.insert(
"bc");
810 moduleExtensions.insert(
"bc+");
811 set<string> archiveExtensions;
812 archiveExtensions.insert(
"vuonode");
815 map<string, ModuleInfo *> fileForModuleKey;
816 for (set<VuoFileUtilities::File *>::iterator i = moduleFiles.begin(); i != moduleFiles.end(); ++i)
820 ModuleInfo *m =
new ModuleInfo(
this, path, moduleFile,
false,
false);
821 fileForModuleKey[m->getModuleKey()] = m;
824 if (path == getCompiledModuleCachePath())
826 for (map<string, ModuleInfo *>::iterator i = fileForModuleKey.begin(); i != fileForModuleKey.end(); )
828 ModuleInfo *sourceInfo = listSourceFile(i->first);
829 if (! sourceInfo || sourceInfo->isNewerThan(i->second))
831 ModuleInfo *m = i->second;
834 fileForModuleKey.erase(i++);
841 moduleFilesAtSearchPath[path] = fileForModuleKey;
849 void VuoCompiler::Environment::updateModuleAtSearchPath(
const string &moduleSearchPath,
const string &moduleRelativePath)
851 string dir, file, ext;
854 set<string> moduleExtensions;
855 moduleExtensions.insert(ext);
856 set<string> archiveExtensions;
857 archiveExtensions.insert(
"vuonode");
861 for (set<VuoFileUtilities::File *>::iterator i = moduleFiles.begin(); i != moduleFiles.end(); ++i)
865 ModuleInfo *m =
new ModuleInfo(
this, moduleSearchPath, moduleFile,
false,
false);
866 moduleFilesAtSearchPath[moduleSearchPath][m->getModuleKey()] = m;
877 void VuoCompiler::Environment::updateSourceFilesAtSearchPath(
const string &path)
879 map<string, map<string, ModuleInfo *> >::iterator sourceFilesIter = sourceFilesAtSearchPath.find(path);
880 if (sourceFilesIter != sourceFilesAtSearchPath.end())
883 set<VuoFileUtilities::File *> sourceFiles;
886 set<string> sourceExtensions;
887 sourceExtensions.insert(
"vuo");
888 sourceExtensions.insert(
"fs");
890 sourceExtensions.insert(cext.begin(), cext.end());
895 for (
const string &sourcePath : expatriateSourceFiles)
897 string dir, file, ext;
900 sourceFiles.insert(sourceFile);
904 map<string, ModuleInfo *> fileForModuleKey;
905 for (set<VuoFileUtilities::File *>::iterator i = sourceFiles.begin(); i != sourceFiles.end(); ++i)
909 string dir, moduleKey, ext;
911 bool isSubcomposition = (ext ==
"vuo");
914 if (path.empty() && !sourceFile->
exists())
917 ModuleInfo *m =
new ModuleInfo(
this, path, sourceFile,
true, isSubcomposition);
918 if (fileForModuleKey.find(m->getModuleKey()) != fileForModuleKey.end())
919 VUserLog(
"Warning: Conflicting source files for module %s are installed at %s", m->getModuleKey().c_str(), path.c_str());
920 fileForModuleKey[m->getModuleKey()] = m;
922 if (isSubcomposition)
924 DependencyGraphVertex *compositionVertex = DependencyGraphVertex::vertexForDependency(moduleKey, compositionDependencyGraph);
925 compositionDependencyGraph->addVertex(compositionVertex);
927 compositionVertex->setEnvironment(
this);
929 set<string> dependencies = m->getContainedNodeClasses();
930 for (set<string>::iterator j = dependencies.begin(); j != dependencies.end(); ++j)
932 DependencyGraphVertex *dependencyVertex = DependencyGraphVertex::vertexForDependency(*j, compositionDependencyGraph);
933 compositionDependencyGraph->addEdge(compositionVertex, dependencyVertex);
938 sourceFilesAtSearchPath[path] = fileForModuleKey;
946 VuoCompiler::ModuleInfo * VuoCompiler::Environment::listModule(
const string &moduleKey)
948 for (
const auto &moduleFiles : moduleFilesAtSearchPath)
950 map<string, ModuleInfo *>::const_iterator foundIter = moduleFiles.second.find(moduleKey);
951 if (foundIter != moduleFiles.second.end())
952 return foundIter->second;
963 VuoCompiler::ModuleInfoIterator VuoCompiler::Environment::listModules(
const set<string> &moduleKeys)
965 for (
const string &path : moduleSearchPaths)
966 updateModulesAtSearchPath(path);
968 return ModuleInfoIterator(&moduleFilesAtSearchPath, moduleKeys);
976 VuoCompiler::ModuleInfoIterator VuoCompiler::Environment::listAllModules(
void)
978 for (
const string &path : moduleSearchPaths)
979 updateModulesAtSearchPath(path);
981 return ModuleInfoIterator(&moduleFilesAtSearchPath);
989 VuoCompiler::ModuleInfo * VuoCompiler::Environment::listSourceFile(
const string &moduleKey)
991 for (
const auto &sourceFiles : sourceFilesAtSearchPath)
993 map<string, ModuleInfo *>::const_iterator foundIter = sourceFiles.second.find(moduleKey);
994 if (foundIter != sourceFiles.second.end())
995 return foundIter->second;
1006 VuoCompiler::ModuleInfoIterator VuoCompiler::Environment::listSourceFiles(
const set<string> &moduleKeys)
1008 for (
const string &path : moduleSearchPaths)
1009 updateSourceFilesAtSearchPath(path);
1011 return ModuleInfoIterator(&sourceFilesAtSearchPath, moduleKeys);
1019 VuoCompiler::ModuleInfoIterator VuoCompiler::Environment::listAllSourceFiles(
void)
1021 for (
const string &path : moduleSearchPaths)
1022 updateSourceFilesAtSearchPath(path);
1024 return ModuleInfoIterator(&sourceFilesAtSearchPath);
1030 vector<string> VuoCompiler::Environment::getBuiltInModuleSearchPaths(
void)
1032 vector<string> builtInModuleSearchPaths;
1034 string vuoFrameworkPath = getVuoFrameworkPath();
1035 if (! vuoFrameworkPath.empty())
1037 builtInModuleSearchPaths.push_back(vuoFrameworkPath +
"/Modules");
1041 builtInModuleSearchPaths.push_back(VUO_BUILD_DIR
"/library");
1042 builtInModuleSearchPaths.push_back(VUO_BUILD_DIR
"/node");
1043 builtInModuleSearchPaths.push_back(VUO_BUILD_DIR
"/type");
1044 builtInModuleSearchPaths.push_back(VUO_BUILD_DIR
"/type/list");
1047 return builtInModuleSearchPaths;
1053 vector<string> VuoCompiler::Environment::getBuiltInHeaderSearchPaths(
void)
1055 vector<string> builtInHeaderSearchPaths;
1057 string vuoFrameworkPath = getVuoFrameworkPath();
1058 if (! vuoFrameworkPath.empty())
1060 builtInHeaderSearchPaths.push_back(vuoFrameworkPath +
"/Headers");
1061 builtInHeaderSearchPaths.push_back(vuoFrameworkPath +
"/Headers/macos");
1062 builtInHeaderSearchPaths.push_back(vuoFrameworkPath +
"/Frameworks/llvm.framework/Versions/A/Headers/lib/c++/v1");
1066 builtInHeaderSearchPaths.push_back(VUO_SOURCE_DIR
"/library");
1067 builtInHeaderSearchPaths.push_back(VUO_SOURCE_DIR
"/node");
1068 builtInHeaderSearchPaths.push_back(VUO_SOURCE_DIR
"/type");
1069 builtInHeaderSearchPaths.push_back(VUO_SOURCE_DIR
"/type/list");
1070 builtInHeaderSearchPaths.push_back(VUO_SOURCE_DIR
"/runtime");
1071 builtInHeaderSearchPaths.push_back(JSONC_ROOT
"/include");
1072 builtInHeaderSearchPaths.push_back(LLVM_ROOT
"/include/c++/v1");
1075 return builtInHeaderSearchPaths;
1081 vector<string> VuoCompiler::Environment::getBuiltInLibrarySearchPaths(
void)
1083 vector<string> builtInLibrarySearchPaths;
1085 string vuoFrameworkPath = getVuoFrameworkPath();
1086 if (! vuoFrameworkPath.empty())
1088 builtInLibrarySearchPaths.push_back(vuoFrameworkPath +
"/Modules");
1091 builtInLibrarySearchPaths.push_back(OPENSSL_ROOT
"/lib");
1095 builtInLibrarySearchPaths.push_back(VUO_BUILD_DIR
"/library");
1096 builtInLibrarySearchPaths.push_back(VUO_BUILD_DIR
"/node");
1097 builtInLibrarySearchPaths.push_back(VUO_BUILD_DIR
"/type");
1098 builtInLibrarySearchPaths.push_back(VUO_BUILD_DIR
"/type/list");
1099 builtInLibrarySearchPaths.push_back(VUO_BUILD_DIR
"/runtime");
1101 builtInLibrarySearchPaths.push_back(LEAP_ROOT);
1104 builtInLibrarySearchPaths.insert(builtInLibrarySearchPaths.end(), conanLibDirs.begin(), conanLibDirs.end());
1107 return builtInLibrarySearchPaths;
1113 vector<string> VuoCompiler::Environment::getBuiltInFrameworkSearchPaths(
void)
1115 vector<string> builtInFrameworkSearchPaths;
1117 string vuoFrameworkPath = getVuoFrameworkPath();
1118 if (! vuoFrameworkPath.empty())
1120 builtInFrameworkSearchPaths.push_back(vuoFrameworkPath +
"/Modules/");
1121 builtInFrameworkSearchPaths.push_back(vuoFrameworkPath +
"/Frameworks/");
1125 builtInFrameworkSearchPaths.push_back(SYPHON_ROOT);
1128 return builtInFrameworkSearchPaths;
1137 void VuoCompiler::Environment::startWatchingModuleSearchPath(
const string &moduleSearchPath)
1140 moduleSearchPathWatchers.insert(watcher);
1148 void VuoCompiler::Environment::stopWatchingModuleSearchPaths(
void)
1150 for (set<VuoFileWatcher *>::iterator i = moduleSearchPathWatchers.begin(); i != moduleSearchPathWatchers.end(); ++i)
1153 moduleSearchPathWatchers.clear();
1160 void VuoCompiler::Environment::fileChanged(
const string &moduleSearchPath)
1162 dispatch_sync(moduleSearchPathContentsChangedQueue, ^{
1163 moduleSearchPathContentsChanged(moduleSearchPath);
1174 void VuoCompiler::Environment::moduleSearchPathContentsChanged(
const string &moduleSearchPath)
1179 __block set<string> modulesAdded;
1180 __block set<string> modulesModifed;
1181 __block set<string> modulesRemoved;
1182 __block set<string> compositionsAdded;
1183 __block set<string> compositionsModifed;
1184 __block set<string> compositionsRemoved;
1186 dispatch_sync(environmentQueue, ^{
1190 map<string, ModuleInfo *> oldModules;
1191 map<string, ModuleInfo *> oldCompositions;
1193 map<string, map<string, ModuleInfo *> >::iterator mf = moduleFilesAtSearchPath.find(moduleSearchPath);
1194 if (mf != moduleFilesAtSearchPath.end())
1196 for (map<string, ModuleInfo *>::iterator j = mf->second.begin(); j != mf->second.end(); ++j)
1198 oldModules[j->first] = j->second;
1201 moduleFilesAtSearchPath.erase(mf);
1204 map<string, map<string, ModuleInfo *> >::iterator cf = sourceFilesAtSearchPath.find(moduleSearchPath);
1205 if (cf != sourceFilesAtSearchPath.end())
1207 for (map<string, ModuleInfo *>::iterator j = cf->second.begin(); j != cf->second.end(); ++j)
1209 oldCompositions[j->first] = j->second;
1211 VuoDirectedAcyclicGraph::Vertex *vertex = compositionDependencyGraph->findVertex(j->first);
1212 compositionDependencyGraph->removeVertex(vertex);
1215 sourceFilesAtSearchPath.erase(cf);
1220 updateModulesAtSearchPath(moduleSearchPath);
1221 updateSourceFilesAtSearchPath(moduleSearchPath);
1225 mf = moduleFilesAtSearchPath.find(moduleSearchPath);
1226 if (mf != moduleFilesAtSearchPath.end())
1228 for (map<string, ModuleInfo *>::iterator n = mf->second.begin(); n != mf->second.end(); ++n)
1230 string moduleKey = n->first;
1232 map<string, ModuleInfo *>::iterator o = oldModules.find(moduleKey);
1233 if (o != oldModules.end())
1235 if (n->second->isNewerThan(o->second))
1237 modulesModifed.insert(moduleKey);
1241 n->second->setAttempted(o->second->getAttempted());
1245 oldModules.erase(o);
1249 modulesAdded.insert(moduleKey);
1254 cf = sourceFilesAtSearchPath.find(moduleSearchPath);
1255 if (cf != sourceFilesAtSearchPath.end())
1257 for (map<string, ModuleInfo *>::iterator n = cf->second.begin(); n != cf->second.end(); ++n)
1259 string moduleKey = n->first;
1261 map<string, ModuleInfo *>::iterator o = oldCompositions.find(moduleKey);
1262 if (o != oldCompositions.end())
1264 if (n->second->isNewerThan(o->second))
1266 compositionsModifed.insert(moduleKey);
1270 n->second->setAttempted(o->second->getAttempted());
1271 n->second->setSourceCode(o->second->getSourceCode());
1275 oldCompositions.erase(o);
1279 compositionsAdded.insert(moduleKey);
1284 for (map<string, ModuleInfo *>::iterator o = oldModules.begin(); o != oldModules.end(); ++o)
1287 modulesRemoved.insert(o->first);
1290 for (map<string, ModuleInfo *>::iterator o = oldCompositions.begin(); o != oldCompositions.end(); ++o)
1293 compositionsRemoved.insert(o->first);
1296 compilerForLoading->loadModulesAndSources(modulesAdded, modulesModifed, modulesRemoved,
1297 compositionsAdded, compositionsModifed, compositionsRemoved,
1298 false,
false,
this,
nullptr,
nullptr,
"");
1301 delete compilerForLoading;
1310 void VuoCompiler::Environment::moduleFileChanged(
const string &modulePath,
const string &moduleSourceCode,
1311 std::function<
void(
void)> moduleLoadedCallback,
1315 dispatch_sync(environmentQueue, ^{
1317 string moduleDir, moduleKey, ext;
1323 bool foundOldModule =
false;
1324 auto moduleSearchPathIter = moduleFilesAtSearchPath.find(moduleDir);
1325 if (moduleSearchPathIter != moduleFilesAtSearchPath.end())
1327 auto moduleIter = moduleSearchPathIter->second.find(moduleKey);
1328 if (moduleIter != moduleSearchPathIter->second.end())
1330 delete moduleIter->second;
1331 moduleSearchPathIter->second.erase(moduleIter);
1332 foundOldModule =
true;
1338 updateModuleAtSearchPath(moduleDir, moduleKey +
"." + ext);
1342 bool foundNewModule =
false;
1343 moduleSearchPathIter = moduleFilesAtSearchPath.find(moduleDir);
1344 if (moduleSearchPathIter != moduleFilesAtSearchPath.end())
1346 auto moduleIter = moduleSearchPathIter->second.find(moduleKey);
1347 if (moduleIter != moduleSearchPathIter->second.end())
1349 foundNewModule =
true;
1353 set<string> modulesAdded;
1354 set<string> modulesModified;
1355 set<string> modulesRemoved;
1359 modulesModified.insert(moduleKey);
1361 else if (! foundOldModule && foundNewModule)
1363 modulesAdded.insert(moduleKey);
1365 else if (foundOldModule && ! foundNewModule)
1367 modulesRemoved.insert(moduleKey);
1370 compiler->loadModulesAndSources(modulesAdded, modulesModified, modulesRemoved,
1371 set<string>(), set<string>(), set<string>(),
1372 false,
false,
this, issues, moduleLoadedCallback, moduleSourceCode);
1380 void VuoCompiler::Environment::notifyCompilers(
const set<VuoCompilerModule *> &modulesAdded,
1381 const set< pair<VuoCompilerModule *, VuoCompilerModule *> > &modulesModified,
1383 bool oldModulesInvalidated)
1385 if (! (modulesAdded.empty() && modulesModified.empty() && modulesRemoved.empty() && issues->
isEmpty()) )
1387 set< pair<VuoCompilerModule *, VuoCompilerModule *> > invalidatedModulesModified;
1388 set<VuoCompilerModule *> invalidatedModulesRemoved;
1389 if (oldModulesInvalidated)
1391 invalidatedModulesModified = modulesModified;
1392 invalidatedModulesRemoved = modulesRemoved;
1395 VuoCompilerDelegate::LoadedModulesData *delegateData =
new VuoCompilerDelegate::LoadedModulesData(invalidatedModulesModified, invalidatedModulesRemoved, issues);
1397 map<string, VuoCompilerModule *> modulesAddedMap;
1398 map<string, pair<VuoCompilerModule *, VuoCompilerModule *> > modulesModifiedMap;
1399 map<string, VuoCompilerModule *> modulesRemovedMap;
1401 modulesAddedMap[m->getPseudoBase()->getModuleKey()] = m;
1402 for (pair<VuoCompilerModule *, VuoCompilerModule *> m : modulesModified)
1403 modulesModifiedMap[m.first->getPseudoBase()->getModuleKey()] = m;
1405 modulesRemovedMap[m->getPseudoBase()->getModuleKey()] = m;
1407 dispatch_sync(compilersToNotifyQueue, ^{
1408 for (set<VuoCompiler *>::iterator i = compilersToNotify.begin(); i != compilersToNotify.end(); ++i) {
1409 delegateData->retain();
1411 for (set<VuoCompiler *>::iterator i = compilersToNotify.begin(); i != compilersToNotify.end(); ++i) {
1412 (*i)->loadedModules(modulesAddedMap, modulesModifiedMap, modulesRemovedMap, issues, delegateData, this);
1432 set<VuoCompilerModule *> VuoCompiler::Environment::loadCompiledModules(
const set<string> &moduleKeys,
const map<string, string> &sourceCodeForModule)
1434 ModuleInfoIterator modulesToLoadIter = listModules(moduleKeys);
1435 set<VuoCompilerModule *> modulesLoaded;
1437 ModuleInfo *moduleInfo;
1438 while ((moduleInfo = modulesToLoadIter.next()))
1440 string moduleKey = moduleInfo->getModuleKey();
1444 if (moduleInfo->getEnvironment() !=
this || findModule(moduleKey) || moduleInfo->getAttempted())
1447 moduleInfo->setAttempted(
true);
1454 modulesLoaded.insert(module);
1455 addToDependencyGraph(module);
1459 string searchPath = moduleInfo->getSearchPath();
1466 ModuleInfo *sourceInfo = listSourceFile(moduleKey);
1470 auto sourceCodeIter = sourceCodeForModule.find(moduleKey);
1471 if (sourceCodeIter != sourceCodeForModule.end())
1478 return modulesLoaded;
1492 set<dispatch_group_t> VuoCompiler::Environment::loadSpecializedModules(
const set<string> &moduleKeys,
1495 set<dispatch_group_t > loadingGroups;
1497 for (
string moduleKey : moduleKeys)
1506 if (findModule(moduleKey))
1511 map<string, dispatch_group_t>::iterator iter = specializedModulesLoading.find(moduleKey);
1512 if (iter != specializedModulesLoading.end())
1514 loadingGroups.insert(iter->second);
1515 dispatch_retain(iter->second);
1519 dispatch_group_t loadingGroup = dispatch_group_create();
1520 specializedModulesLoading[moduleKey] = loadingGroup;
1521 loadingGroups.insert(loadingGroup);
1527 dispatch_queue_t queue = dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0);
1528 dispatch_group_async(loadingGroup, queue, ^{
1542 compiler->loadNodeClassGeneratedAtRuntime(specNodeClass,
this);
1545 compiler->loadModulesIfNeeded(dependencies);
1548 dispatch_sync(environmentQueue, ^{
1549 specializedModulesLoading.erase(moduleKey);
1554 return loadingGroups;
1569 set<dispatch_group_t> VuoCompiler::Environment::compileModulesFromSourceCode(
const set<string> &moduleKeys,
bool shouldRecompileIfUnchanged)
1571 ModuleInfoIterator modulesToLoadIter = listSourceFiles(moduleKeys);
1573 int environmentIndex = sharedEnvironments.size();
1574 for (
int i = 0; i < sharedEnvironments.size(); ++i)
1575 if (
this == sharedEnvironments.at(i).at(0))
1576 environmentIndex = i;
1578 set<dispatch_group_t> sourcesLoading;
1579 int sourcesEnqueued = 0;
1580 ModuleInfo *sourceInfo;
1581 while ((sourceInfo = modulesToLoadIter.next()))
1583 string moduleKey = sourceInfo->getModuleKey();
1585 dispatch_group_t loadingGroup = sourceInfo->getLoadingGroup();
1586 sourcesLoading.insert(loadingGroup);
1591 if (sourceInfo->getAttempted())
1596 string sourceCode = sourceInfo->getSourceCode();
1598 if (! shouldRecompileIfUnchanged)
1601 if (existingNodeClass && existingNodeClass->
getSourceCode() == sourceCode)
1607 sourceInfo->setAttempted(
true);
1609 dispatch_group_enter(loadingGroup);
1610 dispatch_group_enter(moduleSourceCompilersExist);
1614 queueItem->
sourcePath = sourceInfo->getFile()->path();
1616 queueItem->
sourceFile = sourceInfo->getFile();
1617 queueItem->
cachedModulesPath = sourceInfo->isSourceCodeOverridden() ? getOverriddenCompiledModuleCachePath() : getCompiledModuleCachePath();
1620 queueItem->
priority = { environmentIndex, sourceInfo->getLongestDownstreamPath() };
1621 moduleCompilationQueue->enqueue(queueItem);
1630 dispatch_queue_t queue = dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0);
1631 dispatch_async(queue, ^{
1632 for (
int i = 0; i < sourcesEnqueued; ++i)
1635 VUserLog(
"Compiling %s", queueItem->moduleKey.c_str());
1637 dispatch_async(queue, ^{
1646 notifyCompilers(set<VuoCompilerModule *>(), set< pair<VuoCompilerModule *, VuoCompilerModule *> >(), set<VuoCompilerModule *>(), issues,
false);
1652 auto moduleLoadedCallback = [=](){
1653 dispatch_group_leave(queueItemForCallback->
loadingGroup);
1654 moduleCompilationQueue->completed(queueItemForCallback);
1657 string ext = queueItem->sourceFile->extension();
1668 notifyCompilers(set<VuoCompilerModule *>(), set< pair<VuoCompilerModule *, VuoCompilerModule *> >(), set<VuoCompilerModule *>(), issues,
false);
1675 auto getType = [&compiler] (
const string &moduleKey) {
return compiler->
getType(moduleKey); };
1683 writeModuleToBitcode(module, queueItem->compiledModulePath);
1691 moduleFileChanged(queueItem->compiledModulePath, queueItem->sourceCode, moduleLoadedCallback, compiler, issues);
1694 moduleLoadedCallback();
1700 compiler->
compileModule(queueItem->sourcePath, queueItem->compiledModulePath, vector<string>());
1701 moduleFileChanged(queueItem->compiledModulePath, queueItem->sourceCode, moduleLoadedCallback, compiler,
nullptr);
1708 notifyCompilers(set<VuoCompilerModule *>(), set< pair<VuoCompilerModule *, VuoCompilerModule *> >(), set<VuoCompilerModule *>(), issues,
false);
1709 moduleLoadedCallback();
1715 compiler->compileSubcompositionString(queueItem->sourceCode, queueItem->compiledModulePath, moduleLoadedCallback,
this, NULL, queueItem->sourcePath);
1719 dispatch_group_leave(moduleSourceCompilersExist);
1724 return sourcesLoading;
1735 set<VuoCompilerModule *> VuoCompiler::Environment::unloadCompiledModules(
const set<string> &moduleKeys)
1737 set<VuoCompilerModule *> modulesUnloaded;
1739 for (set<string>::const_iterator i = moduleKeys.begin(); i != moduleKeys.end(); ++i)
1741 string moduleKey = *i;
1744 map<string, VuoCompilerNodeClass *>::iterator nodeClassIter = nodeClasses.find(moduleKey);
1745 if (nodeClassIter != nodeClasses.end())
1747 module = nodeClassIter->second;
1748 nodeClasses.erase(nodeClassIter);
1752 map<string, VuoCompilerType *>::iterator typeIter = types.find(moduleKey);
1753 if (typeIter != types.end())
1755 module = typeIter->second;
1756 types.erase(typeIter);
1760 map<string, VuoCompilerModule *>::iterator libraryModuleIter = libraryModules.find(moduleKey);
1761 if (libraryModuleIter != libraryModules.end())
1763 module = libraryModuleIter->second;
1764 libraryModules.erase(libraryModuleIter);
1771 modulesUnloaded.insert(module);
1772 removeFromDependencyGraph(module);
1777 return modulesUnloaded;
1785 void VuoCompiler::Environment::deleteModulesCompiledFromSourceCode(
const set<string> &moduleKeys)
1787 for (set<string>::const_iterator i = moduleKeys.begin(); i != moduleKeys.end(); ++i)
1789 string moduleKey = *i;
1791 map<string, ModuleInfo *>::iterator iter = moduleFilesAtSearchPath[getCompiledModuleCachePath()].find(moduleKey);
1792 if (iter != moduleFilesAtSearchPath[getCompiledModuleCachePath()].end())
1804 VuoCompilerModule * VuoCompiler::Environment::loadModule(ModuleInfo *moduleInfo)
1806 string moduleKey = moduleInfo->getModuleKey();
1813 __block
size_t inputDataBytes;
1814 __block
char *rawInputData;
1818 rawInputData = moduleInfo->getFile()->getContentsAsRawData(inputDataBytes);
1822 rawInputData = NULL;
1823 VUserLog(
"Warning: Couldn't load module '%s'. Its file may have been deleted. (%s)", moduleKey.c_str(), e.
what());
1829 char *processedInputData;
1831 processedInputData = loadModule_Pro0(moduleInfo, moduleKey, inputDataBytes, rawInputData);
1833 processedInputData = rawInputData;
1836 Module *module = NULL;
1837 bool moduleParseError = !processedInputData;
1838 if (!moduleParseError)
1840 string moduleReadError;
1842 module = readModuleFromBitcodeData(processedInputData, inputDataBytes, moduleReadError);
1844 free(processedInputData);
1848 moduleParseError =
true;
1850 string dir, file, ext;
1851 VuoFileUtilities::splitPath(moduleInfo->getFile()->isInArchive() ? moduleInfo->getFile()->getRelativePath() : moduleInfo->getFile()->path(), dir, file, ext);
1853 if (dir == getCompiledModuleCachePath())
1856 VUserLog(
"Error: Couldn't parse module '%s': %s.", moduleInfo->getFile()->getRelativePath().c_str(), moduleReadError.c_str());
1860 if (!moduleParseError)
1863 if (! moduleInfo->getFile()->isInArchive())
1864 modulePath = moduleInfo->getFile()->path();
1878 libraryModules[moduleKey] = compilerModule;
1883 map<string, VuoNodeSet *>::iterator nodeSetIter = nodeSetForName.find(nodeSet->
getName());
1884 if (nodeSetIter == nodeSetForName.end())
1886 nodeSetForName[nodeSet->
getName()] = nodeSet;
1891 nodeSet = nodeSetIter->second;
1893 compilerModule->getPseudoBase()->setNodeSet(nodeSet);
1897 loadModule_Pro1(rawInputData, processedInputData, compilerModule);
1900 compilerModule->setBuiltIn( isBuiltIn() );
1902 return compilerModule;
1906 destroyLlvmModule(module);
1925 removeFromDependencyGraph(oldNodeClass);
1926 destroyModule(oldNodeClass);
1929 nodeClasses[moduleKey] = newNodeClass;
1930 addToDependencyGraph(newNodeClass);
1936 DependencyGraphVertex *vertex = DependencyGraphVertex::vertexForDependency(module->
getPseudoBase()->
getModuleKey(), dependencyGraph);
1937 dependencyGraph->addVertex(vertex);
1939 vertex->setEnvironment(
this);
1946 for (set<string>::iterator i = dependencies.begin(); i != dependencies.end(); ++i)
1948 DependencyGraphVertex *depVertex = DependencyGraphVertex::vertexForDependency(*i, dependencyGraph);
1949 dependencyGraph->addEdge(vertex, depVertex);
1953 void VuoCompiler::Environment::removeFromDependencyGraph(
VuoCompilerModule *module)
1955 DependencyGraphVertex *vertex = DependencyGraphVertex::vertexForDependency(module->
getPseudoBase()->
getModuleKey(), dependencyGraph);
1956 dependencyGraph->removeVertex(vertex);
1969 void VuoCompiler::Environment::reifyPortTypes(
const map<string, VuoCompilerType *> &inheritedTypes)
1971 map<string, VuoCompilerType *> availableTypes = inheritedTypes;
1972 availableTypes.insert(types.begin(), types.end());
1974 for (map<string, VuoCompilerNodeClass *>::const_iterator i = nodeClasses.begin(); i != nodeClasses.end(); ++i)
1980 vector<VuoPortClass *> portClasses;
1981 portClasses.insert(portClasses.end(), inputPortClasses.begin(), inputPortClasses.end());
1982 portClasses.insert(portClasses.end(), outputPortClasses.begin(), outputPortClasses.end());
1984 for (vector<VuoPortClass *>::iterator j = portClasses.begin(); j != portClasses.end(); ++j)
2004 map<string, VuoCompilerType *>::iterator reifiedTypeIter = availableTypes.find(typeName);
2005 if (reifiedTypeIter != availableTypes.end())
2008 reifiedType = reifiedTypeIter->second;
2019 for (vector<VuoCompilerTriggerDescription *>::iterator j = triggers.begin(); j != triggers.end(); ++j)
2027 map<string, VuoCompilerType *>::iterator reifiedTypeIter = availableTypes.find(typeName);
2028 if (reifiedTypeIter != availableTypes.end())
2051 void VuoCompiler::Environment::getCacheableModulesAndDependencies(
bool builtIn,
bool installed, set<string> &cacheableModulesAndDependencies,
2052 set<string> &dylibsNeededToLinkToThisCache,
2053 set<string> &frameworksNeededToLinkToThisCache)
2055 if (isModuleCacheInitialized && ! isModuleCacheableDataDirty)
2057 cacheableModulesAndDependencies = moduleCacheContents;
2058 dylibsNeededToLinkToThisCache = moduleCacheDylibs;
2059 frameworksNeededToLinkToThisCache = moduleCacheFrameworks;
2067 map<string, VuoCompilerModule *> allModules;
2068 allModules.insert(nodeClasses.begin(), nodeClasses.end());
2069 allModules.insert(types.begin(), types.end());
2070 allModules.insert(libraryModules.begin(), libraryModules.end());
2072 set<string> dependencies;
2073 for (map<string, VuoCompilerModule *>::iterator i = allModules.begin(); i != allModules.end(); ++i)
2075 string moduleKey = i->first;
2080 if (module->requiresPro())
2088 cacheableModulesAndDependencies.insert(moduleKey);
2091 dependencies.insert(moduleDependencies.begin(), moduleDependencies.end());
2095 if (builtIn && installed)
2097 vector<string> coreDependencies = getCoreVuoDependencies();
2098 dependencies.insert(coreDependencies.begin(), coreDependencies.end());
2103 for (set<string>::iterator i = dependencies.begin(); i != dependencies.end(); ++i)
2105 string dependency = *i;
2106 if (allModules.find(dependency) == allModules.end())
2109 frameworksNeededToLinkToThisCache.insert(dependency);
2112 string dependencyPath = VuoCompiler::getLibraryPath(dependency, librarySearchPaths);
2113 if (! dependencyPath.empty())
2116 dylibsNeededToLinkToThisCache.insert(dependencyPath);
2118 cacheableModulesAndDependencies.insert(dependency);
2124 moduleCacheSuffix = (installed ?
"installed" :
"generated");
2125 dylibsNeededToLinkToThisCache.insert(moduleCachePath +
"/libVuoModuleCache-" + moduleCacheSuffix +
".dylib");
2127 moduleCacheDylibs = dylibsNeededToLinkToThisCache;
2128 moduleCacheFrameworks = frameworksNeededToLinkToThisCache;
2150 void VuoCompiler::Environment::useModuleCache(
bool shouldUseExistingCache,
VuoCompiler *compiler, set<string> cacheableModulesAndDependencies,
2151 set<string> dylibsNeededToLinkToCaches, set<string> frameworksNeededToLinkToCaches)
2155 static dispatch_once_t checked = 0;
2156 static bool prelinkCache =
true;
2157 dispatch_once(&checked, ^{
2159 bool result = CFPreferencesGetAppBooleanValue(CFSTR(
"prelinkCache"), CFSTR(
"org.vuo.Editor"), &valid);
2161 prelinkCache = result;
2165 VDebugLog(
"Ignoring the module cache since the 'prelinkCache' preference is false.");
2171 if (moduleCachePath.empty())
2176 string cacheDescription = string() +
"the cache of " + moduleCacheSuffix +
" modules at '" + moduleCachePath +
"'";
2177 if (isModuleCacheInitialized && ! isModuleCacheableDataDirty)
2179 VDebugLog(
"No need to recheck %s.", cacheDescription.c_str());
2185 VDebugLog(
"Checking if %s is up-to-date…", cacheDescription.c_str());
2187 bool isCacheUpToDate =
true;
2189 if (isModuleCacheInitialized && isModuleCacheableDataDirty)
2190 isCacheUpToDate =
false;
2192 isModuleCacheInitialized =
true;
2193 isModuleCacheableDataDirty =
false;
2195 const string dylibFileName =
"libVuoModuleCache-" + moduleCacheSuffix +
".dylib";
2196 const string indexFileName =
"moduleCache-" + moduleCacheSuffix +
".txt";
2197 string dylibPath = moduleCachePath +
"/" + dylibFileName;
2198 string indexPath = moduleCachePath +
"/" + indexFileName;
2202 bool dylibFileExists =
false;
2203 bool indexFileExists =
false;
2212 if (! (dirExists && dylibFileExists && indexFileExists))
2214 if (shouldUseExistingCache)
2215 throw VuoException(
"Trying to use the existing cache, but the cache doesn't exist.",
false);
2220 if (! indexFileExists)
2222 if (! dylibFileExists)
2225 isCacheUpToDate =
false;
2233 fileForLocking = moduleCacheFileForLocking[dylibPath];
2234 if (! fileForLocking)
2237 moduleCacheFileForLocking[dylibPath] = fileForLocking;
2241 VDebugLog(
"\tWarning: Couldn't lock for reading.");
2246 if (isCacheUpToDate)
2251 if (shouldUseExistingCache)
2252 throw VuoException(
"Trying to use the existing cache, but the cache doesn't contain readable data.",
false);
2254 isCacheUpToDate =
false;
2260 const char separator =
'\n';
2261 if (isCacheUpToDate || shouldUseExistingCache)
2264 string index = indexFile.getContentsAsString();
2267 moduleCacheContents.clear();
2268 moduleCacheContents.insert(actualIndex.begin(), actualIndex.end());
2270 if (shouldUseExistingCache)
2272 isModuleCacheAvailable =
true;
2279 if (isCacheUpToDate)
2281 if (moduleCacheContents.size() != cacheableModulesAndDependencies.size())
2282 isCacheUpToDate =
false;
2285 set<string> contentsInBoth;
2286 std::set_intersection(moduleCacheContents.begin(), moduleCacheContents.end(),
2287 cacheableModulesAndDependencies.begin(), cacheableModulesAndDependencies.end(),
2288 std::inserter(contentsInBoth, contentsInBoth.begin()));
2290 if (contentsInBoth.size() != cacheableModulesAndDependencies.size())
2291 isCacheUpToDate =
false;
2297 if (isCacheUpToDate)
2301 for (map<
string, map<string, ModuleInfo *> >::iterator i = moduleFilesAtSearchPath.begin(); i != moduleFilesAtSearchPath.end(); ++i)
2303 for (map<string, ModuleInfo *>::iterator j = i->second.begin(); j != i->second.end(); ++j)
2305 if (! j->second->isOlderThan(cacheLastModified))
2307 isCacheUpToDate =
false;
2316 if (isCacheUpToDate)
2320 isModuleCacheAvailable =
true;
2326 dispatch_async(moduleCacheBuildingQueue, ^{
2327 VDebugLog(
"Rebuilding %s…", cacheDescription.c_str());
2329 set<Module *> modulesToLink;
2330 set<string> librariesToLink;
2331 set<string> frameworksToLink;
2333 compiler->getLinkerInputs(cacheableModulesAndDependencies, Optimization_SmallBinary, modulesToLink, librariesToLink, frameworksToLink);
2335 librariesToLink.insert(dylibsNeededToLinkToCaches.begin(), dylibsNeededToLinkToCaches.end());
2336 set<string>::iterator iter = librariesToLink.find(dylibPath);
2337 if (iter != librariesToLink.end())
2338 librariesToLink.erase(iter);
2340 frameworksToLink.insert(frameworksNeededToLinkToCaches.begin(), frameworksNeededToLinkToCaches.end());
2343 bool gotLockForWriting =
false;
2348 if (! gotLockForWriting)
2349 throw VuoException(
"The cache file is being used by another process. "
2350 "If any composition windows are open from previous Vuo sessions, quit them. "
2351 "If any processes whose names start with \"VuoComposition\" or one of your composition file names appear in Activity Monitor, force-quit them.",
2355 string dir, file, ext;
2358 compiler->link(tmpPath, modulesToLink, librariesToLink, frameworksToLink,
true);
2366 getVuoFrameworkPath() +
"/Helpers/install_name_tool",
2373 vector<string> expectedContents(cacheableModulesAndDependencies.begin(), cacheableModulesAndDependencies.end());
2379 VDebugLog(
"\tWarning: Couldn't downgrade the lock back to reading.");
2381 dispatch_sync(environmentQueue, ^{
2382 moduleCacheContents = cacheableModulesAndDependencies;
2383 isModuleCacheAvailable =
true;
2389 if (gotLockForWriting)
2391 VDebugLog(
"\tWarning: Couldn't downgrade the lock back to reading.");
2393 VUserLog(
"Warning: Couldn't rebuild %s for the \"faster build\" optimization: %s", cacheDescription.c_str(), e.
what());
2401 VUserLog(
"Warning: Couldn't use %s for the \"faster build\" optimization: %s", cacheDescription.c_str(), e.
what());
2408 void VuoCompiler::Environment::waitForModuleCachesToBuild(
void)
2410 dispatch_sync(moduleCacheBuildingQueue, ^{});
2422 bool VuoCompiler::Environment::findInModuleCache(
const string &moduleOrDependency,
string &cachePath)
2424 if (isModuleCacheAvailable && moduleCacheContents.find(moduleOrDependency) != moduleCacheContents.end())
2426 cachePath = moduleCachePath +
"/libVuoModuleCache-" + moduleCacheSuffix +
".dylib";
2439 void VuoCompiler::Environment::modulesChanged(
void)
2441 isModuleCacheableDataDirty =
true;
2442 isModuleCacheAvailable =
false;
2448 bool VuoCompiler::Environment::isBuiltIn()
2450 return this == sharedEnvironments[0][0];
2456 string VuoCompiler::Environment::getName()
2460 else if (
this == sharedEnvironments[0][1])
2461 return "builtin (generated)";
2462 else if (
this == sharedEnvironments[1][0])
2464 else if (
this == sharedEnvironments[1][1])
2465 return "system (generated)";
2466 else if (
this == sharedEnvironments[2][0])
2468 else if (
this == sharedEnvironments[2][1])
2469 return "user (generated)";
2470 return "composition-local";
2476 void VuoCompiler::applyToInstalledEnvironments(
void (^doForEnvironment)(Environment *))
2478 dispatch_sync(environmentQueue, ^{
2479 for (vector< vector<Environment *> >::iterator i = environments.begin(); i != environments.end(); ++i) {
2480 doForEnvironment((*i)[0]);
2488 void VuoCompiler::applyToInstalledEnvironments(
void (^doForEnvironment)(Environment *,
bool *,
string),
bool *result,
string arg)
2490 dispatch_sync(environmentQueue, ^{
2491 for (vector< vector<Environment *> >::iterator i = environments.begin(); i != environments.end(); ++i) {
2492 doForEnvironment((*i)[0], result, arg);
2500 void VuoCompiler::applyToAllEnvironments(
void (^doForEnvironment)(Environment *environment))
2502 dispatch_sync(environmentQueue, ^{
2503 for (vector< vector<Environment *> >::iterator i = environments.begin(); i != environments.end(); ++i) {
2504 for (vector<Environment *>::iterator j = i->begin(); j != i->end(); ++j) {
2505 doForEnvironment(*j);
2526 shouldLoadAllModules =
true;
2527 hasLoadedAllModules =
false;
2528 modulesToLoadQueue = dispatch_queue_create(
"org.vuo.compiler.modules", NULL);
2529 moduleCacheBuilding = dispatch_group_create();
2530 dependencyGraph = NULL;
2531 compositionDependencyGraph = NULL;
2533 _shouldShowSplashWindow =
false;
2535 delegateQueue = dispatch_queue_create(
"org.vuo.compiler.delegate", NULL);
2537 string vuoFrameworkPath = getVuoFrameworkPath();
2538 if (! vuoFrameworkPath.empty())
2539 clangPath = vuoFrameworkPath +
"/Helpers/clang";
2541 clangPath = llvm::sys::Path(StringRef(LLVM_ROOT
"/bin/clang"));
2543 dispatch_sync(environmentQueue, ^{
2544 allCompilers.insert(
this);
2546 if (sharedEnvironments.empty())
2548 sharedEnvironments = vector< vector<Environment *> >(3, vector<Environment *>(2, NULL));
2549 for (vector< vector<Environment *> >::iterator i = sharedEnvironments.begin(); i != sharedEnvironments.end(); ++i) {
2550 for (vector<Environment *>::iterator j = i->begin(); j != i->end(); ++j) {
2551 *j = new Environment();
2555 vector<string> builtInModuleSearchPaths = Environment::getBuiltInModuleSearchPaths();
2556 for (vector<string>::iterator i = builtInModuleSearchPaths.begin(); i != builtInModuleSearchPaths.end(); ++i) {
2557 sharedEnvironments[0][0]->addModuleSearchPath(*i, false);
2560 vector<string> builtInHeaderSearchPaths = Environment::getBuiltInHeaderSearchPaths();
2561 for (vector<string>::iterator i = builtInHeaderSearchPaths.begin(); i != builtInHeaderSearchPaths.end(); ++i) {
2562 sharedEnvironments[0][0]->addHeaderSearchPath(*i);
2565 vector<string> builtInLibrarySearchPaths = Environment::getBuiltInLibrarySearchPaths();
2566 for (vector<string>::iterator i = builtInLibrarySearchPaths.begin(); i != builtInLibrarySearchPaths.end(); ++i) {
2567 sharedEnvironments[0][0]->addLibrarySearchPath(*i);
2570 vector<string> builtInFrameworkSearchPaths = Environment::getBuiltInFrameworkSearchPaths();
2571 for (vector<string>::iterator i = builtInFrameworkSearchPaths.begin(); i != builtInFrameworkSearchPaths.end(); ++i) {
2572 sharedEnvironments[0][0]->addFrameworkSearchPath(*i);
2586 string vuoFrameworkPath = getVuoFrameworkPath();
2587 if (! vuoFrameworkPath.empty())
2589 vector<string> moduleCachePaths(3);
2590 moduleCachePaths[0] = vuoFrameworkPath +
"/Modules/Builtin";
2591 moduleCachePaths[1] = VuoFileUtilities::getCachePath() +
"/System";
2592 moduleCachePaths[2] = VuoFileUtilities::getCachePath() +
"/User";
2594 for (size_t i = 0; i < moduleCachePaths.size(); ++i)
2596 string moduleCachePath = moduleCachePaths[i];
2598 sharedEnvironments[i][0]->setModuleCachePath(moduleCachePath);
2599 sharedEnvironments[i][0]->addModuleSearchPath(moduleCachePath +
"/Modules", false);
2601 sharedEnvironments[i][1]->setModuleCachePath(moduleCachePath);
2607 setCompositionPath(compositionPath);
2619 dispatch_sync(environmentQueue, ^{
2620 allCompilers.erase(
this);
2623 dispatch_group_wait(moduleCacheBuilding, DISPATCH_TIME_FOREVER);
2624 dispatch_release(moduleCacheBuilding);
2626 for (vector< vector<Environment *> >::iterator i = environments.begin(); i != environments.end(); ++i)
2627 (*i)[0]->removeCompilerToNotify(
this);
2629 dispatch_sync(delegateQueue, ^{});
2631 dispatch_release(modulesToLoadQueue);
2632 dispatch_release(delegateQueue);
2634 delete dependencyGraph;
2635 delete compositionDependencyGraph;
2641 void VuoCompiler::reset(
void)
2643 dispatch_group_wait(moduleSourceCompilersExist, DISPATCH_TIME_FOREVER);
2645 dispatch_sync(environmentQueue, ^{
2646 for (vector< vector<Environment *> >::iterator i = sharedEnvironments.begin(); i != sharedEnvironments.end(); ++i)
2648 (*i)[0]->stopWatchingModuleSearchPaths();
2649 dispatch_sync((*i)[0]->moduleSearchPathContentsChangedQueue, ^{});
2652 for (map<
string, vector<Environment *> >::iterator i = environmentsForCompositionFamily.begin(); i != environmentsForCompositionFamily.end(); ++i)
2654 (i->second)[0]->stopWatchingModuleSearchPaths();
2655 dispatch_sync((i->second)[0]->moduleSearchPathContentsChangedQueue, ^{});
2658 for (vector< vector<Environment *> >::iterator i = sharedEnvironments.begin(); i != sharedEnvironments.end(); ++i)
2659 for (vector<Environment *>::iterator j = i->begin(); j != i->end(); ++j)
2662 for (map<
string, vector<Environment *> >::iterator i = environmentsForCompositionFamily.begin(); i != environmentsForCompositionFamily.end(); ++i)
2663 for (vector<Environment *>::iterator j = i->second.begin(); j != i->second.end(); ++j)
2666 allCompilers.clear();
2667 sharedEnvironments.clear();
2668 environmentsForCompositionFamily.clear();
2679 dispatch_async(delegateQueue, ^{
2680 this->delegate = delegate;
2695 string compositionModulesDir;
2696 string compositionBaseDir;
2697 lastCompositionIsSubcomposition =
false;
2698 if (! compositionPath.empty())
2706 string compositionDir;
2709 lastCompositionIsSubcomposition = (compositionDir == compositionModulesDir);
2714 dispatch_sync(environmentQueue, ^{
2715 if (! environments.empty() && compositionBaseDir == lastCompositionBaseDir) {
2718 lastCompositionBaseDir = compositionBaseDir;
2722 Environment *oldCompositionFamilyInstalledEnvironment =
nullptr;
2723 vector<Environment *> compositionEnvironments;
2724 if (! environments.empty())
2726 for (vector< vector<Environment *> >::iterator i = environments.begin(); i != environments.end(); ++i) {
2727 (*i)[0]->removeCompilerToNotify(this);
2730 if (environments.size() >= 5) {
2731 oldCompositionFamilyInstalledEnvironment = environments[3][0];
2734 compositionEnvironments = environments.back();
2736 environments.clear();
2743 bool isCompositionInSharedEnvironment =
false;
2744 for (vector< vector<Environment *> >::iterator i = sharedEnvironments.begin(); i != sharedEnvironments.end(); ++i)
2746 environments.push_back(*i);
2748 vector<string> moduleSearchPaths = (*i)[0]->getModuleSearchPaths();
2749 for (vector<string>::iterator j = moduleSearchPaths.begin(); j != moduleSearchPaths.end(); ++j)
2751 string moduleSearchPath = *j;
2752 VuoFileUtilities::canonicalizePath(moduleSearchPath);
2753 if (moduleSearchPath == compositionModulesDir)
2755 isCompositionInSharedEnvironment = true;
2760 if (isCompositionInSharedEnvironment) {
2767 Environment *newCompositionFamilyInstalledEnvironment =
nullptr;
2768 if (! isCompositionInSharedEnvironment && ! compositionPath.empty())
2770 vector<Environment *> compositionFamilyEnvironments = environmentsForCompositionFamily[compositionBaseDir];
2771 if (compositionFamilyEnvironments.empty())
2773 compositionFamilyEnvironments = vector<Environment *>(2, NULL);
2774 compositionFamilyEnvironments[0] = new Environment();
2775 compositionFamilyEnvironments[1] = new Environment();
2776 environmentsForCompositionFamily[compositionBaseDir] = compositionFamilyEnvironments;
2780 compositionFamilyEnvironments[0]->addModuleSearchPath(compositionModulesDir);
2784 string moduleCachePath = getCachePathForComposition(compositionBaseDir);
2786 compositionFamilyEnvironments[0]->setModuleCachePath(moduleCachePath);
2787 compositionFamilyEnvironments[0]->addModuleSearchPath(moduleCachePath +
"/Modules", false);
2789 compositionFamilyEnvironments[1]->setModuleCachePath(moduleCachePath);
2791 environments.push_back(compositionFamilyEnvironments);
2793 newCompositionFamilyInstalledEnvironment = compositionFamilyEnvironments[0];
2798 if (compositionEnvironments.empty())
2800 compositionEnvironments = vector<Environment *>(2, NULL);
2801 compositionEnvironments[0] = new Environment();
2802 compositionEnvironments[1] = new Environment();
2804 environments.push_back(compositionEnvironments);
2806 for (vector< vector<Environment *> >::iterator i = environments.begin(); i != environments.end(); ++i) {
2807 (*i)[0]->addCompilerToNotify(this);
2810 delete dependencyGraph;
2811 delete compositionDependencyGraph;
2812 dependencyGraph = makeDependencyNetwork(environments, ^
VuoDirectedAcyclicGraph * (Environment *env) {
return env->getDependencyGraph(); });
2813 compositionDependencyGraph = makeDependencyNetwork(environments, ^
VuoDirectedAcyclicGraph * (Environment *env) {
return env->getCompositionDependencyGraph(); });
2818 if (oldCompositionFamilyInstalledEnvironment != newCompositionFamilyInstalledEnvironment)
2820 auto getModules = [] (Environment *env)
2822 map<string, VuoCompilerModule *> modules;
2825 for (
auto i : env->getNodeClasses()) {
2828 for (
auto i : env->getTypes()) {
2831 for (
auto i : env->getLibraryModules()) {
2838 map<string, VuoCompilerModule *> modulesAdded = getModules(newCompositionFamilyInstalledEnvironment);
2839 map<string, VuoCompilerModule *> modulesRemoved = getModules(oldCompositionFamilyInstalledEnvironment);
2841 map<string, pair<VuoCompilerModule *, VuoCompilerModule *> > modulesModified;
2842 for (map<string, VuoCompilerModule *>::iterator
add = modulesAdded.begin();
add != modulesAdded.end(); )
2844 map<string, VuoCompilerModule *>::iterator rem = modulesRemoved.find(
add->first);
2845 if (rem != modulesRemoved.end())
2847 modulesModified[
add->first] = make_pair(rem->second,
add->second);
2848 modulesAdded.erase(
add++);
2849 modulesRemoved.erase(rem);
2857 if (! (modulesAdded.empty() && modulesModified.empty() && modulesRemoved.empty()) )
2860 VuoCompilerDelegate::LoadedModulesData *delegateData =
new VuoCompilerDelegate::LoadedModulesData(set< pair<VuoCompilerModule *, VuoCompilerModule *> >(), set<VuoCompilerModule *>(), issues);
2861 delegateData->retain();
2863 Environment *scopeEnvironment = newCompositionFamilyInstalledEnvironment;
2864 if (! scopeEnvironment) {
2865 scopeEnvironment = compositionEnvironments.at(0);
2868 loadedModules(modulesAdded, modulesModified, modulesRemoved, issues, delegateData, scopeEnvironment);
2882 if (!graphForEnvironment)
2887 for (vector< vector<Environment *> >::const_iterator i = environments.begin(); i != environments.end(); ++i)
2891 network->
addEdge(graphForEnvironment(i->at(0)), graphForEnvironment(i->at(1)));
2895 for (vector< vector<Environment *> >::const_iterator ii = environments.begin(); ii != i; ++ii)
2897 network->
addEdge(graphForEnvironment(i->at(0)), graphForEnvironment(ii->at(0)));
2898 network->
addEdge(graphForEnvironment(i->at(1)), graphForEnvironment(ii->at(0)));
2925 void VuoCompiler::loadModulesIfNeeded(
const set<string> &moduleKeys)
2927 __block
bool willLoadAllModules =
false;
2928 if (moduleKeys.empty())
2930 dispatch_sync(modulesToLoadQueue, ^{
2931 if (shouldLoadAllModules && ! hasLoadedAllModules) {
2932 willLoadAllModules =
true;
2933 hasLoadedAllModules =
true;
2938 if (! willLoadAllModules && moduleKeys.empty())
2943 __block set<dispatch_group_t> sourcesLoading;
2944 dispatch_sync(environmentQueue, ^{
2945 sourcesLoading = loadModulesAndSources(moduleKeys, set<string>(), set<string>(),
2946 moduleKeys, set<string>(), set<string>(),
2947 willLoadAllModules,
false,
nullptr,
nullptr,
nullptr,
"");
2953 for (set<dispatch_group_t>::iterator i = sourcesLoading.begin(); i != sourcesLoading.end(); ++i)
2955 dispatch_group_wait(*i, DISPATCH_TIME_FOREVER);
2956 dispatch_release(*i);
2968 set<dispatch_group_t> VuoCompiler::loadModulesAndSources(
const set<string> &modulesAddedKeys,
const set<string> &modulesModifiedKeys,
const set<string> &modulesRemovedKeys,
2969 const set<string> &sourcesAddedKeys,
const set<string> &sourcesModifiedKeys,
const set<string> &sourcesRemovedKeys,
2970 bool willLoadAllModules,
bool shouldRecompileSourcesIfUnchanged,
2971 Environment *currentEnvironment,
VuoCompilerIssues *issuesForCurrentEnvironment,
2972 std::function<
void(
void)> moduleLoadedCallback,
const string &moduleAddedOrModifiedSourceCode)
2984 map<Environment *, set<string> > modulesAdded;
2985 map<Environment *, set<string> > modulesModified;
2986 map<Environment *, set<string> > modulesRemoved;
2987 map<Environment *, set<string> > sourcesAdded;
2988 map<Environment *, set<string> > sourcesModified;
2989 map<Environment *, set<string> > sourcesRemoved;
2990 map<Environment *, set<string> > potentialSpecializedModules;
2992 if (currentEnvironment)
2994 modulesAdded[currentEnvironment] = modulesAddedKeys;
2995 modulesModified[currentEnvironment] = modulesModifiedKeys;
2996 modulesRemoved[currentEnvironment] = modulesRemovedKeys;
2997 sourcesAdded[currentEnvironment] = sourcesAddedKeys;
2998 sourcesModified[currentEnvironment] = sourcesModifiedKeys;
2999 sourcesRemoved[currentEnvironment] = sourcesRemovedKeys;
3003 Environment *genEnv =
nullptr;
3004 if (! willLoadAllModules)
3010 int scope = environments.size() - (lastCompositionIsSubcomposition ? 2 : 1);
3011 genEnv = environments.at(scope).at(1);
3012 potentialSpecializedModules[genEnv] = modulesAddedKeys;
3015 for (vector< vector<Environment *> >::iterator i = environments.begin(); i != environments.end(); ++i)
3017 Environment *env = (*i).at(0);
3019 ModuleInfoIterator modulesAddedIter = (willLoadAllModules ? env->listAllModules() : env->listModules(modulesAddedKeys));
3020 ModuleInfoIterator sourcesAddedIter = (willLoadAllModules ? env->listAllSourceFiles() : env->listSourceFiles(sourcesAddedKeys));
3021 ModuleInfoIterator sourcesModifiedIter = (willLoadAllModules ? env->listAllSourceFiles() : env->listSourceFiles(sourcesModifiedKeys));
3023 ModuleInfo *moduleInfo;
3024 while ((moduleInfo = modulesAddedIter.next()))
3026 string moduleKey = moduleInfo->getModuleKey();
3028 modulesAdded[env].insert(moduleKey);
3030 if (! willLoadAllModules)
3032 auto foundIter = potentialSpecializedModules[genEnv].find(moduleKey);
3033 if (foundIter != potentialSpecializedModules[genEnv].end())
3034 potentialSpecializedModules[genEnv].erase(foundIter);
3040 auto isCompiledModuleAtSameSearchPath = [&env] (ModuleInfo *sourceInfo)
3042 ModuleInfo *compiledModuleInfo = env->listModule(sourceInfo->getModuleKey());
3043 return (compiledModuleInfo && compiledModuleInfo->getSearchPath() == sourceInfo->getSearchPath());
3046 while ((moduleInfo = sourcesAddedIter.next()))
3048 if (isCompiledModuleAtSameSearchPath(moduleInfo))
3051 sourcesAdded[env].insert( moduleInfo->getModuleKey() );
3054 while ((moduleInfo = sourcesModifiedIter.next()))
3056 if (isCompiledModuleAtSameSearchPath(moduleInfo))
3059 sourcesModified[env].insert( moduleInfo->getModuleKey() );
3064 map<Environment *, VuoCompilerIssues *> issues;
3065 for (vector< vector<Environment *> >::iterator i = environments.begin(); i != environments.end(); ++i)
3067 Environment *env = (*i).at(0);
3068 issues[env] = (env == currentEnvironment && issuesForCurrentEnvironment ? issuesForCurrentEnvironment :
new VuoCompilerIssues());
3073 for (vector< vector<Environment *> >::iterator i = environments.begin(); i != environments.end(); ++i)
3075 Environment *env = (*i).at(0);
3080 set<VuoDirectedAcyclicGraph::Vertex *> circularDependencies = env->getCompositionDependencyGraph()->getCycleVertices();
3082 set<string> sourcesAddedModified;
3083 sourcesAddedModified.insert(sourcesAdded[env].begin(), sourcesAdded[env].end());
3084 sourcesAddedModified.insert(sourcesModified[env].begin(), sourcesModified[env].end());
3086 for (set<string>::iterator j = sourcesAddedModified.begin(); j != sourcesAddedModified.end(); ++j)
3088 string moduleKey = *j;
3091 for (set<VuoDirectedAcyclicGraph::Vertex *>::iterator k = circularDependencies.begin(); k != circularDependencies.end(); ++k)
3093 DependencyGraphVertex *vertex =
static_cast<DependencyGraphVertex *
>(*k);
3094 if (vertex->getDependency() == moduleKey)
3103 sourcesAdded[env].erase(moduleKey);
3104 sourcesModified[env].erase(moduleKey);
3107 "Subcomposition contains itself",
3108 "%moduleKey contains an instance of itself, "
3109 "or contains another subcomposition that contains an instance of %moduleKey.");
3110 issue.setModuleKey(moduleKey);
3112 ModuleInfo *sourceInfo = env->listSourceFile(moduleKey);
3114 issue.setFilePath(sourceInfo->getFile()->path());
3116 issues[env]->append(issue);
3123 for (
const vector<Environment *> &envs : environments)
3125 Environment *env = envs.at(0);
3127 set<string> sourcesAddedModified;
3128 sourcesAddedModified.insert(sourcesAdded[env].begin(), sourcesAdded[env].end());
3129 sourcesAddedModified.insert(sourcesModified[env].begin(), sourcesModified[env].end());
3131 for (
const string &moduleKey : sourcesAddedModified)
3134 int pathLength = env->getCompositionDependencyGraph()->getLongestDownstreamPath(vertex);
3136 ModuleInfo *sourceInfo = env->listSourceFile(moduleKey);
3137 sourceInfo->setLongestDownstreamPath(pathLength);
3145 map<Environment *, set<string> > modulesDepOnModulesModified;
3146 map<Environment *, set<string> > sourcesDepOnModulesModified;
3147 map<Environment *, set<string> > modulesDepOnModulesRemoved;
3148 map<Environment *, set<string> > sourcesDepOnModulesRemoved;
3151 __block map<Environment *, set<string> > modulesDepOnModulesModified_otherCompiler;
3152 __block map<Environment *, set<string> > sourcesDepOnModulesModified_otherCompiler;
3153 __block map<Environment *, set<string> > modulesDepOnModulesRemoved_otherCompiler;
3154 __block map<Environment *, set<string> > sourcesDepOnModulesRemoved_otherCompiler;
3156 vector<VuoDirectedAcyclicNetwork *> searchDependencyGraphs;
3158 searchDependencyGraphs.push_back(compiler->dependencyGraph);
3160 VuoDirectedAcyclicGraph *currentEnvironmentDependencyGraph = (currentEnvironment ? currentEnvironment->getDependencyGraph() :
nullptr);
3162 findDependentModulesAndSources(modulesModified, searchDependencyGraphs, currentEnvironmentDependencyGraph,
3163 modulesDepOnModulesModified, modulesDepOnModulesModified_otherCompiler,
3164 sourcesDepOnModulesModified, sourcesDepOnModulesModified_otherCompiler);
3166 findDependentModulesAndSources(modulesRemoved, searchDependencyGraphs, currentEnvironmentDependencyGraph,
3167 modulesDepOnModulesRemoved, modulesDepOnModulesRemoved_otherCompiler,
3168 sourcesDepOnModulesRemoved, sourcesDepOnModulesRemoved_otherCompiler);
3170 set<Environment *> otherEnvironments;
3171 for (map<Environment *, set<string> >::iterator i = modulesDepOnModulesModified_otherCompiler.begin(); i != modulesDepOnModulesModified_otherCompiler.end(); ++i)
3172 otherEnvironments.insert(i->first);
3173 for (map<Environment *, set<string> >::iterator i = sourcesDepOnModulesModified_otherCompiler.begin(); i != sourcesDepOnModulesModified_otherCompiler.end(); ++i)
3174 otherEnvironments.insert(i->first);
3175 for (map<Environment *, set<string> >::iterator i = modulesDepOnModulesRemoved_otherCompiler.begin(); i != modulesDepOnModulesRemoved_otherCompiler.end(); ++i)
3176 otherEnvironments.insert(i->first);
3177 for (map<Environment *, set<string> >::iterator i = sourcesDepOnModulesRemoved_otherCompiler.begin(); i != sourcesDepOnModulesRemoved_otherCompiler.end(); ++i)
3178 otherEnvironments.insert(i->first);
3180 for (Environment *env : otherEnvironments)
3184 for (
const vector<Environment *> &ee : c->environments)
3185 for (Environment *e : ee)
3189 goto foundOtherCompiler;
3193 dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{
3194 dispatch_sync(environmentQueue, ^{
3195 otherCompiler->loadModulesAndSources(set<string>(), modulesDepOnModulesModified_otherCompiler[env], modulesDepOnModulesRemoved_otherCompiler[env],
3196 set<string>(), sourcesDepOnModulesModified_otherCompiler[env], sourcesDepOnModulesRemoved_otherCompiler[env],
3197 false,
true, env,
nullptr,
nullptr,
"");
3207 map<Environment *, set<VuoCompilerModule *> > actualModulesRemoved;
3208 for (
const vector<Environment *> &envs : environments)
3210 for (Environment *env : envs)
3212 set<string> modulesToUnload;
3213 modulesToUnload.insert(modulesRemoved[env].begin(), modulesRemoved[env].end());
3214 modulesToUnload.insert(modulesModified[env].begin(), modulesModified[env].end());
3215 modulesToUnload.insert(modulesDepOnModulesRemoved[env].begin(), modulesDepOnModulesRemoved[env].end());
3216 modulesToUnload.insert(modulesDepOnModulesModified[env].begin(), modulesDepOnModulesModified[env].end());
3218 actualModulesRemoved[env] = env->unloadCompiledModules(modulesToUnload);
3220 if (!env->isBuiltIn() && !actualModulesRemoved[env].empty())
3222 set<string> actualModulesRemovedKeys;
3223 for (
auto m : actualModulesRemoved[env])
3224 actualModulesRemovedKeys.insert(m->getPseudoBase()->getModuleKey());
3237 map<Environment *, set<string> > modulesToLoad;
3238 map<Environment *, map<string, string> > modulesToLoadSourceCode;
3239 for (vector< vector<Environment *> >::iterator i = environments.begin(); i != environments.end(); ++i)
3241 Environment *env = (*i).at(0);
3243 if (! modulesAdded[env].empty())
3244 modulesToLoad[env].insert(modulesAdded[env].begin(), modulesAdded[env].end());
3245 if (! modulesModified[env].empty())
3246 modulesToLoad[env].insert(modulesModified[env].begin(), modulesModified[env].end());
3247 if (! modulesDepOnModulesModified[env].empty())
3248 modulesToLoad[env].insert(modulesDepOnModulesModified[env].begin(), modulesDepOnModulesModified[env].end());
3250 if (env == currentEnvironment && moduleLoadedCallback)
3252 if (modulesAdded[env].size() == 1)
3253 modulesToLoadSourceCode[env][*modulesAdded[env].begin()] = moduleAddedOrModifiedSourceCode;
3254 else if (modulesModified[env].size() == 1)
3255 modulesToLoadSourceCode[env][*modulesModified[env].begin()] = moduleAddedOrModifiedSourceCode;
3259 map<Environment *, set<VuoCompilerModule *> > actualModulesAdded;
3260 while (! modulesToLoad.empty())
3262 set<string> dependenciesToLoad;
3263 map<Environment *, set<string> > potentialSpecializedDependencies;
3264 for (vector< vector<Environment *> >::iterator i = environments.begin(); i != environments.end(); ++i)
3266 Environment *env = (*i).at(0);
3267 Environment *genEnv = (*i).at(1);
3269 set<VuoCompilerModule *> actualModulesLoaded = env->loadCompiledModules(modulesToLoad[env], modulesToLoadSourceCode[env]);
3271 actualModulesAdded[env].insert(actualModulesLoaded.begin(), actualModulesLoaded.end());
3272 modulesToLoad.erase(env);
3274 for (set<VuoCompilerModule *>::iterator j = actualModulesLoaded.begin(); j != actualModulesLoaded.end(); ++j)
3276 set<string> dependencies = (*j)->getDependencies();
3277 dependenciesToLoad.insert(dependencies.begin(), dependencies.end());
3278 potentialSpecializedDependencies[genEnv].insert(dependencies.begin(), dependencies.end());
3281 if (!env->isBuiltIn() && !actualModulesLoaded.empty())
3283 map<string, string> actualFilesAndHashesLoaded;
3284 for (
auto module : actualModulesLoaded)
3292 if (cnc && !cnc->getSourcePath().empty())
3295 if (!cnc->getSourceCode().empty())
3310 actualFilesAndHashesLoaded[path] = hash;
3313 for (pair<string, string> item : actualFilesAndHashesLoaded)
3314 VUserLog(
"Loaded into %s environment: [%8.8s] %s", env->getName().c_str(), item.second.c_str(), item.first.c_str());
3318 for (vector< vector<Environment *> >::iterator i = environments.begin(); i != environments.end(); ++i)
3320 Environment *env = (*i).at(0);
3322 ModuleInfoIterator dependenciesInEnv = env->listModules(dependenciesToLoad);
3323 ModuleInfo *moduleInfo;
3324 while ((moduleInfo = dependenciesInEnv.next()))
3326 modulesToLoad[env].insert( moduleInfo->getModuleKey() );
3328 for (map<Environment *, set<string> >::iterator j = potentialSpecializedDependencies.begin(); j != potentialSpecializedDependencies.end(); ++j)
3330 auto foundIter = j->second.find( moduleInfo->getModuleKey() );
3331 if (foundIter != j->second.end())
3332 j->second.erase(foundIter);
3337 for (vector< vector<Environment *> >::iterator i = environments.begin(); i != environments.end(); ++i)
3339 Environment *genEnv = (*i).at(1);
3340 potentialSpecializedModules[genEnv].insert(potentialSpecializedDependencies[genEnv].begin(), potentialSpecializedDependencies[genEnv].end());
3347 set<dispatch_group_t> specializedModulesLoading;
3348 for (vector< vector<Environment *> >::iterator i = environments.begin(); i != environments.end(); ++i)
3350 Environment *genEnv = (*i).at(1);
3351 set<dispatch_group_t> s = genEnv->loadSpecializedModules(potentialSpecializedModules[genEnv],
this,
llvmQueue);
3352 specializedModulesLoading.insert(s.begin(), s.end());
3357 if (moduleLoadedCallback)
3358 moduleLoadedCallback();
3362 map<Environment *, set< pair<VuoCompilerModule *, VuoCompilerModule *> > > actualModulesModified;
3363 for (vector< vector<Environment *> >::iterator i = environments.begin(); i != environments.end(); ++i)
3365 Environment *env = (*i).at(0);
3367 for (set<VuoCompilerModule *>::iterator
add = actualModulesAdded[env].begin();
add != actualModulesAdded[env].end(); )
3369 set<VuoCompilerModule *>::iterator rem;
3370 for (rem = actualModulesRemoved[env].begin(); rem != actualModulesRemoved[env].end(); ++rem)
3371 if ((*rem)->getPseudoBase()->getModuleKey() == (*add)->getPseudoBase()->getModuleKey())
3374 if (rem != actualModulesRemoved[env].end())
3376 actualModulesModified[env].insert( make_pair(*rem, *
add) );
3377 actualModulesRemoved[env].erase(rem);
3378 actualModulesAdded[env].erase(
add++);
3387 bool wereModulesAddedOrModified =
false;
3388 for (vector< vector<Environment *> >::iterator i = environments.begin(); i != environments.end(); ++i)
3390 Environment *env = (*i).at(0);
3391 if (! (actualModulesAdded[env].empty() && actualModulesModified[env].empty()) )
3393 wereModulesAddedOrModified =
true;
3398 if (wereModulesAddedOrModified)
3400 map<string, VuoCompilerType *> inheritedTypes;
3401 for (
const vector<Environment *> &envs : environments)
3403 for (Environment *env : envs)
3405 env->reifyPortTypes(inheritedTypes);
3406 map<string, VuoCompilerType *> envTypes = env->getTypes();
3407 inheritedTypes.insert(envTypes.begin(), envTypes.end());
3416 for (vector< vector<Environment *> >::iterator i = environments.begin(); i != environments.end(); ++i)
3418 Environment *env = (*i).at(0);
3420 set<string> sourcesToUnload;
3421 sourcesToUnload.insert(sourcesRemoved[env].begin(), sourcesRemoved[env].end());
3422 sourcesToUnload.insert(sourcesDepOnModulesRemoved[env].begin(), sourcesDepOnModulesRemoved[env].end());
3423 if (! sourcesToUnload.empty())
3425 string moduleSearchPath = env->getModuleSearchPaths().front();
3427 dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{
3430 dispatch_sync(environmentQueue, ^{
3431 otherCompiler->loadModulesAndSources(set<string>(), set<string>(), sourcesToUnload,
3432 set<string>(), set<string>(), set<string>(),
3433 false,
false, env,
nullptr,
nullptr,
"");
3436 delete otherCompiler;
3440 if (!env->isBuiltIn() && !sourcesToUnload.empty())
3443 env->deleteModulesCompiledFromSourceCode(sourcesToUnload);
3451 map<Environment *, set<string> > sourcesDepOnModulesAdded;
3453 map<Environment *, set<string> > modulesDepOnModulesAdded;
3454 __block map<Environment *, set<string> > modulesDepOnModulesAdded_otherCompiler;
3455 __block map<Environment *, set<string> > sourcesDepOnModulesAdded_otherCompiler;
3457 map<Environment *, set<string> > actualModuleKeysAdded;
3458 for (
const vector<Environment *> &envs : environments)
3460 Environment *env = envs.at(0);
3465 vector<VuoDirectedAcyclicNetwork *> searchDependencyGraphs;
3466 searchDependencyGraphs.push_back(compositionDependencyGraph);
3467 for (map<
string, vector<Environment *> >::iterator ii = environmentsForCompositionFamily.begin(); ii != environmentsForCompositionFamily.end(); ++ii)
3469 vector< vector<Environment *> > otherEnvs = sharedEnvironments;
3470 otherEnvs.push_back(ii->second);
3472 searchDependencyGraphs.push_back(other);
3475 VuoDirectedAcyclicGraph *currentEnvironmentDependencyGraph = (currentEnvironment ? currentEnvironment->getCompositionDependencyGraph() :
nullptr);
3477 findDependentModulesAndSources(actualModuleKeysAdded, searchDependencyGraphs, currentEnvironmentDependencyGraph,
3478 modulesDepOnModulesAdded, modulesDepOnModulesAdded_otherCompiler,
3479 sourcesDepOnModulesAdded, sourcesDepOnModulesAdded_otherCompiler);
3481 set<Environment *> otherEnvironments;
3482 for (map<Environment *, set<string> >::iterator i = sourcesDepOnModulesAdded_otherCompiler.begin(); i != sourcesDepOnModulesAdded_otherCompiler.end(); ++i)
3483 otherEnvironments.insert(i->first);
3485 for (Environment *env : otherEnvironments)
3487 dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{
3488 string moduleSearchPath = env->getModuleSearchPaths().front();
3491 dispatch_sync(environmentQueue, ^{
3492 otherCompiler->loadModulesAndSources(set<string>(), set<string>(), set<string>(),
3493 sourcesDepOnModulesAdded_otherCompiler[env], set<string>(), set<string>(),
3494 false,
true, env,
nullptr,
nullptr,
"");
3497 delete otherCompiler;
3502 set<dispatch_group_t> sourcesLoading;
3503 for (vector< vector<Environment *> >::iterator i = environments.begin(); i != environments.end(); ++i)
3505 Environment *env = (*i).at(0);
3507 set<string> sourcesToCompile;
3508 sourcesToCompile.insert(sourcesAdded[env].begin(), sourcesAdded[env].end());
3509 sourcesToCompile.insert(sourcesModified[env].begin(), sourcesModified[env].end());
3511 if (sourcesToCompile.size() == 0)
3514 set<dispatch_group_t> s = env->compileModulesFromSourceCode(sourcesToCompile, shouldRecompileSourcesIfUnchanged);
3515 sourcesLoading.insert(s.begin(), s.end());
3518 for (vector< vector<Environment *> >::iterator i = environments.begin(); i != environments.end(); ++i)
3520 Environment *env = (*i).at(0);
3522 set<string> sourcesToCompile;
3523 sourcesToCompile.insert(sourcesDepOnModulesAdded[env].begin(), sourcesDepOnModulesAdded[env].end());
3524 sourcesToCompile.insert(sourcesDepOnModulesModified[env].begin(), sourcesDepOnModulesModified[env].end());
3526 if (sourcesToCompile.size() == 0)
3529 env->compileModulesFromSourceCode(sourcesToCompile,
true);
3534 for (vector< vector<Environment *> >::iterator i = environments.begin(); i != environments.end(); ++i)
3536 Environment *env = (*i).at(0);
3537 env->notifyCompilers(actualModulesAdded[env], actualModulesModified[env], actualModulesRemoved[env], issues[env]);
3544 for (
const dispatch_group_t &group : sourcesLoading)
3545 dispatch_retain(group);
3547 set<dispatch_group_t> loadingGroups;
3548 loadingGroups.insert(specializedModulesLoading.begin(), specializedModulesLoading.end());
3549 loadingGroups.insert(sourcesLoading.begin(), sourcesLoading.end());
3550 return loadingGroups;
3564 void VuoCompiler::findDependentModulesAndSources(map<Environment *, set<string> > &changedModules,
3565 const vector<VuoDirectedAcyclicNetwork *> &searchDependencyGraphs,
3567 map<Environment *, set<string> > &modulesDepOnChangedModules_this,
3568 map<Environment *, set<string> > &modulesDepOnChangedModules_other,
3569 map<Environment *, set<string> > &sourcesDepOnChangedModules_this,
3570 map<Environment *, set<string> > &sourcesDepOnChangedModules_other)
3572 for (
const vector<Environment *> &envs : environments)
3574 Environment *env = envs.at(0);
3576 for (
const string &module : changedModules[env])
3578 set<VuoDirectedAcyclicGraph::Vertex *> dependents;
3581 vector<VuoDirectedAcyclicGraph::Vertex *> moduleVertices;
3582 if (currentEnvironmentDependencyGraph)
3588 moduleVertices.push_back(mv);
3591 moduleVertices = searchDependencyGraph->findVertex(module);
3595 DependencyGraphVertex *moduleVertex =
static_cast<DependencyGraphVertex *
>(moduleVertexRaw);
3596 if (moduleVertex->getEnvironment())
3598 vector<VuoDirectedAcyclicGraph::Vertex *> upstream = searchDependencyGraph->getUpstreamVertices(moduleVertex);
3599 dependents.insert(upstream.begin(), upstream.end());
3604 set< pair<Environment *, string> > dependentsMap;
3607 DependencyGraphVertex *v =
static_cast<DependencyGraphVertex *
>(dependentVertexRaw);
3608 Environment *dependentEnv = v->getEnvironment();
3612 string dependent = v->getDependency();
3614 dependentsMap.insert({dependentEnv, dependent});
3620 for (
auto i : envs.at(1)->getNodeClasses())
3624 dependentsMap.insert({envs.at(1), i.first});
3627 for (
auto i : dependentsMap)
3629 Environment *dependentEnv = i.first;
3630 string dependent = i.second;
3634 if (changedModules[dependentEnv].find(dependent) != changedModules[dependentEnv].end())
3637 ModuleInfo *foundSourceInfo = dependentEnv->listSourceFile(dependent);
3638 ModuleInfo *foundModuleInfo = dependentEnv->listModule(dependent);
3640 bool belongsToCurrentCompiler =
false;
3641 for (
const vector<Environment *> &envs2 : environments)
3643 if (find(envs2.begin(), envs2.end(), dependentEnv) != envs2.end())
3645 belongsToCurrentCompiler =
true;
3650 map<Environment *, set<string> > *whicheverDependents =
nullptr;
3651 ModuleInfo *moduleInfo =
nullptr;
3652 if (foundSourceInfo)
3654 moduleInfo = foundSourceInfo;
3655 whicheverDependents = (belongsToCurrentCompiler ? &sourcesDepOnChangedModules_this : &sourcesDepOnChangedModules_other);
3657 else if (foundModuleInfo)
3659 moduleInfo = foundModuleInfo;
3660 whicheverDependents = (belongsToCurrentCompiler ? &modulesDepOnChangedModules_this : &modulesDepOnChangedModules_other);
3664 whicheverDependents = (belongsToCurrentCompiler ? &modulesDepOnChangedModules_this : &modulesDepOnChangedModules_other);
3667 (*whicheverDependents)[dependentEnv].insert(dependent);
3669 moduleInfo->setAttempted(
false);
3678 void VuoCompiler::loadedModules(map<string, VuoCompilerModule *> modulesAdded,
3679 map<
string, pair<VuoCompilerModule *, VuoCompilerModule *> > modulesModified,
3680 map<string, VuoCompilerModule *> modulesRemoved,
3681 VuoCompilerIssues *issues,
void *delegateDataV, Environment *currentEnvironment)
3693 auto findVersionsOfModule = [
this, currentEnvironment] (
const string &moduleKey)
3695 vector< pair<Environment *, VuoCompilerModule *> > moduleVersions;
3696 for (
const vector<Environment *> &envs : environments)
3698 Environment *env = envs.at(0);
3700 if (module || env == currentEnvironment)
3701 moduleVersions.push_back( make_pair(env, module) );
3703 return moduleVersions;
3706 for (map<string, VuoCompilerModule *>::iterator i = modulesAdded.begin(); i != modulesAdded.end(); )
3708 string moduleKey = i->first;
3711 vector< pair<Environment *, VuoCompilerModule *> > moduleVersions = findVersionsOfModule(moduleKey);
3713 if (moduleVersions.size() > 1)
3715 modulesAdded.erase(i++);
3717 if (moduleVersions.back().second == moduleAdded)
3719 VuoCompilerModule *moduleSuperseded = moduleVersions.at(moduleVersions.size()-2).second;
3720 modulesModified[moduleKey] = make_pair(moduleSuperseded, moduleAdded);
3727 for (map<
string, pair<VuoCompilerModule *, VuoCompilerModule *> >::iterator i = modulesModified.begin(); i != modulesModified.end(); )
3729 string moduleKey = i->first;
3732 vector< pair<Environment *, VuoCompilerModule *> > moduleVersions = findVersionsOfModule(moduleKey);
3734 if (moduleVersions.size() > 1 && moduleVersions.back().second != moduleModified)
3735 modulesModified.erase(i++);
3740 for (map<string, VuoCompilerModule *>::iterator i = modulesRemoved.begin(); i != modulesRemoved.end(); )
3742 string moduleKey = i->first;
3745 vector< pair<Environment *, VuoCompilerModule *> > moduleVersions = findVersionsOfModule(moduleKey);
3747 if (moduleVersions.size() > 1)
3749 modulesRemoved.erase(i++);
3751 if (moduleVersions.back().first == currentEnvironment)
3753 VuoCompilerModule *moduleUnsuperseded = moduleVersions.at(moduleVersions.size()-2).second;
3754 modulesModified[moduleKey] = make_pair(moduleRemoved, moduleUnsuperseded);
3761 dispatch_async(delegateQueue, ^{
3762 VuoCompilerDelegate::LoadedModulesData *delegateData =
static_cast<VuoCompilerDelegate::LoadedModulesData *
>(delegateDataV);
3764 if (delegate && ! (modulesAdded.empty() && modulesModified.empty() && modulesRemoved.empty() && issues->
isEmpty()))
3766 delegate->enqueueData(delegateData);
3767 delegate->
loadedModules(modulesAdded, modulesModified, modulesRemoved, issues);
3771 delegateData->release();
3781 void VuoCompiler::loadNodeClassGeneratedAtRuntime(
VuoCompilerNodeClass *nodeClass, Environment *env)
3783 Module *module = nodeClass->
getModule();
3787 setTargetForModule(nodeClass->
getModule());
3791 dispatch_sync(environmentQueue, ^{
3792 env->replaceNodeClass(nodeClass);
3795 __block map<string, VuoCompilerType *> inheritedTypes;
3796 void (^envReifyPortTypes)(Environment *) = ^
void (Environment *env) {
3797 env->reifyPortTypes(inheritedTypes);
3798 map<string, VuoCompilerType *> currentTypes = env->getTypes();
3799 inheritedTypes.insert(currentTypes.begin(), currentTypes.end());
3801 applyToAllEnvironments(envReifyPortTypes);
3810 reifyGenericPortTypes(node->getBase());
3818 void VuoCompiler::reifyGenericPortTypes(
VuoNode *node)
3828 vector<VuoPort *> ports;
3829 ports.insert(ports.end(), inputPorts.begin(), inputPorts.end());
3830 ports.insert(ports.end(), outputPorts.begin(), outputPorts.end());
3832 for (vector<VuoPort *>::iterator j = ports.begin(); j != ports.end(); ++j)
3879 vector<string> allIncludePaths = includePaths;
3880 string preprocessedInputPath = inputPath;
3882 string tmpPreprocessedInputDir;
3883 string dir, file, ext;
3888 string preprocessedInputContents = inputContents;
3890 if (inputContents != preprocessedInputContents)
3893 allIncludePaths.push_back(dir.empty() ?
"." : dir);
3895 preprocessedInputPath = tmpPreprocessedInputDir +
"/" + file +
"." + ext;
3900 else if (ext ==
"fs")
3906 auto getType = [
this] (
const string &moduleKey) {
return this->
getType(moduleKey); };
3911 writeModuleToBitcode(module, outputPath);
3921 vector<string> extraArgs;
3922 for (vector<string>::iterator i = allIncludePaths.begin(); i != allIncludePaths.end(); ++i)
3924 extraArgs.push_back(
"-I");
3925 extraArgs.push_back(*i);
3931 string buildTimeMacOSSDKFolder = MACOS_SDK_ROOT;
3934 extraArgs.push_back(
"-isysroot");
3935 extraArgs.push_back(buildTimeMacOSSDKFolder);
3939 __block vector<string> headerSearchPaths;
3940 void (^envGetHeaderSearchPaths)(Environment *) = ^
void (Environment *env) {
3941 vector<string> result = env->getHeaderSearchPaths();
3942 headerSearchPaths.insert(headerSearchPaths.end(), result.begin(), result.end());
3944 applyToInstalledEnvironments(envGetHeaderSearchPaths);
3947 __block Module *module;
3949 module = readModuleFromC(preprocessedInputPath, headerSearchPaths, extraArgs, issues);
3952 if (! tmpPreprocessedInputDir.empty())
3954 remove(tmpPreprocessedInputDir.c_str());
3963 if (! compilerModule)
3965 VUserLog(
"Error: Didn't recognize '%s' as a node class, type, or library.", inputPath.c_str());
3969 setTargetForModule(module, target);
3970 writeModuleToBitcode(module, outputPath);
3979 Module * VuoCompiler::compileCompositionToModule(
VuoCompilerComposition *composition,
const string &moduleKey,
bool isTopLevelComposition,
3982 composition->
check(issues);
3984 reifyGenericPortTypes(composition);
3987 isTopLevelComposition,
3989 if (telemetry ==
"console")
3992 __block Module *module =
nullptr;
3997 setTargetForModule(module, target);
4024 Module *module = compileCompositionToModule(composition, moduleKey, isTopLevelComposition, issues);
4029 writeModuleToBitcode(module, outputPath);
4048 VDebugLog(
"Compiling '%s'…", inputPath.c_str());
4058 "",
"The composition file couldn't be read or was empty.");
4096 delete baseComposition;
4102 void VuoCompiler::compileSubcompositionString(
const string &compositionString,
const string &outputPath,
4103 std::function<
void(
void)> moduleLoadedCallback, Environment *environment,
4109 bool compilationSucceeded =
false;
4113 compilationSucceeded =
true;
4121 if (! compilationSucceeded)
4127 environment->moduleFileChanged(outputPath, compositionString, moduleLoadedCallback,
this, issues);
4145 linkCompositionToCreateExecutableOrDynamicLibrary(inputPath, outputPath, optimization,
false, rPath);
4166 linkCompositionToCreateExecutableOrDynamicLibrary(inputPath, outputPath, optimization,
true);
4183 void VuoCompiler::linkCompositionToCreateExecutableOrDynamicLibrary(
string compiledCompositionPath,
string linkedCompositionPath,
4184 Optimization optimization,
bool isDylib,
string rPath)
4190 shouldLoadAllModules =
false;
4192 set<string> dependencies = getDependenciesForComposition(compiledCompositionPath);
4193 dependencies.insert(getRuntimeDependency());
4195 dependencies.insert(getRuntimeMainDependency());
4197 set<Module *> modules;
4198 set<string> libraries;
4199 set<string> frameworks;
4200 getLinkerInputs(dependencies, optimization, modules, libraries, frameworks);
4202 libraries.insert(compiledCompositionPath);
4204 link(linkedCompositionPath, modules, libraries, frameworks, isDylib, rPath);
4230 set<string> allDependencies = getDependenciesForComposition(compiledCompositionPath);
4231 set<string> addedDependencies;
4232 std::set_difference(allDependencies.begin(), allDependencies.end(),
4233 carriedOverDependencies.begin(), carriedOverDependencies.end(),
4234 std::inserter(addedDependencies, addedDependencies.end()));
4240 set<string> carriedOverExternalLibraries = runningCompositionLibraries->
getExternalLibraries();
4245 string nonUnloadableResourcePath;
4246 string unloadableResourcePath;
4247 set<string> nonUnloadableDependencies;
4248 set<string> unloadableDependencies;
4249 map<string, set<string> > builtInCacheDependencies;
4250 map<string, set<string> > userCacheDependencies;
4251 set<string> builtInLibraries;
4252 set<string> userLibraries;
4253 set<string> addedExternalLibraries;
4254 set<string> addedFrameworks;
4255 set<string> allFrameworks;
4256 if (! addedDependencies.empty())
4258 set<string> builtInModuleAndLibraryDependencies;
4259 set<string> userModuleAndLibraryDependencies;
4260 set<Module *> builtInModules;
4261 set<Module *> userModules;
4264 builtInModuleAndLibraryDependencies, userModuleAndLibraryDependencies, builtInCacheDependencies, userCacheDependencies,
4265 builtInModules, userModules, builtInLibraries, userLibraries, addedExternalLibraries, addedFrameworks);
4267 allFrameworks.insert(carriedOverFrameworks.begin(), carriedOverFrameworks.end());
4268 allFrameworks.insert(addedFrameworks.begin(), addedFrameworks.end());
4270 string dir, linkedCompositionFile, ext;
4273 if (! builtInModules.empty() || builtInLibraries.size() > builtInCacheDependencies.size())
4276 nonUnloadableDependencies = builtInModuleAndLibraryDependencies;
4278 set<string> librariesForNonUnloadableResource = builtInLibraries;
4279 librariesForNonUnloadableResource.insert(carriedOverNonUnloadableLibraries.begin(), carriedOverNonUnloadableLibraries.end());
4280 librariesForNonUnloadableResource.insert(carriedOverExternalLibraries.begin(), carriedOverExternalLibraries.end());
4281 librariesForNonUnloadableResource.insert(addedExternalLibraries.begin(), addedExternalLibraries.end());
4283 link(nonUnloadableResourcePath, builtInModules, librariesForNonUnloadableResource, allFrameworks,
true);\
4285 for (set<string>::iterator i = builtInLibraries.begin(); i != builtInLibraries.end(); )
4288 builtInLibraries.erase(i++);
4293 for (set<string>::iterator i = addedExternalLibraries.begin(); i != addedExternalLibraries.end(); )
4296 addedExternalLibraries.erase(i++);
4302 if (! userModules.empty() || userLibraries.size() > userCacheDependencies.size())
4305 unloadableDependencies = userModuleAndLibraryDependencies;
4307 set<string> librariesForUnloadableResource = userLibraries;
4308 librariesForUnloadableResource.insert(builtInLibraries.begin(), builtInLibraries.end());
4309 librariesForUnloadableResource.insert(carriedOverUnloadableLibraries.begin(), carriedOverUnloadableLibraries.end());
4310 librariesForUnloadableResource.insert(carriedOverNonUnloadableLibraries.begin(), carriedOverNonUnloadableLibraries.end());
4311 librariesForUnloadableResource.insert(carriedOverExternalLibraries.begin(), carriedOverExternalLibraries.end());
4312 librariesForUnloadableResource.insert(addedExternalLibraries.begin(), addedExternalLibraries.end());
4313 if (! nonUnloadableResourcePath.empty())
4314 librariesForUnloadableResource.insert(nonUnloadableResourcePath);
4316 link(unloadableResourcePath, userModules, librariesForUnloadableResource, allFrameworks,
true);
4318 for (set<string>::iterator i = userLibraries.begin(); i != userLibraries.end(); )
4321 userLibraries.erase(i++);
4330 set<string> vuoRuntimePaths;
4332 set<Module *> modules;
4333 set<string> libraries;
4334 set<string> frameworks;
4336 set<string> dependencies;
4337 dependencies.insert(getRuntimeDependency());
4339 vuoRuntimePaths = libraries;
4345 set<Module *> modules;
4346 set<string> libraries;
4348 libraries.insert(compiledCompositionPath);
4349 libraries.insert(carriedOverExternalLibraries.begin(), carriedOverExternalLibraries.end());
4350 libraries.insert(addedExternalLibraries.begin(), addedExternalLibraries.end());
4351 libraries.insert(carriedOverNonUnloadableLibraries.begin(), carriedOverNonUnloadableLibraries.end());
4352 libraries.insert(carriedOverUnloadableLibraries.begin(), carriedOverUnloadableLibraries.end());
4353 libraries.insert(builtInLibraries.begin(), builtInLibraries.end());
4354 libraries.insert(userLibraries.begin(), userLibraries.end());
4355 if (! nonUnloadableResourcePath.empty())
4356 libraries.insert(nonUnloadableResourcePath);
4357 if (! unloadableResourcePath.empty())
4358 libraries.insert(unloadableResourcePath);
4359 libraries.insert(vuoRuntimePaths.begin(), vuoRuntimePaths.end());
4360 link(linkedCompositionPath, modules, libraries, allFrameworks,
true);
4365 if (! nonUnloadableResourcePath.empty())
4368 if (! unloadableResourcePath.empty())
4371 for (map<
string, set<string> >::iterator i = builtInCacheDependencies.begin(); i != builtInCacheDependencies.end(); ++i)
4374 for (map<
string, set<string> >::iterator i = userCacheDependencies.begin(); i != userCacheDependencies.end(); ++i)
4387 set<string> VuoCompiler::getDependenciesForComposition(
const string &compiledCompositionPath)
4389 VDebugLog(
"Gathering dependencies for '%s'…", compiledCompositionPath.c_str());
4392 __block set<string> directDependencies;
4394 Module *module = readModuleFromBitcode(compiledCompositionPath);
4398 delete compilerModule;
4404 auto deps = getDependenciesForComposition(directDependencies,
true);
4423 set<string> directDependencies;
4432 vector<VuoPublishedPort *> publishedPorts;
4433 publishedPorts.insert(publishedPorts.end(), publishedInputPorts.begin(), publishedInputPorts.end());
4434 publishedPorts.insert(publishedPorts.end(), publishedOutputPorts.begin(), publishedOutputPorts.end());
4437 if (publishedPort->getClass()->hasCompiler())
4453 directDependencies.insert(dependency);
4458 return directDependencies;
4470 return getDependenciesForComposition(directDependencies,
false);
4480 __block vector<string> librarySearchPaths;
4481 applyToInstalledEnvironments(^
void (Environment *env) {
4482 vector<string> result = env->getLibrarySearchPaths();
4483 librarySearchPaths.insert(librarySearchPaths.end(), result.begin(), result.end());
4486 set<string> dylibDeps;
4487 for (
string dep : getDependenciesForComposition(composition))
4489 string path = getLibraryPath(dep, librarySearchPaths);
4491 dylibDeps.insert(path);
4510 set<string> VuoCompiler::getDependenciesForComposition(
const set<string> &directDependencies,
bool checkCompatibility)
4513 for (set<string>::const_iterator i = directDependencies.begin(); i != directDependencies.end(); ++i)
4516 set<string> dependencies;
4517 for (set<string>::iterator i = directDependencies.begin(); i != directDependencies.end(); ++i)
4519 string moduleKey = *i;
4521 dependencies.insert(moduleKey);
4524 vector<VuoDirectedAcyclicGraph::Vertex *> firstPassVertices = dependencyGraph->
findVertex(moduleKey);
4525 set<VuoDirectedAcyclicGraph::Vertex *> firstPassDependencies(firstPassVertices.begin(), firstPassVertices.end());
4526 for (vector<VuoDirectedAcyclicGraph::Vertex *>::iterator j = firstPassVertices.begin(); j != firstPassVertices.end(); ++j)
4529 firstPassDependencies.insert(downstream.begin(), downstream.end());
4533 for (set<VuoDirectedAcyclicGraph::Vertex *>::iterator j = firstPassDependencies.begin(); j != firstPassDependencies.end(); ++j)
4535 DependencyGraphVertex *v =
static_cast<DependencyGraphVertex *
>(*j);
4540 vector<VuoDirectedAcyclicGraph::Vertex *> moduleVertices = dependencyGraph->
findVertex(moduleKey);
4541 set<VuoDirectedAcyclicGraph::Vertex *> moduleDependencies(moduleVertices.begin(), moduleVertices.end());
4542 for (vector<VuoDirectedAcyclicGraph::Vertex *>::iterator j = moduleVertices.begin(); j != moduleVertices.end(); ++j)
4545 moduleDependencies.insert(downstream.begin(), downstream.end());
4549 set<string> dependenciesToAdd;
4550 set<string> incompatibleDependencies;
4551 for (set<VuoDirectedAcyclicGraph::Vertex *>::iterator j = moduleDependencies.begin(); j != moduleDependencies.end(); ++j)
4553 DependencyGraphVertex *v =
static_cast<DependencyGraphVertex *
>(*j);
4554 if (! checkCompatibility || ! v->getEnvironment() || v->isCompatible())
4555 dependenciesToAdd.insert(v->getDependency());
4557 incompatibleDependencies.insert(v->getDependency());
4560 if (! checkCompatibility || incompatibleDependencies.empty())
4562 dependencies.insert(dependenciesToAdd.begin(), dependenciesToAdd.end());
4569 string dependencyTargetString;
4574 for (set<string>::iterator i = incompatibleDependencies.begin(); i != incompatibleDependencies.end(); ++i)
4583 dependencyTargetString = dependencyTargets.
toString();
4586 dependencyTargetString =
"(unknown operating systems)";
4588 string modulePlaceholder = (module ?
"%module" :
"%moduleKey");
4590 "Node incompatible with operating system",
4591 modulePlaceholder +
" is only compatible with " + dependencyTargetString +
4592 ", so this composition can't run on your macOS version (" + compositionTargets.
toString() +
").");
4594 issue.setModuleKey(moduleKey);
4600 vector<string> coreDependencies = getCoreVuoDependencies();
4601 dependencies.insert(coreDependencies.begin(), coreDependencies.end());
4603 return dependencies;
4610 void VuoCompiler::getLinkerInputs(
const set<string> &dependencies, Optimization optimization,
4611 set<Module *> &modules, set<string> &libraries, set<string> &frameworks)
4613 set<string> builtInModuleAndLibraryDependencies;
4614 set<string> userModuleAndLibraryDependencies;
4615 map<string, set<string> > builtInCacheDependencies;
4616 map<string, set<string> > userCacheDependencies;
4617 set<Module *> builtInModules;
4618 set<Module *> userModules;
4619 set<string> builtInLibraries;
4620 set<string> userLibraries;
4621 set<string> externalLibraries;
4623 getLinkerInputs(dependencies, optimization,
4624 builtInModuleAndLibraryDependencies, userModuleAndLibraryDependencies, builtInCacheDependencies, userCacheDependencies,
4625 builtInModules, userModules, builtInLibraries, userLibraries, externalLibraries, frameworks);
4627 modules.insert(builtInModules.begin(), builtInModules.end());
4628 modules.insert(userModules.begin(), userModules.end());
4629 libraries.insert(builtInLibraries.begin(), builtInLibraries.end());
4630 libraries.insert(userLibraries.begin(), userLibraries.end());
4631 libraries.insert(externalLibraries.begin(), externalLibraries.end());
4647 void VuoCompiler::getLinkerInputs(
const set<string> &dependencies, Optimization optimization,
4648 set<string> &builtInModuleAndLibraryDependencies, set<string> &userModuleAndLibraryDependencies,
4649 map<
string, set<string> > &builtInCacheDependencies, map<
string, set<string> > &userCacheDependencies,
4650 set<Module *> &builtInModules, set<Module *> &userModules,
4651 set<string> &builtInLibraries, set<string> &userLibraries,
4652 set<string> &externalLibraries, set<string> &externalFrameworks)
4655 if (shouldUseModuleCache)
4658 __block vector<string> librarySearchPaths;
4659 void (^envGetLibrarySearchPaths)(Environment *) = ^
void (Environment *env) {
4660 vector<string> result = env->getLibrarySearchPaths();
4661 librarySearchPaths.insert(librarySearchPaths.end(), result.begin(), result.end());
4663 applyToInstalledEnvironments(envGetLibrarySearchPaths);
4665 for (set<string>::iterator i = dependencies.begin(); i != dependencies.end(); ++i)
4667 string dependency = *i;
4669 bool foundInCache =
false;
4670 string moduleCachePath;
4671 bool isInBuiltInModuleCache =
false;
4672 if (shouldUseModuleCache)
4673 foundInCache = findInModuleCache(dependency, moduleCachePath, isInBuiltInModuleCache);
4677 if (isInBuiltInModuleCache)
4679 builtInLibraries.insert(moduleCachePath);
4680 builtInCacheDependencies[moduleCachePath].insert(dependency);
4684 userLibraries.insert(moduleCachePath);
4685 userCacheDependencies[moduleCachePath].insert(dependency);
4691 void (^envFindModule)(Environment *) = ^
void (Environment *env) {
4696 applyToAllEnvironments(envFindModule);
4706 if (module->isBuiltIn())
4707 builtInLibraries.insert(modulePath);
4709 userLibraries.insert(modulePath);
4713 if (module->isBuiltIn())
4714 builtInModules.insert(module->getModule());
4716 userModules.insert(module->getModule());
4719 if (module->isBuiltIn())
4720 builtInModuleAndLibraryDependencies.insert(dependency);
4722 userModuleAndLibraryDependencies.insert(dependency);
4728 externalFrameworks.insert(dependency);
4731 string dependencyPath = getLibraryPath(dependency, librarySearchPaths);
4732 if (! dependencyPath.empty())
4733 externalLibraries.insert(dependencyPath);
4737 else if (dependency !=
"c"
4738 && dependency !=
"objc")
4739 VUserLog(
"Warning: Could not locate dependency '%s'.", dependency.c_str());
4751 string VuoCompiler::getLibraryPath(
const string &dependency, vector<string> librarySearchPaths)
4758 if (dependency !=
"crypto"
4759 && dependency !=
"ssl")
4760 librarySearchPaths.push_back(
"/usr/lib");
4762 for (
auto &path : librarySearchPaths)
4764 vector<string> variations;
4765 variations.push_back(path +
"/" + dependency);
4766 variations.push_back(path +
"/lib" + dependency);
4767 variations.push_back(path +
"/lib" + dependency +
".dylib");
4768 variations.push_back(path +
"/lib" + dependency +
".a");
4769 for (
auto &variation : variations)
4784 void VuoCompiler::useModuleCache(
bool shouldUseExistingBuiltInCaches,
bool shouldUseExistingOtherCaches)
4786 loadModulesIfNeeded();
4790 dispatch_sync(environmentQueue, ^{
4791 set<string> dylibsForCachesOfInstalledModules;
4792 set<string> frameworksForCachesOfInstalledModules;
4793 for (vector< vector<Environment *> >::iterator i = environments.begin(); i != environments.end(); ++i)
4795 bool builtin = (i == environments.begin());
4796 set<string> dylibsForCacheOfGeneratedModules;
4797 set<string> frameworksForCacheOfGeneratedModules;
4799 for (int j = i->size() - 1; j >= 0; --j)
4801 Environment *env = i->at(j);
4802 bool installed = (j == 0);
4804 set<string> cacheableModulesAndDependencies;
4805 set<string> dylibsNeededToLinkToThisCache;
4806 set<string> frameworksNeededToLinkToThisCache;
4807 env->getCacheableModulesAndDependencies(builtin, installed, cacheableModulesAndDependencies,
4808 dylibsNeededToLinkToThisCache, frameworksNeededToLinkToThisCache);
4810 set<string> accumulatedDylibs;
4811 accumulatedDylibs.insert(dylibsNeededToLinkToThisCache.begin(), dylibsNeededToLinkToThisCache.end());
4812 accumulatedDylibs.insert(dylibsForCachesOfInstalledModules.begin(), dylibsForCachesOfInstalledModules.end());
4813 accumulatedDylibs.insert(dylibsForCacheOfGeneratedModules.begin(), dylibsForCacheOfGeneratedModules.end());
4815 set<string> accumulatedFrameworks;
4816 accumulatedFrameworks.insert(frameworksNeededToLinkToThisCache.begin(), frameworksNeededToLinkToThisCache.end());
4817 accumulatedFrameworks.insert(frameworksForCachesOfInstalledModules.begin(), frameworksForCachesOfInstalledModules.end());
4818 accumulatedFrameworks.insert(frameworksForCacheOfGeneratedModules.begin(), frameworksForCacheOfGeneratedModules.end());
4820 bool shouldUseExistingCache = (builtin ? shouldUseExistingBuiltInCaches : shouldUseExistingOtherCaches);
4821 env->useModuleCache(shouldUseExistingCache, this, cacheableModulesAndDependencies,
4822 accumulatedDylibs, accumulatedFrameworks);
4826 dylibsForCachesOfInstalledModules.insert(dylibsNeededToLinkToThisCache.begin(), dylibsNeededToLinkToThisCache.end());
4827 frameworksForCachesOfInstalledModules.insert(frameworksNeededToLinkToThisCache.begin(), frameworksNeededToLinkToThisCache.end());
4831 dylibsForCacheOfGeneratedModules.insert(dylibsNeededToLinkToThisCache.begin(), dylibsNeededToLinkToThisCache.end());
4832 frameworksForCacheOfGeneratedModules.insert(frameworksNeededToLinkToThisCache.begin(), frameworksNeededToLinkToThisCache.end());
4838 Environment::waitForModuleCachesToBuild();
4849 bool VuoCompiler::findInModuleCache(
const string &moduleOrDependency,
string &cachePath,
bool &isBuiltinCache)
4851 __block
bool found =
false;
4852 __block
string outPath;
4853 __block
bool outBuiltin;
4854 dispatch_sync(environmentQueue, ^{
4855 for (vector< vector<Environment *> >::iterator i = environments.begin(); i != environments.end(); ++i)
4857 bool builtin = (i == environments.begin());
4859 for (int j = i->size() - 1; j >= 0; --j)
4861 Environment *env = i->at(j);
4864 bool resultFound = env->findInModuleCache(moduleOrDependency, resultPath);
4868 outPath = resultPath;
4869 outBuiltin = builtin;
4875 cachePath = outPath;
4876 isBuiltinCache = outBuiltin;
4889 dispatch_group_async(moduleCacheBuilding, dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{
4890 useModuleCache(
true,
false);
4908 vuoFrameworkInProgressPath = vuoFrameworkPath;
4911 compiler.useModuleCache(
false,
true);
4923 unsigned long maxSeconds = 30 * 24 * 60 * 60;
4926 for (set<VuoFileUtilities::File *>::iterator i = cacheDirs.begin(); i != cacheDirs.end(); ++i)
4928 string path = (*i)->path();
4930 string file = (*i)->basename();
4931 if (file !=
"Builtin" && file !=
"System" && file !=
"User")
4934 if (fileSeconds > maxSeconds)
4940 string pidAsString = file.substr(Environment::pidCacheDirPrefix.length());
4941 int pid = atoi(pidAsString.c_str());
4942 if (kill(pid, 0) != 0)
4956 dispatch_sync(modulesToLoadQueue, ^{
4957 this->shouldLoadAllModules = shouldLoadAllModules;
4972 void VuoCompiler::link(
string outputPath,
const set<Module *> &modules,
const set<string> &libraries,
const set<string> &frameworks,
bool isDylib,
string rPath,
VuoCompilerIssues *issues)
4974 VDebugLog(
"Linking '%s'…", outputPath.c_str());
4982 Module *compositeModule =
new Module(
"composite", getGlobalContext());
4983 setTargetForModule(compositeModule);
4984 for (set<Module *>::const_iterator i = modules.begin(); i != modules.end(); ++i)
4987 if (Linker::LinkModules(compositeModule, *i, Linker::PreserveSource, &error))
4988 VUserLog(
"Error: Failed to link compositeModule: %s", error.c_str());
4990 writeModuleToBitcode(compositeModule, compositeModulePath);
4991 delete compositeModule;
4999 llvm::sys::Path clangPath(getClangPath().str() +
"++");
5001 vector<const char *> args;
5002 vector<char *> argsToFree;
5003 args.push_back(clangPath.c_str());
5006 char *outputPathZ = strdup((
"-o" + outputPath).c_str());
5007 args.push_back(outputPathZ);
5008 argsToFree.push_back(outputPathZ);
5011 args.push_back(compositeModulePath.c_str());
5013 vector<string> coreDependencies = getCoreVuoDependencies();
5014 for (set<string>::const_iterator i = libraries.begin(); i != libraries.end(); ++i)
5016 string library = *i;
5018 for (vector<string>::iterator j = coreDependencies.begin(); j != coreDependencies.end(); ++j)
5020 string coreDependency = *j;
5022 args.push_back(
"-force_load");
5030 library = libraryObject;
5033 char *libraryZ = strdup(library.c_str());
5034 args.push_back(libraryZ);
5035 argsToFree.push_back(libraryZ);
5039 vector<string> frameworkArguments;
5041 __block vector<string> frameworkSearchPaths;
5042 void (^envGetFrameworkSearchPaths)(Environment *) = ^
void (Environment *env) {
5043 vector<string> result = env->getFrameworkSearchPaths();
5044 frameworkSearchPaths.insert(frameworkSearchPaths.end(), result.begin(), result.end());
5046 applyToInstalledEnvironments(envGetFrameworkSearchPaths);
5048 for (vector<string>::const_iterator i = frameworkSearchPaths.begin(); i != frameworkSearchPaths.end(); ++i)
5052 frameworkArguments.push_back(a);
5053 char *frameworkArgument = strdup(a.c_str());
5054 args.push_back(frameworkArgument);
5055 argsToFree.push_back(frameworkArgument);
5058 for (set<string>::const_iterator i = frameworks.begin(); i != frameworks.end(); ++i)
5060 args.push_back(
"-framework");
5062 string frameworkName = *i;
5063 frameworkName = frameworkName.substr(0, frameworkName.length() -
string(
".framework").length());
5064 char *frameworkNameZ = strdup(frameworkName.c_str());
5065 args.push_back(frameworkNameZ);
5066 argsToFree.push_back(frameworkNameZ);
5070 llvm::sys::Path cRuntimePath;
5071 llvm::sys::Path crt1Path;
5072 string vuoFrameworkPath = getVuoFrameworkPath();
5073 string vuoFrameworkContainingFolder = vuoFrameworkPath +
"/..";
5074 if (! vuoFrameworkPath.empty())
5076 cRuntimePath = vuoFrameworkPath +
"/Modules/";
5077 crt1Path = cRuntimePath;
5078 crt1Path.appendComponent(
"crt1.o");
5083 if (!isDylib && crt1Path.canRead())
5085 args.push_back(
"-nostartfiles");
5086 args.push_back(crt1Path.c_str());
5092 string frameworkMacOSSDKFolder = getVuoFrameworkPath() +
"/SDKs/MacOSX10.11.sdk";
5096 args.push_back(
"-Xlinker");
5097 args.push_back(
"-syslibroot");
5098 args.push_back(
"-Xlinker");
5099 char *frameworkMacOSSDKFolderZ = strdup(frameworkMacOSSDKFolder.c_str());
5100 args.push_back(frameworkMacOSSDKFolderZ);
5101 argsToFree.push_back(frameworkMacOSSDKFolderZ);
5105 args.push_back(
"-Xlinker");
5106 args.push_back(
"--no-demangle");
5109 args.push_back(
"-v");
5112 args.push_back(
"-dynamiclib");
5114 args.push_back(
"-Xlinker");
5115 args.push_back(
"-headerpad_max_install_names");
5119 args.push_back(
"-rpath");
5120 string rPathArg = (rPath.empty() ? vuoFrameworkContainingFolder : rPath);
5121 args.push_back(rPathArg.c_str());
5124 args.push_back(
"-rpath");
5125 args.push_back(LLVM_ROOT
"/lib");
5128 args.push_back(
"-std=c++11");
5129 args.push_back(
"-stdlib=libc++");
5130 args.push_back(
"-mmacosx-version-min=10.10");
5134 clang::DiagnosticOptions *diagOptions =
new clang::DiagnosticOptions();
5135 IntrusiveRefCntPtr<clang::DiagnosticIDs> DiagID(
new clang::DiagnosticIDs());
5136 clang::DiagnosticsEngine Diags(DiagID, diagOptions, diagnosticConsumer);
5141 for (vector<const char *>::iterator i = args.begin(); i != args.end(); ++i)
5148 const llvm::sys::Path stdoutPath(stdoutFile);
5149 const llvm::sys::Path *redirects[] = {
5156 const char **argsz = (
const char **)malloc(
sizeof(
char *) * args.size() + 1);
5157 for (
int i = 0; i < args.size(); ++i)
5159 argsz[args.size()] =
nullptr;
5162 bool executionFailed;
5164 int ret = llvm::sys::Program::ExecuteAndWait(llvm::sys::Path(args[0]), argsz,
nullptr, redirects, 0, 0, &errMsg, &executionFailed);
5166 for (
auto i : argsToFree)
5170 remove(compositeModulePath.c_str());
5176 chmod(outputPath.c_str(), 0755);
5181 if (!errMsg.empty())
5184 details +=
"\n" + errMsg +
"\n";
5187 if (!stdoutFileContents.empty())
5189 VUserLog(
"%s", stdoutFileContents.c_str());
5190 details +=
"\n" + stdoutFileContents +
"\n";
5195 "Node broken or outdated", details);
5208 Module *VuoCompiler::readModuleFromC(
string inputPath,
const vector<string> &headerSearchPaths,
const vector<string> &extraArgs,
VuoCompilerIssues *issues)
5212 vector<const char *> args;
5213 args.push_back(inputPath.c_str());
5214 args.push_back(
"-DVUO_COMPILER");
5215 args.push_back(
"-fblocks");
5218 args.push_back(
"-Wall");
5219 args.push_back(
"-Wextra");
5220 args.push_back(
"-Wimplicit-fallthrough");
5221 args.push_back(
"-Wno-unused-parameter");
5222 args.push_back(
"-Wno-c++11-extensions");
5223 args.push_back(
"-Wno-sign-compare");
5224 args.push_back(
"-Werror=implicit");
5228 args.push_back(
"-std=c++11");
5229 args.push_back(
"-stdlib=libc++");
5232 for (vector<string>::const_iterator i = headerSearchPaths.begin(); i != headerSearchPaths.end(); ++i)
5234 args.push_back(
"-I");
5235 args.push_back(i->c_str());
5239 args.push_back(
"-v");
5241 for (vector<string>::const_iterator i = extraArgs.begin(); i != extraArgs.end(); ++i)
5242 args.push_back(i->c_str());
5245 clang::DiagnosticOptions * diagOptions =
new clang::DiagnosticOptions();
5246 IntrusiveRefCntPtr<clang::DiagnosticIDs> DiagID(
new clang::DiagnosticIDs());
5247 clang::DiagnosticsEngine *diags =
new clang::DiagnosticsEngine(DiagID, diagOptions, diagnosticConsumer);
5249 OwningPtr<clang::CompilerInvocation> CI(
new clang::CompilerInvocation);
5250 clang::CompilerInvocation::CreateFromArgs(*CI, &args[0], &args[0] + args.size(), *diags);
5252 clang::CompilerInstance Clang;
5253 Clang.setInvocation(CI.take());
5255 Clang.setDiagnostics(diags);
5256 if (!Clang.hasDiagnostics())
5260 llvm::sys::Path builtinHeaderSearchPath;
5261 string vuoFrameworkPath = getVuoFrameworkPath();
5262 if (vuoFrameworkPath.empty())
5264 llvm::sys::Path clangPath = getClangPath();
5265 builtinHeaderSearchPath = clangPath;
5266 builtinHeaderSearchPath.eraseComponent();
5267 builtinHeaderSearchPath.eraseComponent();
5268 builtinHeaderSearchPath.appendComponent(
"lib");
5269 builtinHeaderSearchPath.appendComponent(
"clang");
5270 builtinHeaderSearchPath.appendComponent(CLANG_VERSION_STRING);
5274 builtinHeaderSearchPath = vuoFrameworkPath;
5275 builtinHeaderSearchPath.appendComponent(
"Frameworks");
5276 builtinHeaderSearchPath.appendComponent(
"llvm.framework");
5277 builtinHeaderSearchPath.appendComponent(
"Versions");
5278 builtinHeaderSearchPath.appendComponent(
"A");
5279 builtinHeaderSearchPath.appendComponent(
"lib");
5280 builtinHeaderSearchPath.appendComponent(
"clang");
5281 builtinHeaderSearchPath.appendComponent(CLANG_VERSION_STRING);
5283 Clang.getHeaderSearchOpts().ResourceDir = builtinHeaderSearchPath.str();
5286 clang::CodeGenAction *Act =
new clang::EmitLLVMOnlyAction();
5287 if (!Clang.ExecuteAction(*Act))
5290 return Act->takeModule();
5298 Module * VuoCompiler::readModuleFromBitcode(
string inputPath)
5300 string dir, file, ext;
5303 return readModuleFromBitcode(&inputFile);
5315 size_t inputDataBytes;
5320 Module *module = readModuleFromBitcodeData(inputData, inputDataBytes, error);
5335 Module * VuoCompiler::readModuleFromBitcodeData(
char *inputData,
size_t inputDataBytes,
string &error)
5337 __block Module *module;
5339 StringRef inputDataAsStringRef(inputData, inputDataBytes);
5340 MemoryBuffer *mb = MemoryBuffer::getMemBuffer(inputDataAsStringRef,
"",
false);
5341 module = ParseBitcodeFile(&(*mb), getGlobalContext(), &error);
5354 bool VuoCompiler::writeModuleToBitcode(Module *module,
string outputPath)
5356 if (verifyModule(*module, PrintMessageAction))
5358 VUserLog(
"Error: Module verification failed.");
5363 raw_fd_ostream out(outputPath.c_str(), err);
5366 VUserLog(
"Error: Couldn't open file '%s' for writing: %s", outputPath.c_str(), err.c_str());
5369 WriteBitcodeToFile(module, out);
5379 void VuoCompiler::setTargetForModule(Module *module,
string target)
5406 module->setTargetTriple(
"x86_64-apple-macosx10.10.0");
5417 Module *llvmModule = module->
getModule();
5446 delete baseNodeClass;
5479 if (nodeClassForNode)
5480 return nodeClassForNode->
newNode(title, x, y);
5494 if (nodeClassForNode)
5495 return nodeClassForNode->
newNode(nodeToCopyMetadataFrom);
5506 return createPublishedNode(nodeClassName, publishedInputPorts);
5515 return createPublishedNode(nodeClassName, publishedOutputPorts);
5521 VuoNode * VuoCompiler::createPublishedNode(
const string &nodeClassName,
const vector<VuoPublishedPort *> &publishedPorts)
5529 for (
size_t i = 0; i < publishedPorts.size(); ++i)
5531 VuoType *publishedPortType =
static_cast<VuoCompilerPort *
>(publishedPorts[i]->getCompiler())->getDataVuoType();
5535 set<VuoPort *> nodePorts;
5539 nodePorts.insert(inputPort);
5542 nodePorts.insert(outputPort);
5547 nodePorts.insert(inputPort);
5550 for (
VuoPort *port : nodePorts)
5557 reifyGenericPortTypes(node);
5568 dispatch_sync(environmentQueue, ^{
5569 if (environments.size() >= 5)
5570 environments.at(3).at(0)->addExpatriateSourceFile(sourcePath);
5580 dispatch_sync(environmentQueue, ^{
5581 if (environments.size() >= 5)
5583 environments.at(3).at(0)->removeExpatriateSourceFile(sourcePath);
5585 set<string> sourcesRemoved;
5586 sourcesRemoved.insert(getModuleKeyForPath(sourcePath));
5587 loadModulesAndSources(set<string>(), set<string>(), set<string>(), set<string>(), set<string>(), sourcesRemoved,
5588 false, false, environments.at(3).at(0), nullptr, nullptr,
"");
5611 string sourcePathCopy = sourcePath;
5612 string sourceCodeCopy = sourceCode;
5614 dispatch_async(environmentQueue, ^{
5616 ModuleInfo *sourceInfo = NULL;
5618 for (
const vector<Environment *> &envs : environments)
5620 for (Environment *env : envs)
5622 ModuleInfo *potentialSourceInfo = env->listSourceFile(nodeClassName);
5625 sourceInfo = potentialSourceInfo;
5634 bool shouldRecompileSourcesIfUnchanged;
5635 if (! sourceCodeCopy.empty())
5637 sourceInfo->setSourceCode(sourceCodeCopy);
5638 sourceInfo->setSourceCodeOverridden(
true);
5640 shouldRecompileSourcesIfUnchanged =
false;
5644 sourceInfo->revertSourceCode();
5645 sourceInfo->setSourceCodeOverridden(
false);
5647 shouldRecompileSourcesIfUnchanged =
true;
5649 sourceInfo->setAttempted(
false);
5650 sourceInfo->setLastModifiedToNow();
5652 set<string> sourcesModified;
5653 sourcesModified.insert(nodeClassName);
5655 loadModulesAndSources(set<string>(), set<string>(), set<string>(), set<string>(), sourcesModified, set<string>(),
5656 false, shouldRecompileSourcesIfUnchanged,
nullptr,
nullptr,
nullptr,
"");
5695 set<string> nodeClassNameSet;
5696 nodeClassNameSet.insert(nodeClassName);
5697 loadModulesIfNeeded(nodeClassNameSet);
5702 void (^envGetNodeClass)(Environment *) = ^
void (Environment *env) {
5707 applyToAllEnvironments(envGetNodeClass);
5718 loadModulesIfNeeded();
5720 __block map<string, VuoCompilerNodeClass *> nodeClasses;
5721 void (^envGetNodeClasses)(Environment *) = ^
void (Environment *env) {
5722 map<string, VuoCompilerNodeClass *> result = env->getNodeClasses();
5723 nodeClasses.insert(result.begin(), result.end());
5725 applyToInstalledEnvironments(envGetNodeClasses);
5736 set<string> typeNameSet;
5737 typeNameSet.insert(typeName);
5738 loadModulesIfNeeded(typeNameSet);
5741 void (^envGetType)(Environment *) = ^
void (Environment *env) {
5746 applyToInstalledEnvironments(envGetType);
5768 loadModulesIfNeeded();
5770 __block map<string, VuoCompilerType *> types;
5771 void (^envGetTypes)(Environment *) = ^
void (Environment *env) {
5772 map<string, VuoCompilerType *> result = env->getTypes();
5773 types.insert(result.begin(), result.end());
5775 applyToInstalledEnvironments(envGetTypes);
5786 loadModulesIfNeeded();
5788 __block map<string, VuoCompilerModule *> libraryModules;
5789 void (^envGetLibraryModules)(Environment *) = ^
void (Environment *env) {
5790 map<string, VuoCompilerModule *> result = env->getLibraryModules();
5791 libraryModules.insert(result.begin(), result.end());
5793 applyToInstalledEnvironments(envGetLibraryModules);
5794 return libraryModules;
5804 loadModulesIfNeeded();
5806 __block map<string, VuoNodeSet *> nodeSets;
5807 void (^envGetNodeSets)(Environment *) = ^
void (Environment *env) {
5808 map<string, VuoNodeSet *> result = env->getNodeSets();
5809 nodeSets.insert(result.begin(), result.end());
5811 applyToInstalledEnvironments(envGetNodeSets);
5822 loadModulesIfNeeded();
5825 void (^envFindNodeSet)(Environment *) = ^
void (Environment *env) {
5830 applyToInstalledEnvironments(envFindNodeSet);
5840 void (^envFindModule)(Environment *) = ^
void (Environment *env) {
5845 applyToAllEnvironments(envFindModule);
5862 map<string, VuoCompilerNodeClass *> nodeClasses =
getNodeClasses();
5863 for (map<string, VuoCompilerNodeClass *>::const_iterator i = nodeClasses.begin(); i != nodeClasses.end(); ++i)
5870 else if (format ==
"path")
5874 else if (format ==
"dot")
5892 vector<string> VuoCompiler::getCoreVuoDependencies(
void)
5894 vector<string> dependencies;
5896 dependencies.push_back(
"VuoHeap");
5897 dependencies.push_back(
"VuoApp");
5899 dependencies.push_back(
"zmq");
5900 dependencies.push_back(
"json-c");
5901 dependencies.push_back(
"objc");
5902 dependencies.push_back(
"c");
5903 dependencies.push_back(
"AppKit.framework");
5906 dependencies.push_back(LLVM_ROOT
"/lib/libprofile_rt.dylib");
5908 return dependencies;
5914 string VuoCompiler::getRuntimeMainDependency(
void)
5916 return "VuoRuntimeMain.o";
5926 string VuoCompiler::getRuntimeDependency(
void)
5928 return "VuoRuntime.o";
5937 string VuoCompiler::getVuoFrameworkPath(
void)
5939 if (! vuoFrameworkInProgressPath.empty())
5940 return vuoFrameworkInProgressPath;
5948 llvm::sys::Path VuoCompiler::getClangPath(
void)
5959 string dir, moduleKey, ext;
5967 while (nodeClassNameParts.size() > 1 && nodeClassNameParts.back() == ext)
5968 nodeClassNameParts.pop_back();
5971 for (
string &part : nodeClassNameParts)
5975 if (nodeClassNameParts.size() == 1)
5976 nodeClassNameParts.insert(nodeClassNameParts.begin(),
"isf");
5990 __block
bool isLocal =
false;
5992 dispatch_sync(environmentQueue, ^{
5993 if (environments.size() >= 5)
5994 isLocal = environments.at(3).at(0)->findModule(moduleKey);
6008 return lastCompositionBaseDir.empty() ?
"" : lastCompositionBaseDir +
"/Modules";
6023 return lastCompositionBaseDir;
6029 void VuoCompiler::addModuleSearchPath(
string path)
6031 dispatch_sync(environmentQueue, ^{
6032 environments.back().at(0)->addModuleSearchPath(path);
6042 dispatch_sync(environmentQueue, ^{
6052 dispatch_sync(environmentQueue, ^{
6062 dispatch_sync(environmentQueue, ^{
6072 this->telemetry = telemetry;
6080 this->target = target;
6088 this->isVerbose = isVerbose;
6098 if (VuoPro::getProAccess())
6100 _shouldShowSplashWindow =
false;
6105 _shouldShowSplashWindow = potentiallyShow;
6113 return _shouldShowSplashWindow;
6119 void VuoCompiler::setClangPath(
const string &clangPath)
6121 this->clangPath = llvm::sys::Path(StringRef(clangPath));
6129 string vuoFrameworkPath = getVuoFrameworkPath();
6130 return (vuoFrameworkPath.empty() ?
6131 VUO_BUILD_DIR
"/bin/VuoCompositionLoader.app/Contents/MacOS/VuoCompositionLoader" :
6132 vuoFrameworkPath +
"/Helpers/VuoCompositionLoader.app/Contents/MacOS/VuoCompositionLoader");
6138 string VuoCompiler::getCompositionStubPath(
void)
6140 string vuoFrameworkPath = getVuoFrameworkPath();
6141 return (vuoFrameworkPath.empty() ?
6142 VUO_BUILD_DIR
"/lib/libVuoCompositionStub.dylib" :
6143 vuoFrameworkPath +
"/Modules/libVuoCompositionStub.dylib");
6149 string VuoCompiler::getCachePathForComposition(
const string compositionDir)
6151 string modifierLetterColon(
"꞉");
6152 string cachedModulesName = compositionDir;
6162 __block vector<string> moduleSearchPaths;
6163 void (^envGetModuleSearchPaths)(Environment *) = ^
void (Environment *env) {
6164 vector<string> result = env->getModuleSearchPaths();
6165 moduleSearchPaths.insert(moduleSearchPaths.end(), result.begin(), result.end());
6167 applyToInstalledEnvironments(envGetModuleSearchPaths);
6169 __block vector<string> headerSearchPaths;
6170 void (^envGetHeaderSearchPaths)(Environment *) = ^
void (Environment *env) {
6171 vector<string> result = env->getHeaderSearchPaths();
6172 headerSearchPaths.insert(headerSearchPaths.end(), result.begin(), result.end());
6174 applyToInstalledEnvironments(envGetHeaderSearchPaths);
6176 __block vector<string> librarySearchPaths;
6177 void (^envGetLibrarySearchPaths)(Environment *) = ^
void (Environment *env) {
6178 vector<string> result = env->getLibrarySearchPaths();
6179 librarySearchPaths.insert(librarySearchPaths.end(), result.begin(), result.end());
6181 applyToInstalledEnvironments(envGetLibrarySearchPaths);
6183 __block vector<string> frameworkSearchPaths;
6184 void (^envGetFrameworkSearchPaths)(Environment *) = ^
void (Environment *env) {
6185 vector<string> result = env->getFrameworkSearchPaths();
6186 frameworkSearchPaths.insert(frameworkSearchPaths.end(), result.begin(), result.end());
6188 applyToInstalledEnvironments(envGetFrameworkSearchPaths);
6190 fprintf(stderr,
"Module (node class, type, library) search paths:\n");
6191 for (vector<string>::iterator i = moduleSearchPaths.begin(); i != moduleSearchPaths.end(); ++i)
6192 fprintf(stderr,
" %s\n", (*i).c_str());
6193 fprintf(stderr,
"Header search paths:\n");
6194 for (vector<string>::iterator i = headerSearchPaths.begin(); i != headerSearchPaths.end(); ++i)
6195 fprintf(stderr,
" %s\n", (*i).c_str());
6196 fprintf(stderr,
"Other library search paths:\n");
6197 for (vector<string>::iterator i = librarySearchPaths.begin(); i != librarySearchPaths.end(); ++i)
6198 fprintf(stderr,
" %s\n", (*i).c_str());
6199 fprintf(stderr,
"Other framework search paths:\n");
6200 for (vector<string>::iterator i = frameworkSearchPaths.begin(); i != frameworkSearchPaths.end(); ++i)
6201 fprintf(stderr,
" %s\n", (*i).c_str());
6202 fprintf(stderr,
"Framework path:\n");
6203 if (! getVuoFrameworkPath().empty())
6204 fprintf(stderr,
" %s\n", getVuoFrameworkPath().c_str());
6205 fprintf(stderr,
"Clang path:\n");
6206 if (! getClangPath().str().empty())
6207 fprintf(stderr,
" %s\n", getClangPath().c_str());
6223 string directory, file, extension;
6227 compiler.
compileComposition(compositionFilePath, compiledCompositionPath,
true, issues);
6229 remove(compiledCompositionPath.c_str());
6258 string compositionPath = (workingDirectory.empty() ?
"." : workingDirectory) +
"/" + processName +
".vuo";
6264 remove(compiledCompositionPath.c_str());
6288 string directory, file, extension;
6291 compiler.
compileComposition(compositionFilePath, compiledCompositionPath,
true, issues);
6294 remove(compiledCompositionPath.c_str());
6323 string compositionPath = (workingDirectory.empty() ?
"." : workingDirectory) +
"/UntitledComposition.vuo";
6329 remove(compiledCompositionPath.c_str());
6342 pendingDataQueue = dispatch_queue_create(
"org.vuo.compiler.delegate.pending", 0);
6347 LoadedModulesData *data = dequeueData();
6354 void VuoCompilerDelegate::enqueueData(LoadedModulesData *data)
6356 dispatch_sync(pendingDataQueue, ^{
6357 pendingData.push_back(data);
6364 VuoCompilerDelegate::LoadedModulesData * VuoCompilerDelegate::dequeueData(
void)
6366 __block LoadedModulesData *ret;
6367 dispatch_sync(pendingDataQueue, ^{
6368 ret = pendingData.front();
6369 pendingData.pop_front();
6378 VuoCompilerDelegate::LoadedModulesData::LoadedModulesData(
const set< pair<VuoCompilerModule *, VuoCompilerModule *> > &modulesModified,
6381 referenceCountQueue = dispatch_queue_create(
"org.vuo.compiler.delegate.reference", 0);
6384 this->modulesModified = modulesModified;
6385 this->modulesRemoved = modulesRemoved;
6386 this->issues = issues;
6392 VuoCompilerDelegate::LoadedModulesData::~LoadedModulesData(
void)
6396 for (set< pair<VuoCompilerModule *, VuoCompilerModule *> >::iterator i = modulesModified.begin(); i != modulesModified.end(); ++i)
6399 for (set<VuoCompilerModule *>::iterator i = modulesRemoved.begin(); i != modulesRemoved.end(); ++i)
6406 void VuoCompilerDelegate::LoadedModulesData::retain(
void)
6408 dispatch_sync(referenceCountQueue, ^{
6416 void VuoCompilerDelegate::LoadedModulesData::release(
void)
6418 dispatch_sync(referenceCountQueue, ^{
6420 if (referenceCount == 0) {