VuoCompiler.hh

Go to the documentation of this file.

 
#pragma once
 
#include "VuoFileUtilities.hh"
#include "VuoFileWatcher.hh"
#include "VuoDirectedAcyclicGraph.hh"
 
class VuoCompilerComposition;
class VuoCompilerDelegate;
class VuoCompilerGenericType;
class VuoCompilerIssues;
class VuoCompilerModule;
class VuoCompilerNodeClass;
class VuoCompilerType;
class VuoModuleCompilationQueue;
class VuoNode;
class VuoNodeSet;
class VuoPublishedPort;
class VuoRunner;
class VuoRunningCompositionLibraries;
 
 
class VuoCompiler
{
public:
    enum Optimization
    {
        Optimization_FastBuild,  
        Optimization_FastBuildExistingCache,  
        Optimization_SmallBinary  
    };
 
private:
    class Environment;
 
    class ModuleInfo
    {
    public:
        ModuleInfo(Environment *environment, const string &searchPath, const string &relativePath, bool isSourceFile, bool isSubcomposition);
        ModuleInfo(Environment *environment, const string &searchPath, VuoFileUtilities::File *file, bool isSourceFile, bool isSubcomposition);
        ~ModuleInfo(void);
        Environment * getEnvironment(void) const;
        string getSearchPath(void) const;
        string getModuleKey(void) const;
        VuoFileUtilities::File * getFile(void) const;
        string getSourceCode(void) const;
        void setSourceCode(const string &sourceCode);
        void revertSourceCode(void);
        bool isSourceCodeOverridden(void) const;
        void setSourceCodeOverridden(bool overridden);
        bool isNewerThan(ModuleInfo *other) const;
        bool isNewerThan(unsigned long seconds) const;
        bool isOlderThan(unsigned long seconds) const;
        void setLastModifiedToNow(void);
        set<string> getContainedNodeClasses(void) const;
        bool getAttempted(void) const;
        void setAttempted(bool attempted);
        dispatch_group_t getLoadingGroup(void) const;
        void dump() const;
 
    private:
        void initialize(Environment *environment, const string &searchPath, VuoFileUtilities::File *file, bool isSourceFile, bool isSubcomposition);
 
        Environment *environment;  
        string searchPath;   
        string moduleKey;   
        VuoFileUtilities::File *file;  
        string sourceCode;  
        bool sourceCodeOverridden;  
        unsigned long lastModified;   
        set<string> containedNodeClasses;  
        bool attempted;  
        dispatch_group_t loading;  
 
#ifdef VUO_PRO
#include "pro/VuoCompilerModuleInfo_Pro.hh"
#endif
    };
 
    class DependencyGraphVertex : public VuoDirectedAcyclicGraph::Vertex
    {
    public:
        static DependencyGraphVertex * vertexForDependency(const string &dependency, VuoDirectedAcyclicGraph *graph);
        string getDependency(void);
        Environment * getEnvironment(void);
        void setEnvironment(Environment *environment);
        bool isCompatible(void);
        void setCompatible(bool compatible);
        string key(void);
 
    private:
        string dependency;
        Environment *environment;
        bool compatible;
    };
 
    class ModuleInfoIterator
    {
    public:
        ModuleInfoIterator(map<string, map<string, ModuleInfo *> > *allModuleInfos);
        ModuleInfoIterator(map<string, map<string, ModuleInfo *> > *allModuleInfos, const set<string> &searchModuleKeys);
        ModuleInfo * next(void);
 
    private:
        void initialize(void);
 
        map<string, map<string, ModuleInfo *> > *allModuleInfos;
        set<string> searchModuleKeys;
        bool hasSearchModuleKeys;
        map<string, map<string, ModuleInfo *> >::iterator currSearchPath;
        map<string, ModuleInfo *>::iterator currModuleKey;
        bool hasCurrModuleKey;
    };
 
    class Environment : public VuoFileWatcherDelegate
    {
    private:
        set<VuoCompiler *> compilersToNotify;  
        dispatch_queue_t compilersToNotifyQueue;  
        set<VuoFileWatcher *> moduleSearchPathWatchers;  
        map<string, VuoCompilerNodeClass *> nodeClasses;  
        map<string, VuoCompilerType *> types;  
        set<VuoCompilerGenericType *> genericTypes;  
        map<string, VuoNodeSet *> nodeSetForName;  
        map<string, VuoCompilerModule *> libraryModules;  
        map<string, dispatch_group_t> specializedModulesLoading;  
        map<string, map<string, ModuleInfo *> > moduleFilesAtSearchPath;  
        map<string, map<string, ModuleInfo *> > sourceFilesAtSearchPath;  
        vector<string> moduleSearchPaths;  
        vector<string> headerSearchPaths;  
        vector<string> librarySearchPaths;  
        vector<string> frameworkSearchPaths;  
        vector<string> expatriateSourceFiles;  
        string moduleCachePath;  
        string moduleCacheSuffix;  
        bool isModuleCacheableDataDirty;  
        bool isModuleCacheInitialized;  
        bool isModuleCacheAvailable;  
        set<string> moduleCacheContents;  
        set<string> moduleCacheDylibs;  
        set<string> moduleCacheFrameworks;  
        static map<string, VuoFileUtilities::File *> moduleCacheFileForLocking;  
        static dispatch_queue_t moduleCacheBuildingQueue;  
        VuoDirectedAcyclicGraph *dependencyGraph;  
        VuoDirectedAcyclicGraph *compositionDependencyGraph;  
        VuoModuleCompilationQueue *moduleCompilationQueue;  
 
        void updateModulesAtSearchPath(const string &path, bool shouldCleanModuleCache=true);
        void updateSourceFilesAtSearchPath(const string &path);
        void startWatchingModuleSearchPath(const string &moduleSearchPath);
        VuoCompilerModule * loadModule(ModuleInfo *moduleInfo);
 
        friend class TestCompilingAndLinking;
 
    public:
        dispatch_queue_t moduleSearchPathContentsChangedQueue;  
        static const string pidCacheDirPrefix;  
 
        Environment(void);
        virtual ~Environment(void);
        void addCompilerToNotify(VuoCompiler *compiler);
        void removeCompilerToNotify(VuoCompiler *compiler);
        map<string, VuoCompilerNodeClass *> getNodeClasses(void);
        VuoCompilerNodeClass * getNodeClass(const string &moduleKey);
        VuoCompilerNodeClass * takeNodeClass(const string &moduleKey);
        map<string, VuoCompilerType *> getTypes(void);
        VuoCompilerType * getType(const string &moduleKey);
        map<string, VuoNodeSet *> getNodeSets();
        map<string, VuoCompilerModule *> getLibraryModules(void);
        VuoCompilerModule * findModule(const string &moduleKey);
        VuoNodeSet * findNodeSet(const string &name);
        void addModuleSearchPath(const string &path, bool shouldWatch = true);
        vector<string> getModuleSearchPaths(void);
        void addHeaderSearchPath(const string &path);
        vector<string> getHeaderSearchPaths(void);
        void addLibrarySearchPath(const string &path);
        vector<string> getLibrarySearchPaths(void);
        void addFrameworkSearchPath(const string &path);
        void setModuleCachePath(const string &path);
        string getCompiledModuleCachePath(void);
        string getOverriddenCompiledModuleCachePath(void);
        vector<string> getFrameworkSearchPaths(void);
        VuoDirectedAcyclicGraph * getDependencyGraph(void);
        VuoDirectedAcyclicGraph * getCompositionDependencyGraph(void);
        void addExpatriateSourceFile(const string &sourcePath);
        void removeExpatriateSourceFile(const string &sourcePath);
        ModuleInfo * listModule(const string &moduleKey);
        ModuleInfoIterator listModules(const set<string> &moduleKeys);
        ModuleInfoIterator listAllModules(void);
        ModuleInfo * listSourceFile(const string &moduleKey);
        ModuleInfoIterator listSourceFiles(const set<string> &moduleKeys);
        ModuleInfoIterator listAllSourceFiles(void);
        static vector<string> getBuiltInModuleSearchPaths(void);
        static vector<string> getBuiltInHeaderSearchPaths(void);
        static vector<string> getBuiltInLibrarySearchPaths(void);
        static vector<string> getBuiltInFrameworkSearchPaths(void);
        void addSearchPathsToSharedEnvironment(void);
        void stopWatchingModuleSearchPaths(void);
        void fileChanged(const string &moduleSearchPath);
        void moduleSearchPathContentsChanged(const string &moduleSearchPath, const string &moduleAddedOrModifiedPath = "", const string &moduleAddedOrModifiedSourceCode = "", std::function<void(void)> moduleLoadedCallback = nullptr, VuoCompiler *compiler = nullptr, VuoCompilerIssues *issues = nullptr);
        void notifyCompilers(const set<VuoCompilerModule *> &modulesAdded, const set<pair<VuoCompilerModule *, VuoCompilerModule *> > &modulesModified, const set<VuoCompilerModule *> &modulesRemoved, VuoCompilerIssues *issues, bool oldModulesInvalidated = true);
        set<VuoCompilerModule *> loadCompiledModules(const set<string> &moduleKeys, const map<string, string> &sourceCodeForModule);
        set<dispatch_group_t> loadSpecializedModules(const set<string> &moduleKeys, VuoCompiler *compiler, dispatch_queue_t llvmQueue);
        set<dispatch_group_t> compileModulesFromSourceCode(const set<string> &moduleKeys, bool shouldRecompileIfUnchanged);
        set<VuoCompilerModule *> unloadCompiledModules(const set<string> &moduleKeys);
        void deleteModulesCompiledFromSourceCode(const set<string> &moduleKeys);
        void replaceNodeClass(VuoCompilerNodeClass *newNodeClass);
        void addToDependencyGraph(VuoCompilerModule *module);
        void removeFromDependencyGraph(VuoCompilerModule *module);
        void reifyPortTypes(const map<string, VuoCompilerType *> &inheritedTypes);
        void getCacheableModulesAndDependencies(bool isBuiltIn, bool installed, set<string> &cacheableModulesAndDependencies, set<string> &dylibsNeededToLinkToThisCache, set<string> &frameworksNeededToLinkToThisCache);
        void useModuleCache(bool shouldUseExistingCache, VuoCompiler *compiler, set<string> cacheableModulesAndDependencies, set<string> dylibsNeededToLinkToCaches, set<string> frameworksNeededToLinkToCaches);
        static void waitForModuleCachesToBuild(void);
        bool findInModuleCache(const string &moduleOrDependency, string &cachePath);
        void modulesChanged(void);
 
        bool isBuiltIn();
        string getName();
 
#ifdef VUO_PRO
#include "pro/VuoCompilerEnvironment_Pro.hh"
#endif
    };
 
    static dispatch_queue_t environmentQueue;  
    static vector< vector<Environment *> > sharedEnvironments;  
    static map<string, vector<Environment *> > environmentsForCompositionFamily;  
    vector< vector<Environment *> > environments;  
    string lastCompositionBaseDir;  
    bool lastCompositionIsSubcomposition;  
    bool shouldLoadAllModules;  
    bool hasLoadedAllModules;  
    dispatch_queue_t modulesToLoadQueue;  
    static dispatch_group_t moduleSourceCompilersExist;  
    dispatch_group_t moduleCacheBuilding;  
    VuoDirectedAcyclicNetwork *dependencyGraph;  
    VuoDirectedAcyclicNetwork *compositionDependencyGraph;  
    static string vuoFrameworkInProgressPath;  
    llvm::sys::Path clangPath;
    string telemetry;
    string target;
    bool isVerbose;
    bool _shouldShowSplashWindow;
    VuoCompilerDelegate *delegate;
    dispatch_queue_t delegateQueue;  
 
    void applyToInstalledEnvironments(void (^doForEnvironment)(Environment *));
    void applyToInstalledEnvironments(void (^doForEnvironment)(Environment *, bool *, string), bool *, string);
    void applyToAllEnvironments(void (^doForEnvironment)(Environment *));
 
    static void reset(void);
    VuoDirectedAcyclicNetwork * makeDependencyNetwork(const vector< vector<Environment *> > &environments, VuoDirectedAcyclicGraph * (^graphForEnvironment)(Environment *));
    void loadModulesIfNeeded(const set<string> &moduleKeys = set<string>());
    set<dispatch_group_t> loadModulesAndSources(const set<string> &modulesAddedKeys, const set<string> &modulesModifiedKeys, const set<string> &modulesRemovedKeys, const set<string> &sourcesAddedKeys, const set<string> &sourcesModifiedKeys, const set<string> &sourcesRemovedKeys, bool willLoadAllModules, bool shouldRecompileSourcesIfUnchanged, Environment *currentEnvironment, VuoCompilerIssues *issuesForCurrentEnvironment, std::function<void(void)> moduleLoadedCallback, const string &moduleAddedOrModifiedSourceCode);
    void findDependentModulesAndSources(map<Environment *, set<string> > &changedModules, const vector<VuoDirectedAcyclicNetwork *> &searchDependencyGraphs, VuoDirectedAcyclicGraph *currentEnvironmentDependencyGraph, map<Environment *, set<string> > &modulesDepOnChangedModules_this, map<Environment *, set<string> > &modulesDepOnChangedModules_other, map<Environment *, set<string> > &sourcesDepOnChangedModules_this, map<Environment *, set<string> > &sourcesDepOnChangedModules_other);
    void loadedModules(map<string, VuoCompilerModule *> modulesAdded, map<string, pair<VuoCompilerModule *, VuoCompilerModule *> > modulesModified, map<string, VuoCompilerModule *> modulesRemoved, VuoCompilerIssues *issues, void *delegateData, Environment *currentEnvironment);
    void loadNodeClassGeneratedAtRuntime(VuoCompilerNodeClass *nodeClass, Environment *env);
    void reifyGenericPortTypes(VuoCompilerComposition *composition);
    void reifyGenericPortTypes(VuoNode *node);
    Module * compileCompositionToModule(VuoCompilerComposition *composition, const string &moduleKey, bool isTopLevelComposition, VuoCompilerIssues *issues);
    void compileSubcompositionString(const string &compositionString, const string &outputPath, std::function<void(void)> moduleLoadedCallback, Environment *environment, VuoCompilerIssues *issues = NULL, const string inputPathForIssues = "");
    void linkCompositionToCreateExecutableOrDynamicLibrary(string compiledCompositionPath, string linkedCompositionPath, Optimization optimization, bool isDylib, string rPath="");
    set<string> getDependenciesForComposition(const string &compiledCompositionPath);
    set<string> getDependenciesForComposition(const set<string> &directDependencies, bool checkCompatibility);
    void getLinkerInputs(const set<string> &dependencies, Optimization optimization, set<Module *> &modules, set<string> &libraries, set<string> &frameworks);
    void getLinkerInputs(const set<string> &dependencies, Optimization optimization, set<string> &builtInModuleAndLibraryDependencies, set<string> &userModuleAndLibraryDependencies, map<string, set<string> > &builtInCacheDependencies, map<string, set<string> > &userCacheDependencies, set<Module *> &builtInModules, set<Module *> &userModules, set<string> &builtInLibraries, set<string> &userLibraries, set<string> &externalLibraries, set<string> &externalFrameworks);
    static string getLibraryPath(const string &dependency, vector<string> librarySearchPaths);
    void useModuleCache(bool shouldUseExistingBuiltInCaches, bool shouldUseExistingOtherCaches);
    bool findInModuleCache(const string &moduleOrDependency, string &cachePath, bool &isBuiltinCache) VuoWarnUnusedResult;
    void link(string outputPath, const set<Module *> &modules, const set<string> &libraries, const set<string> &frameworks, bool isDylib, string rPath="");
    Module * readModuleFromC(string inputPath, const vector<string> &headerSearchPaths, const vector<string> &extraArgs);
    static Module * readModuleFromBitcode(VuoFileUtilities::File *inputFile);
    static Module * readModuleFromBitcodeData(char *inputData, size_t inputDataBytes, string &error);
    static bool writeModuleToBitcode(Module *module, string outputPath);
    VuoNode * createPublishedNode(const string &nodeClassName, const vector<VuoPublishedPort *> &publishedPorts);
    static void setTargetForModule(Module *module, string target = "");
    VuoCompilerModule * getModule(const string &moduleKey);
    static vector<string> getCoreVuoDependencies(void);
    static string getRuntimeMainDependency(void);
    static string getRuntimeDependency(void);
    void addModuleSearchPath(string path);
    static string getVuoFrameworkPath(void);
    llvm::sys::Path getClangPath(void);
    void setClangPath(const string &clangPath);
    string getCompositionStubPath(void);
    static string getCachePathForComposition(const string compositionDir);
 
    friend class TestVuoCompiler;       
    friend class TestCompositionExecution;  
    friend class TestCompilerDelegate;
    friend class TestVuoCompilerModule;
    friend class TestVuoCompilerBitcodeGenerator;
    friend class TestCompilingAndLinking;  
    friend class TestSubcompositions;  
    friend class TestModuleLoading;  
    friend class TestEventDropping;  
    friend class TestVuoIsfModuleCompiler;  
 
public:
    VuoCompiler(const string &compositionPath = "");
    ~VuoCompiler(void);
    void setDelegate(VuoCompilerDelegate *delegate);
    void setCompositionPath(const string &compositionPath);
    static Module * readModuleFromBitcode(string inputPath);
    static void destroyModule(VuoCompilerModule *module);
    static void destroyLlvmModule(Module *module);
    void compileComposition(VuoCompilerComposition *composition, string outputPath, bool isTopLevelComposition, VuoCompilerIssues *issues);
    void compileComposition(string inputPath, string outputPath, bool isTopLevelComposition, VuoCompilerIssues *issues);
    void compileCompositionString(const string &compositionString, string outputPath, bool isTopLevelComposition, VuoCompilerIssues *issues);
    void compileModule(string inputPath, string outputPath);
    void compileModule(string inputPath, string outputPath, const vector<string> &includeDirs);
    void linkCompositionToCreateExecutable(string inputPath, string outputPath, Optimization optimization, string rPath="");
    void linkCompositionToCreateDynamicLibrary(string inputPath, string outputPath, Optimization optimization);
    void linkCompositionToCreateDynamicLibraries(string compiledCompositionPath, string linkedCompositionPath, VuoRunningCompositionLibraries *runningCompositionLibraries);
    set<string> getDirectDependenciesForComposition(VuoCompilerComposition *composition);
    set<string> getDependenciesForComposition(VuoCompilerComposition *composition);
    set<string> getDylibDependencyPathsForComposition(VuoCompilerComposition *composition);
    void prepareForFastBuild(void);
    static void generateBuiltInModuleCaches(const string &vuoFrameworkPath);
    static void deleteOldModuleCaches(void);
    void setLoadAllModules(bool shouldLoadAllModules);
    VuoNode * createNode(VuoCompilerNodeClass *nodeClass, string title="", double x=0, double y=0);
    VuoNode * createPublishedInputNode(vector<VuoPublishedPort *> publishedInputPorts);
    VuoNode * createPublishedOutputNode(vector<VuoPublishedPort *> publishedOutputPorts);
    void installNodeClassAtCompositionLocalScope(const string &sourcePath);
    void uninstallNodeClassAtCompositionLocalScope(const string &sourcePath);
    void overrideInstalledNodeClass(const string &sourcePath, const string &sourceCode);
    void revertOverriddenNodeClass(const string &sourcePath);
    VuoCompilerNodeClass * getNodeClass(const string &nodeClassName);
    map<string, VuoCompilerNodeClass *> getNodeClasses(void);
    VuoCompilerType * getType(const string &typeName);
    map<string, VuoCompilerType *> getTypes(void);
    map<string, VuoCompilerModule *> getLibraryModules();
    map<string, VuoNodeSet *> getNodeSets();
    VuoNodeSet * getNodeSetForName(const string &name);
    void listNodeClasses(const string &format = "");
    static string getModuleKeyForPath(string path);
    bool isCompositionLocalModule(string moduleKey);
    string getCompositionLocalModulesPath(void);
    string getCompositionLocalPath(void);
    void addHeaderSearchPath(const string &path);
    void addLibrarySearchPath(const string &path);
    void addFrameworkSearchPath(const string &path);
    void setTelemetry(const string &telemetry);
    void setTarget(const string &target);
    void setVerbose(bool isVerbose);
    void setShouldPotentiallyShowSplashWindow(bool potentiallyShow);
    bool shouldShowSplashWindow();
    string getCompositionLoaderPath(void);
    void print(void);
 
    static VuoRunner * newSeparateProcessRunnerFromCompositionFile(string compositionFilePath, VuoCompilerIssues *issues);
    static VuoRunner * newSeparateProcessRunnerFromCompositionString(string composition, string processName, string workingDirectory, VuoCompilerIssues *issues);
    static VuoRunner * newCurrentProcessRunnerFromCompositionFile(string compositionFilePath, VuoCompilerIssues *issues);
    static VuoRunner * newCurrentProcessRunnerFromCompositionString(string composition, string workingDirectory, VuoCompilerIssues *issues);
 
#ifdef VUO_PRO
#include "pro/VuoCompiler_Pro.hh"
#endif
};
 
class VuoCompilerDelegate
{
public:
    VuoCompilerDelegate(void);
 
    virtual void loadedModules(const map<string, VuoCompilerModule *> &modulesAdded,
                               const map<string, pair<VuoCompilerModule *, VuoCompilerModule *> > &modulesModified,
                               const map<string, VuoCompilerModule *> &modulesRemoved, VuoCompilerIssues *issues) = 0;
 
    void loadedModulesCompleted(void);
 
private:
    class LoadedModulesData
    {
    public:
        LoadedModulesData(const set< pair<VuoCompilerModule *, VuoCompilerModule *> > &modulesModified,
                          const set<VuoCompilerModule *> &modulesRemoved, VuoCompilerIssues *issues);
        ~LoadedModulesData(void);
        void retain(void);
        void release(void);
 
    private:
        int referenceCount;
        dispatch_queue_t referenceCountQueue;
        set< pair<VuoCompilerModule *, VuoCompilerModule *> > modulesModified;
        set<VuoCompilerModule *> modulesRemoved;
        VuoCompilerIssues *issues;
    };
 
    void enqueueData(LoadedModulesData *data);
    LoadedModulesData * dequeueData(void);
 
    list<LoadedModulesData *> pendingData;
    dispatch_queue_t pendingDataQueue;
 
    friend class VuoCompiler;
};