Shader: Add module resolver + use modules for engine shaders

examples/RenderTest/main.cpp

@@ -140,22 +140,12 @@ int main()
 		return __LINE__;
 	}
 
+	auto directoryModuleResolver = std::make_shared<Nz::DirectoryModuleResolver>();
+	directoryModuleResolver->RegisterModuleFile("Test/Bar", barModuleSource, sizeof(barModuleSource));
+	directoryModuleResolver->RegisterModuleFile("Test/Data", dataModuleSource, sizeof(dataModuleSource));
+
 	Nz::ShaderAst::SanitizeVisitor::Options sanitizeOpt;
-	sanitizeOpt.moduleCallback = [](const std::vector<std::string>& modulePath) -> Nz::ShaderAst::ModulePtr
+	sanitizeOpt.moduleResolver = directoryModuleResolver;
-	{
-		if (modulePath.size() != 2)
-			return {};
-
-		if (modulePath[0] != "Test")
-			return {};
-
-		if (modulePath[1] == "Bar")
-			return Nz::ShaderLang::Parse(std::string_view(barModuleSource, sizeof(barModuleSource)));
-		else if (modulePath[1] == "Data")
-			return Nz::ShaderLang::Parse(std::string_view(dataModuleSource, sizeof(dataModuleSource)));
-		else
-			return {};
-	};
 
 	shaderModule = Nz::ShaderAst::Sanitize(*shaderModule, sanitizeOpt);
 	if (!shaderModule)

include/Nazara/Core.hpp

@@ -94,5 +94,6 @@
 #include <Nazara/Core/Unicode.hpp>
 #include <Nazara/Core/Updatable.hpp>
 #include <Nazara/Core/Uuid.hpp>
+#include <Nazara/Core/VirtualDirectory.hpp>
 
 #endif // NAZARA_GLOBAL_CORE_HPP

include/Nazara/Core/VirtualDirectory.hpp

@@ -0,0 +1,83 @@
+// Copyright (C) 2022 Jérôme "Lynix" Leclercq (lynix680@gmail.com)
+// This file is part of the "Nazara Engine - Core module"
+// For conditions of distribution and use, see copyright notice in Config.hpp
+
+#pragma once
+
+#ifndef NAZARA_CORE_VIRTUALDIRECTORY_HPP
+#define NAZARA_CORE_VIRTUALDIRECTORY_HPP
+
+#include <Nazara/Prerequisites.hpp>
+#include <filesystem>
+#include <map>
+#include <memory>
+#include <optional>
+#include <string>
+#include <variant>
+#include <vector>
+
+namespace Nz
+{
+	class VirtualDirectory : public std::enable_shared_from_this<VirtualDirectory>
+	{
+		public:
+			struct DataPointerEntry;
+			struct FileContentEntry;
+			using PhysicalFileEntry = std::filesystem::path;
+			using VirtualDirectoryEntry = std::shared_ptr<VirtualDirectory>;
+
+			using Entry = std::variant<DataPointerEntry, FileContentEntry, PhysicalFileEntry, VirtualDirectoryEntry>;
+
+			inline VirtualDirectory(VirtualDirectoryEntry parentDirectory = nullptr);
+			inline VirtualDirectory(std::filesystem::path physicalPath, VirtualDirectoryEntry parentDirectory = nullptr);
+			VirtualDirectory(const VirtualDirectory&) = delete;
+			VirtualDirectory(VirtualDirectory&&) = delete;
+			~VirtualDirectory() = default;
+
+			template<typename F> void Foreach(F&& cb, bool includeDots = false);
+
+			inline bool GetEntry(std::string_view path, Entry* entry);
+
+			inline VirtualDirectoryEntry& StoreDirectory(std::string_view path, VirtualDirectoryEntry directory);
+			inline VirtualDirectoryEntry& StoreDirectory(std::string_view path, std::filesystem::path directoryPath);
+			inline FileContentEntry& StoreFile(std::string_view path, std::vector<UInt8> file);
+			inline PhysicalFileEntry& StoreFile(std::string_view path, std::filesystem::path filePath);
+			inline DataPointerEntry& StoreFile(std::string_view path, const void* data, std::size_t size);
+
+			VirtualDirectory& operator=(const VirtualDirectory&) = delete;
+			VirtualDirectory& operator=(VirtualDirectory&&) = delete;
+
+			struct DataPointerEntry
+			{
+				const void* data;
+				std::size_t size;
+			};
+
+			struct FileContentEntry
+			{
+				std::shared_ptr<std::vector<UInt8>> data;
+			};
+
+		private:
+			inline void EnsureDots();
+			inline bool GetEntryInternal(std::string_view name, Entry* entry);
+			inline bool RetrieveDirectory(std::string_view path, bool allowCreation, std::shared_ptr<VirtualDirectory>& directory, std::string_view& entryName);
+
+			inline VirtualDirectoryEntry& StoreDirectoryInternal(std::string name, std::filesystem::path directoryPath);
+			inline VirtualDirectoryEntry& StoreDirectoryInternal(std::string name, VirtualDirectoryEntry directory);
+			inline FileContentEntry& StoreFileInternal(std::string name, std::vector<UInt8> fileContent);
+			inline PhysicalFileEntry& StoreFileInternal(std::string name, std::filesystem::path filePath);
+			inline DataPointerEntry& StoreFileInternal(std::string name, const void* data, std::size_t size);
+
+			template<typename F1, typename F2> static bool SplitPath(std::string_view path, F1&& dirCB, F2&& fileCB);
+
+			std::map<std::string /*name*/, Entry, std::less<>> m_content;
+			std::optional<std::filesystem::path> m_physicalPath;
+			VirtualDirectoryEntry m_parent;
+			bool m_wereDotRegistered;
+	};
+}
+
+#include <Nazara/Core/VirtualDirectory.inl>
+
+#endif // NAZARA_CORE_VIRTUALDIRECTORY_HPP

include/Nazara/Core/VirtualDirectory.inl

@@ -0,0 +1,266 @@
+// Copyright (C) 2022 Jérôme "Lynix" Leclercq (lynix680@gmail.com)
+// This file is part of the "Nazara Engine - Core module"
+// For conditions of distribution and use, see copyright notice in Config.hpp
+
+#include <Nazara/Core/VirtualDirectory.hpp>
+#include <cassert>
+#include <Nazara/Core/Debug.hpp>
+
+namespace Nz
+{
+	inline VirtualDirectory::VirtualDirectory(VirtualDirectoryEntry parentDirectory) :
+	m_parent(std::move(parentDirectory)),
+	m_wereDotRegistered(false)
+	{
+	}
+
+	inline VirtualDirectory::VirtualDirectory(std::filesystem::path physicalPath, VirtualDirectoryEntry parentDirectory) :
+	m_physicalPath(std::move(physicalPath)),
+	m_parent(std::move(parentDirectory)),
+	m_wereDotRegistered(false)
+	{
+	}
+
+	template<typename F>
+	void VirtualDirectory::Foreach(F&& cb, bool includeDots)
+	{
+		if (includeDots)
+			EnsureDots();
+
+		for (auto&& pair : m_content)
+		{
+			if (!includeDots && (pair.first == "." || pair.first == ".."))
+				continue;
+	
+			cb(pair.first, pair.second);
+		}
+
+		if (m_physicalPath)
+		{
+			for (auto&& physicalEntry : std::filesystem::directory_iterator(*m_physicalPath))
+			{
+				Entry entry;
+
+				std::string filename = physicalEntry.path().filename().generic_u8string();
+				if (m_content.find(filename) != m_content.end())
+					continue; //< Physical file/directory has been overridden by a virtual one
+
+				if (physicalEntry.is_regular_file())
+					entry.emplace<PhysicalFileEntry>(physicalEntry.path());
+				else if (physicalEntry.is_directory())
+				{
+					// FIXME: Allocating a shared_ptr on iteration is bad, not sure about a workaround
+					entry.emplace<VirtualDirectoryEntry>(std::make_shared<VirtualDirectory>(physicalEntry.path(), shared_from_this()));
+				}
+				else
+					continue;
+
+				cb(physicalEntry.path().filename().generic_u8string(), entry);
+			}
+		}
+	}
+
+	inline bool VirtualDirectory::GetEntry(std::string_view path, Entry* entry)
+	{
+		std::shared_ptr<VirtualDirectory> dir;
+		std::string_view entryName;
+		if (!RetrieveDirectory(path, false, dir, entryName))
+			return false;
+
+		if (!dir->GetEntryInternal(entryName, entry))
+			return false;
+
+		return true;
+	}
+
+	inline auto VirtualDirectory::StoreDirectory(std::string_view path, VirtualDirectoryEntry directory) -> VirtualDirectoryEntry&
+	{
+		std::shared_ptr<VirtualDirectory> dir;
+		std::string_view entryName;
+		if (!RetrieveDirectory(path, true, dir, entryName))
+			throw std::runtime_error("invalid path");
+
+		return dir->StoreDirectoryInternal(std::string(entryName), std::move(directory));
+	}
+
+	inline auto VirtualDirectory::StoreDirectory(std::string_view path, std::filesystem::path directoryPath) -> VirtualDirectoryEntry&
+	{
+		std::shared_ptr<VirtualDirectory> dir;
+		std::string_view entryName;
+		if (!RetrieveDirectory(path, true, dir, entryName))
+			throw std::runtime_error("invalid path");
+
+		return dir->StoreDirectoryInternal(std::string(entryName), std::move(directoryPath));
+	}
+
+	inline auto VirtualDirectory::StoreFile(std::string_view path, std::vector<UInt8> file) -> FileContentEntry&
+	{
+		std::shared_ptr<VirtualDirectory> dir;
+		std::string_view entryName;
+		if (!RetrieveDirectory(path, true, dir, entryName))
+			throw std::runtime_error("invalid path");
+
+		return dir->StoreFileInternal(std::string(entryName), std::move(file));
+	}
+
+	inline auto VirtualDirectory::StoreFile(std::string_view path, std::filesystem::path filePath) -> PhysicalFileEntry&
+	{
+		std::shared_ptr<VirtualDirectory> dir;
+		std::string_view entryName;
+		if (!RetrieveDirectory(path, true, dir, entryName))
+			throw std::runtime_error("invalid path");
+
+		return dir->StoreFileInternal(std::string(entryName), std::move(filePath));
+	}
+
+	inline auto VirtualDirectory::StoreFile(std::string_view path, const void* data, std::size_t size) -> DataPointerEntry&
+	{
+		std::shared_ptr<VirtualDirectory> dir;
+		std::string_view entryName;
+		if (!RetrieveDirectory(path, true, dir, entryName))
+			throw std::runtime_error("invalid path");
+
+		return dir->StoreFileInternal(std::string(entryName), data, size);
+	}
+
+	inline void VirtualDirectory::EnsureDots()
+	{
+		if (!m_wereDotRegistered)
+		{
+			StoreDirectoryInternal(".", shared_from_this());
+			if (m_parent)
+				StoreDirectoryInternal("..", m_parent);
+			else
+				StoreDirectoryInternal("..", shared_from_this());
+
+			m_wereDotRegistered = true;
+		}
+	}
+
+	inline bool VirtualDirectory::RetrieveDirectory(std::string_view path, bool allowCreation, std::shared_ptr<VirtualDirectory>& directory, std::string_view& entryName)
+	{
+		directory = shared_from_this();
+
+		return SplitPath(path, [&](std::string_view dirName)
+		{
+			Entry entry;
+			if (directory->GetEntryInternal(dirName, &entry))
+			{
+				if (auto dir = std::get_if<VirtualDirectoryEntry>(&entry))
+					directory = *dir;
+				else
+					return false;
+			}
+			else
+			{
+				if (allowCreation)
+				{
+					auto newDirectory = std::make_shared<VirtualDirectory>(directory);
+					directory = directory->StoreDirectoryInternal(std::string(dirName), newDirectory);
+				}
+				else
+					return false;
+			}
+
+			return true;
+		}, 
+		[&](std::string_view name)
+		{
+			entryName = name;
+		});
+	}
+
+	inline bool VirtualDirectory::GetEntryInternal(std::string_view name, Entry* entry)
+	{
+		EnsureDots();
+
+		auto it = m_content.find(name);
+		if (it != m_content.end())
+		{
+			*entry = it->second;
+			return true;
+		}
+		else
+		{
+			if (m_physicalPath)
+			{
+				std::filesystem::path entryPath = *m_physicalPath / name;
+
+				if (std::filesystem::is_regular_file(entryPath))
+					entry->emplace<PhysicalFileEntry>(entryPath);
+				else if (std::filesystem::is_directory(entryPath))
+				{
+					// FIXME: Allocating a shared_ptr on iteration is bad, not sure about a workaround
+					*entry = StoreDirectoryInternal(std::string(name), entryPath);
+				}
+				else
+					return false;
+
+				return true;
+			}
+
+			return false;
+		}
+	}
+
+	inline auto VirtualDirectory::StoreDirectoryInternal(std::string name, std::filesystem::path directoryPath) -> VirtualDirectoryEntry&
+	{
+		assert(name.find_first_of("\\/:") == name.npos);
+
+		auto it = m_content.insert_or_assign(std::move(name), std::make_shared<VirtualDirectory>(directoryPath, shared_from_this())).first;
+		return std::get<VirtualDirectoryEntry>(it->second);
+	}
+
+	inline auto VirtualDirectory::StoreDirectoryInternal(std::string name, VirtualDirectoryEntry directory) -> VirtualDirectoryEntry&
+	{
+		assert(name.find_first_of("\\/:") == name.npos);
+
+		auto it = m_content.insert_or_assign(std::move(name), std::move(directory)).first;
+		return std::get<VirtualDirectoryEntry>(it->second);
+	}
+
+	inline auto VirtualDirectory::StoreFileInternal(std::string name, std::vector<UInt8> fileContent) -> FileContentEntry&
+	{
+		assert(name.find_first_of("\\/:") == name.npos);
+
+		FileContentEntry fileEntry;
+		fileEntry.data = std::make_shared<std::vector<UInt8>>(std::move(fileContent));
+
+		auto it = m_content.insert_or_assign(std::move(name), std::move(fileEntry)).first;
+		return std::get<FileContentEntry>(it->second);
+	}
+
+	inline auto VirtualDirectory::StoreFileInternal(std::string name, std::filesystem::path filePath) -> PhysicalFileEntry&
+	{
+		assert(name.find_first_of("\\/:") == name.npos);
+
+		auto it = m_content.insert_or_assign(std::move(name), std::move(filePath)).first;
+		return std::get<PhysicalFileEntry>(it->second);
+	}
+
+	inline auto VirtualDirectory::StoreFileInternal(std::string name, const void* data, std::size_t size) -> DataPointerEntry&
+	{
+		assert(name.find_first_of("\\/:") == name.npos);
+
+		auto it = m_content.insert_or_assign(std::move(name), DataPointerEntry{ data, size }).first;
+		return std::get<DataPointerEntry>(it->second);
+	}
+
+	template<typename F1, typename F2>
+	inline bool VirtualDirectory::SplitPath(std::string_view path, F1&& dirCB, F2&& lastCB)
+	{
+		std::size_t pos;
+		while ((pos = path.find_first_of("\\/:")) != std::string::npos)
+		{
+			if (!dirCB(path.substr(0, pos)))
+				return false;
+
+			path = path.substr(pos + 1);
+		}
+
+		lastCB(path);
+		return true;
+	}
+}
+
+#include <Nazara/Core/DebugOff.hpp>

include/Nazara/Graphics/Graphics.hpp

@@ -15,6 +15,7 @@
 #include <Nazara/Renderer/RenderPassCache.hpp>
 #include <Nazara/Renderer/RenderPipelineLayout.hpp>
 #include <Nazara/Renderer/Renderer.hpp>
+#include <Nazara/Shader/DirectoryModuleResolver.hpp>
 #include <optional>
 
 namespace Nz
@@ -45,6 +46,7 @@ namespace Nz
 			inline const std::shared_ptr<RenderDevice>& GetRenderDevice() const;
 			inline const RenderPassCache& GetRenderPassCache() const;
 			inline TextureSamplerCache& GetSamplerCache();
+			inline const std::shared_ptr<DirectoryModuleResolver>& GetShaderModuleResolver() const;
 
 			struct Config
 			{
@@ -62,10 +64,12 @@ namespace Nz
 			void BuildDefaultTextures();
 			void BuildFullscreenVertexBuffer();
 			void RegisterMaterialPasses();
+			void RegisterShaderModules();
 			void SelectDepthStencilFormats();
 
 			std::optional<RenderPassCache> m_renderPassCache;
 			std::optional<TextureSamplerCache> m_samplerCache;
+			std::shared_ptr<DirectoryModuleResolver> m_shaderModuleResolver;
 			std::shared_ptr<RenderBuffer> m_fullscreenVertexBuffer;
 			std::shared_ptr<RenderDevice> m_renderDevice;
 			std::shared_ptr<RenderPipeline> m_blitPipeline;

include/Nazara/Graphics/Graphics.inl

@@ -62,6 +62,11 @@ namespace Nz
 	{
 		return *m_samplerCache;
 	}
+
+	inline const std::shared_ptr<DirectoryModuleResolver>& Graphics::GetShaderModuleResolver() const
+	{
+		return m_shaderModuleResolver;
+	}
 }
 
 #include <Nazara/Graphics/DebugOff.hpp>

include/Nazara/Shader.hpp

@@ -30,12 +30,14 @@
 #define NAZARA_GLOBAL_SHADER_HPP
 
 #include <Nazara/Shader/Config.hpp>
+#include <Nazara/Shader/DirectoryModuleResolver.hpp>
 #include <Nazara/Shader/GlslWriter.hpp>
 #include <Nazara/Shader/LangWriter.hpp>
 #include <Nazara/Shader/Shader.hpp>
 #include <Nazara/Shader/ShaderBuilder.hpp>
 #include <Nazara/Shader/ShaderLangLexer.hpp>
 #include <Nazara/Shader/ShaderLangParser.hpp>
+#include <Nazara/Shader/ShaderModuleResolver.hpp>
 #include <Nazara/Shader/ShaderWriter.hpp>
 #include <Nazara/Shader/SpirvAstVisitor.hpp>
 #include <Nazara/Shader/SpirvBlock.hpp>

include/Nazara/Shader/Ast/IndexRemapperVisitor.hpp

@@ -4,8 +4,8 @@
 
 #pragma once
 
-#ifndef NAZARA_SHADER_AST_INDEXREMAPPER_HPP
+#ifndef NAZARA_SHADER_AST_INDEXREMAPPERVISITOR_HPP
-#define NAZARA_SHADER_AST_INDEXREMAPPER_HPP
+#define NAZARA_SHADER_AST_INDEXREMAPPERVISITOR_HPP
 
 #include <Nazara/Prerequisites.hpp>
 #include <Nazara/Shader/Config.hpp>
@@ -60,4 +60,4 @@ namespace Nz::ShaderAst
 
 #include <Nazara/Shader/Ast/IndexRemapperVisitor.inl>
 
-#endif // NAZARA_SHADER_AST_INDEXREMAPPER_HPP
+#endif // NAZARA_SHADER_AST_INDEXREMAPPERVISITOR_HPP

include/Nazara/Shader/Ast/SanitizeVisitor.hpp

@@ -10,6 +10,7 @@
 #include <Nazara/Prerequisites.hpp>
 #include <Nazara/Core/Bitset.hpp>
 #include <Nazara/Shader/Config.hpp>
+#include <Nazara/Shader/ShaderModuleResolver.hpp>
 #include <Nazara/Shader/Ast/AstCloner.hpp>
 #include <Nazara/Shader/Ast/AstTypes.hpp>
 #include <Nazara/Shader/Ast/Module.hpp>
@@ -42,7 +43,7 @@ namespace Nz::ShaderAst
 
 			struct Options
 			{
-				std::function<ModulePtr(const std::vector<std::string>& /*modulePath*/)> moduleCallback;
+				std::shared_ptr<ShaderModuleResolver> moduleResolver;
 				std::unordered_set<std::string> reservedIdentifiers;
 				std::unordered_map<UInt32, ConstantValue> optionValues;
 				bool makeVariableNameUnique = false;

include/Nazara/Shader/DirectoryModuleResolver.hpp

@@ -0,0 +1,45 @@
+// Copyright (C) 2022 Jérôme "Lynix" Leclercq (lynix680@gmail.com)
+// This file is part of the "Nazara Engine - Shader module"
+// For conditions of distribution and use, see copyright notice in Config.hpp
+
+#pragma once
+
+#ifndef NAZARA_SHADER_DIRECTORYMODULERESOLVER_HPP
+#define NAZARA_SHADER_DIRECTORYMODULERESOLVER_HPP
+
+#include <Nazara/Prerequisites.hpp>
+#include <Nazara/Core/VirtualDirectory.hpp>
+#include <Nazara/Shader/Config.hpp>
+#include <Nazara/Shader/ShaderModuleResolver.hpp>
+#include <string>
+#include <unordered_map>
+
+namespace Nz
+{
+	class NAZARA_SHADER_API DirectoryModuleResolver : public ShaderModuleResolver
+	{
+		public:
+			inline DirectoryModuleResolver();
+			DirectoryModuleResolver(const DirectoryModuleResolver&) = delete;
+			DirectoryModuleResolver(DirectoryModuleResolver&&) = delete;
+			~DirectoryModuleResolver() = default;
+
+			inline void RegisterModuleDirectory(std::string_view path, std::filesystem::path realPath);
+			inline void RegisterModuleFile(std::string_view path, std::filesystem::path realPath);
+			inline void RegisterModuleFile(std::string_view path, std::vector<UInt8> fileContent);
+			inline void RegisterModuleFile(std::string_view path, const void* staticData, std::size_t size);
+
+			ShaderAst::ModulePtr Resolve(const std::vector<std::string>& modulePath) override;
+
+			DirectoryModuleResolver& operator=(const DirectoryModuleResolver&) = delete;
+			DirectoryModuleResolver& operator=(DirectoryModuleResolver&&) = delete;
+
+		private:
+			std::shared_ptr<VirtualDirectory> m_searchDirectory;
+			std::unordered_map<std::string, ShaderAst::ModulePtr> m_knownModules;
+	};
+}
+
+#include <Nazara/Shader/DirectoryModuleResolver.inl>
+
+#endif // NAZARA_SHADER_DIRECTORYMODULERESOLVER_HPP

include/Nazara/Shader/DirectoryModuleResolver.inl

@@ -0,0 +1,36 @@
+// Copyright (C) 2022 Jérôme "Lynix" Leclercq (lynix680@gmail.com)
+// This file is part of the "Nazara Engine - Shader module"
+// For conditions of distribution and use, see copyright notice in Config.hpp
+
+#include <Nazara/Shader/DirectoryModuleResolver.hpp>
+#include <Nazara/Shader/Debug.hpp>
+
+namespace Nz
+{
+	inline DirectoryModuleResolver::DirectoryModuleResolver() :
+	m_searchDirectory(std::make_shared<VirtualDirectory>())
+	{
+	}
+
+	inline void DirectoryModuleResolver::RegisterModuleDirectory(std::string_view path, std::filesystem::path realPath)
+	{
+		m_searchDirectory->StoreDirectory(path, realPath);
+	}
+
+	inline void DirectoryModuleResolver::RegisterModuleFile(std::string_view path, std::filesystem::path realPath)
+	{
+		m_searchDirectory->StoreFile(path, realPath);
+	}
+
+	inline void DirectoryModuleResolver::RegisterModuleFile(std::string_view path, std::vector<UInt8> fileContent)
+	{
+		m_searchDirectory->StoreFile(path, std::move(fileContent));
+	}
+
+	inline void DirectoryModuleResolver::RegisterModuleFile(std::string_view path, const void* staticData, std::size_t size)
+	{
+		m_searchDirectory->StoreFile(path, staticData, size);
+	}
+}
+
+#include <Nazara/Shader/DebugOff.hpp>

include/Nazara/Shader/GlslWriter.hpp

@@ -13,6 +13,7 @@
 #include <Nazara/Shader/Ast/AstExpressionVisitorExcept.hpp>
 #include <Nazara/Shader/Ast/AstStatementVisitorExcept.hpp>
 #include <Nazara/Shader/Ast/Module.hpp>
+#include <Nazara/Shader/Ast/SanitizeVisitor.hpp>
 #include <set>
 #include <sstream>
 #include <string>
@@ -48,7 +49,7 @@ namespace Nz
 			};
 
 			static const char* GetFlipYUniformName();
-			static ShaderAst::ModulePtr Sanitize(const ShaderAst::Module& module, std::unordered_map<UInt32, ShaderAst::ConstantValue> optionValues, std::string* error = nullptr);
+			static ShaderAst::SanitizeVisitor::Options GetSanitizeOptions();
 
 		private:
 			void Append(const ShaderAst::AliasType& aliasType);

include/Nazara/Shader/ShaderModuleResolver.hpp

@@ -0,0 +1,40 @@
+// Copyright (C) 2022 Jérôme "Lynix" Leclercq (lynix680@gmail.com)
+// This file is part of the "Nazara Engine - Shader module"
+// For conditions of distribution and use, see copyright notice in Config.hpp
+
+#pragma once
+
+#ifndef NAZARA_SHADER_SHADERMODULERESOLVER_HPP
+#define NAZARA_SHADER_SHADERMODULERESOLVER_HPP
+
+#include <Nazara/Prerequisites.hpp>
+#include <Nazara/Shader/Config.hpp>
+#include <memory>
+#include <string>
+#include <vector>
+
+namespace Nz
+{
+	namespace ShaderAst
+	{
+		using ModulePtr = std::shared_ptr<class Module>;
+	}
+
+	class NAZARA_SHADER_API ShaderModuleResolver
+	{
+		public:
+			ShaderModuleResolver() = default;
+			ShaderModuleResolver(const ShaderModuleResolver&) = default;
+			ShaderModuleResolver(ShaderModuleResolver&&) = default;
+			virtual ~ShaderModuleResolver();
+
+			virtual ShaderAst::ModulePtr Resolve(const std::vector<std::string>& /*modulePath*/) = 0;
+
+			ShaderModuleResolver& operator=(const ShaderModuleResolver&) = default;
+			ShaderModuleResolver& operator=(ShaderModuleResolver&&) = default;
+	};
+}
+
+#include <Nazara/Shader/ShaderModuleResolver.inl>
+
+#endif // NAZARA_SHADER_SHADERMODULERESOLVER_HPP

include/Nazara/Shader/ShaderModuleResolver.inl

@@ -0,0 +1,12 @@
+// Copyright (C) 2022 Jérôme "Lynix" Leclercq (lynix680@gmail.com)
+// This file is part of the "Nazara Engine - Shader module"
+// For conditions of distribution and use, see copyright notice in Config.hpp
+
+#include <Nazara/Shader/ShaderModuleResolver.hpp>
+#include <Nazara/Shader/Debug.hpp>
+
+namespace Nz
+{
+}
+
+#include <Nazara/Shader/DebugOff.hpp>

include/Nazara/Shader/ShaderWriter.hpp

@@ -10,11 +10,14 @@
 #include <Nazara/Prerequisites.hpp>
 #include <Nazara/Shader/Config.hpp>
 #include <Nazara/Shader/Ast/ConstantValue.hpp>
+#include <memory>
 #include <string>
 #include <unordered_map>
 
 namespace Nz
 {
+	class ShaderModuleResolver;
+
 	class NAZARA_SHADER_API ShaderWriter
 	{
 		public:
@@ -27,6 +30,7 @@ namespace Nz
 
 			struct States
 			{
+				std::shared_ptr<ShaderModuleResolver> shaderModuleResolver;
 				std::unordered_map<UInt32, ShaderAst::ConstantValue> optionValues;
 				bool optimize = false;
 				bool sanitized = false;

src/Nazara/Graphics/Graphics.cpp

@@ -17,6 +17,14 @@ namespace Nz
 		const UInt8 r_blitShader[] = {
 			#include <Nazara/Graphics/Resources/Shaders/blit.nzsl.h>
 		};
+
+		const UInt8 r_instanceDataModule[] = {
+			#include <Nazara/Graphics/Resources/Shaders/Modules/Engine/InstanceData.nzsl.h>
+		};
+
+		const UInt8 r_viewerDataModule[] = {
+			#include <Nazara/Graphics/Resources/Shaders/Modules/Engine/ViewerData.nzsl.h>
+		};
 	}
 
 	/*!
@@ -66,6 +74,7 @@ namespace Nz
 
 		BuildDefaultTextures();
 		BuildFullscreenVertexBuffer();
+		RegisterShaderModules();
 		BuildBlitPipeline();
 		RegisterMaterialPasses();
 		SelectDepthStencilFormats();
@@ -125,7 +134,10 @@ namespace Nz
 		if (!m_blitPipelineLayout)
 			throw std::runtime_error("failed to instantiate fullscreen renderpipeline layout");
 
-		auto blitShader = m_renderDevice->InstantiateShaderModule(ShaderStageType::Fragment | ShaderStageType::Vertex, ShaderLanguage::NazaraShader, r_blitShader, sizeof(r_blitShader), {});
+		ShaderWriter::States states;
+		states.shaderModuleResolver = m_shaderModuleResolver;
+
+		auto blitShader = m_renderDevice->InstantiateShaderModule(ShaderStageType::Fragment | ShaderStageType::Vertex, ShaderLanguage::NazaraShader, r_blitShader, sizeof(r_blitShader), states);
 		if (!blitShader)
 			throw std::runtime_error("failed to instantiate blit shader");
 
@@ -203,6 +215,13 @@ namespace Nz
 		m_materialPassRegistry.RegisterPass("DepthPass");
 	}
 
+	void Graphics::RegisterShaderModules()
+	{
+		m_shaderModuleResolver = std::make_shared<DirectoryModuleResolver>();
+		m_shaderModuleResolver->RegisterModuleFile("Engine/InstanceData", r_instanceDataModule, sizeof(r_instanceDataModule));
+		m_shaderModuleResolver->RegisterModuleFile("Engine/ViewerData", r_viewerDataModule, sizeof(r_viewerDataModule));
+	}
+
 	void Graphics::SelectDepthStencilFormats()
 	{
 		for (PixelFormat depthStencilCandidate : { PixelFormat::Depth24Stencil8, PixelFormat::Depth32FStencil8, PixelFormat::Depth16Stencil8 })

src/Nazara/Graphics/Resources/Shaders/Modules/Engine/InstanceData.nzsl

@@ -0,0 +1,10 @@
+[nzsl_version("1.0")]
+module;
+
+[export]
+[layout(std140)]
+struct InstanceData
+{
+	worldMatrix: mat4[f32],
+	invWorldMatrix: mat4[f32]
+}

src/Nazara/Graphics/Resources/Shaders/Modules/Engine/ViewerData.nzsl

@@ -0,0 +1,17 @@
+[nzsl_version("1.0")]
+module;
+
+[export]
+[layout(std140)]
+struct ViewerData
+{
+	projectionMatrix: mat4[f32],
+	invProjectionMatrix: mat4[f32],
+	viewMatrix: mat4[f32],
+	invViewMatrix: mat4[f32],
+	viewProjMatrix: mat4[f32],
+	invViewProjMatrix: mat4[f32],
+	renderTargetSize: vec2[f32],
+	invRenderTargetSize: vec2[f32],
+	eyePosition: vec3[f32]
+}

src/Nazara/Graphics/Resources/Shaders/basic_material.nzsl

@@ -1,6 +1,9 @@
 [nzsl_version("1.0")]
 module;
 
+import Engine/InstanceData;
+import Engine/ViewerData;
+
 option HasDiffuseTexture: bool = false;
 option HasAlphaTexture: bool = false;
 option AlphaTest: bool = false;
@@ -27,27 +30,6 @@ struct MaterialSettings
 	DiffuseColor: vec4[f32]
 }
 
-[layout(std140)]
-struct InstanceData
-{
-	worldMatrix: mat4[f32],
-	invWorldMatrix: mat4[f32]
-}
-
-[layout(std140)]
-struct ViewerData
-{
-	projectionMatrix: mat4[f32],
-	invProjectionMatrix: mat4[f32],
-	viewMatrix: mat4[f32],
-	invViewMatrix: mat4[f32],
-	viewProjMatrix: mat4[f32],
-	invViewProjMatrix: mat4[f32],
-	renderTargetSize: vec2[f32],
-	invRenderTargetSize: vec2[f32],
-	eyePosition: vec3[f32]
-}
-
 external
 {
 	[binding(0)] settings: uniform[MaterialSettings],

src/Nazara/Graphics/Resources/Shaders/depth_material.nzsl

@@ -1,6 +1,9 @@
 [nzsl_version("1.0")]
 module;
 
+import Engine/InstanceData;
+import Engine/ViewerData;
+
 option HasDiffuseTexture: bool = false;
 option HasAlphaTexture: bool = false;
 option AlphaTest: bool = false;
@@ -14,27 +17,6 @@ struct BasicSettings
 	DiffuseColor: vec4[f32]
 }
 
-[layout(std140)]
-struct InstanceData
-{
-	worldMatrix: mat4[f32],
-	invWorldMatrix: mat4[f32]
-}
-
-[layout(std140)]
-struct ViewerData
-{
-	projectionMatrix: mat4[f32],
-	invProjectionMatrix: mat4[f32],
-	viewMatrix: mat4[f32],
-	invViewMatrix: mat4[f32],
-	viewProjMatrix: mat4[f32],
-	invViewProjMatrix: mat4[f32],
-	renderTargetSize: vec2[f32],
-	invRenderTargetSize: vec2[f32],
-	eyePosition: vec3[f32]
-}
-
 external
 {
 	[binding(0)] settings: uniform[BasicSettings],

src/Nazara/Graphics/Resources/Shaders/phong_material.nzsl

@@ -1,6 +1,9 @@
 [nzsl_version("1.0")]
 module;
 
+import Engine/InstanceData;
+import Engine/ViewerData;
+
 // Basic material options
 option HasDiffuseTexture: bool = false;
 option HasAlphaTexture: bool = false;
@@ -47,13 +50,6 @@ struct MaterialSettings
 	Shininess: f32,
 }
 
-[layout(std140)]
-struct InstanceData
-{
-	worldMatrix: mat4[f32],
-	invWorldMatrix: mat4[f32]
-}
-
 // TODO: Add enums
 const DirectionalLight = 0;
 const PointLight = 1;
@@ -78,20 +74,6 @@ struct LightData
 	lightCount: u32,
 }
 
-[layout(std140)]
-struct ViewerData
-{
-	projectionMatrix: mat4[f32],
-	invProjectionMatrix: mat4[f32],
-	viewMatrix: mat4[f32],
-	invViewMatrix: mat4[f32],
-	viewProjMatrix: mat4[f32],
-	invViewProjMatrix: mat4[f32],
-	renderTargetSize: vec2[f32],
-	invRenderTargetSize: vec2[f32],
-	eyePosition: vec3[f32]
-}
-
 external
 {
 	[binding(0)] settings: uniform[MaterialSettings],

src/Nazara/Graphics/UberShader.cpp

@@ -57,6 +57,7 @@ namespace Nz
 		{
 			ShaderWriter::States states;
 			states.optionValues = config.optionValues;
+			states.shaderModuleResolver = Graphics::Instance()->GetShaderModuleResolver();
 
 			std::shared_ptr<ShaderModule> stage = Graphics::Instance()->GetRenderDevice()->InstantiateShaderModule(m_shaderStages, *m_shaderModule, std::move(states));
 

src/Nazara/OpenGLRenderer/OpenGLShaderModule.cpp

@@ -141,7 +141,12 @@ namespace Nz
 	{
 		m_states = states;
 		m_states.sanitized = true; //< Shader is always sanitized (because of keywords)
-		ShaderAst::ModulePtr sanitized = GlslWriter::Sanitize(shaderModule, states.optionValues);
+
+		ShaderAst::SanitizeVisitor::Options options = GlslWriter::GetSanitizeOptions();
+		options.optionValues = states.optionValues;
+		options.moduleResolver = states.shaderModuleResolver;
+
+		ShaderAst::ModulePtr sanitized = ShaderAst::Sanitize(shaderModule, options);
 
 		for (std::size_t i = 0; i < ShaderStageTypeCount; ++i)
 		{

src/Nazara/Shader/Ast/SanitizeVisitor.cpp

@@ -1398,7 +1398,7 @@ namespace Nz::ShaderAst
 
 	StatementPtr SanitizeVisitor::Clone(ImportStatement& node)
 	{
-		if (!m_context->options.moduleCallback)
+		if (!m_context->options.moduleResolver)
 			return static_unique_pointer_cast<ImportStatement>(AstCloner::Clone(node));
 
 		auto ModulePathAsString = [&]() -> std::string
@@ -1419,7 +1419,7 @@ namespace Nz::ShaderAst
 			return ss.str();
 		};
 
-		ModulePtr targetModule = m_context->options.moduleCallback(node.modulePath);
+		ModulePtr targetModule = m_context->options.moduleResolver->Resolve(node.modulePath);
 		if (!targetModule)
 			throw AstError{ "module " + ModulePathAsString() + " not found" };
 
@@ -1506,16 +1506,20 @@ namespace Nz::ShaderAst
 		AstExportVisitor exportVisitor;
 		exportVisitor.Visit(*m_context->currentModule->importedModules[moduleIndex].module->rootNode, callbacks);
 
-		if (aliasStatements.empty() || m_context->options.removeAliases)
+		if (aliasStatements.empty())
 			return ShaderBuilder::NoOp();
 
-		// Register module and aliases
+		// Register aliases
+		for (auto& aliasPtr : aliasStatements)
+			Validate(*aliasPtr);
+
+		if (m_context->options.removeAliases)
+			return ShaderBuilder::NoOp();
+
+		// Generate alias statements
 		MultiStatementPtr aliasBlock = std::make_unique<MultiStatement>();
 		for (auto& aliasPtr : aliasStatements)
-		{
-			Validate(*aliasPtr);
 			aliasBlock->statements.push_back(std::move(aliasPtr));
-		}
 
 		return aliasBlock;
 	}

src/Nazara/Shader/DirectoryModuleResolver.cpp

@@ -0,0 +1,57 @@
+// Copyright (C) 2022 Jérôme "Lynix" Leclercq (lynix680@gmail.com)
+// This file is part of the "Nazara Engine - Shader module"
+// For conditions of distribution and use, see copyright notice in Config.hpp
+
+#include <Nazara/Shader/DirectoryModuleResolver.hpp>
+#include <Nazara/Shader/ShaderLangParser.hpp>
+#include <filesystem>
+#include <Nazara/Shader/Debug.hpp>
+
+namespace Nz
+{
+	ShaderAst::ModulePtr DirectoryModuleResolver::Resolve(const std::vector<std::string>& modulePath)
+	{
+		if (modulePath.empty())
+			return {};
+
+		std::string fullPath;
+		for (const auto& part : modulePath)
+		{
+			if (!fullPath.empty())
+				fullPath += '/';
+
+			fullPath += part;
+		}
+
+		if (auto it = m_knownModules.find(fullPath); it != m_knownModules.end())
+			return it->second;
+
+		VirtualDirectory::Entry entry;
+		if (!m_searchDirectory->GetEntry(fullPath, &entry))
+			return {};
+
+		ShaderAst::ModulePtr shaderModule;
+		if (std::holds_alternative<VirtualDirectory::DataPointerEntry>(entry))
+		{
+			const auto& content = std::get<VirtualDirectory::DataPointerEntry>(entry);
+			shaderModule = ShaderLang::Parse(std::string_view(reinterpret_cast<const char*>(content.data), content.size));
+		}
+		else if (std::holds_alternative<VirtualDirectory::FileContentEntry>(entry))
+		{
+			const auto& content = std::get<VirtualDirectory::FileContentEntry>(entry);
+			shaderModule = ShaderLang::Parse(std::string_view(reinterpret_cast<const char*>(content.data->data()), content.data->size()));
+		}
+		else if (std::holds_alternative<VirtualDirectory::PhysicalFileEntry>(entry))
+		{
+			const auto& filePath = std::get<VirtualDirectory::PhysicalFileEntry>(entry);
+			shaderModule = ShaderLang::ParseFromFile(filePath);
+		}
+
+		if (!shaderModule)
+			return {};
+
+		m_knownModules.emplace(fullPath, shaderModule);
+
+		return shaderModule;
+	}
+}

src/Nazara/Shader/GlslWriter.cpp

@@ -13,7 +13,6 @@
 #include <Nazara/Shader/Ast/AstRecursiveVisitor.hpp>
 #include <Nazara/Shader/Ast/AstUtils.hpp>
 #include <Nazara/Shader/Ast/EliminateUnusedPassVisitor.hpp>
-#include <Nazara/Shader/Ast/SanitizeVisitor.hpp>
 #include <optional>
 #include <set>
 #include <stdexcept>
@@ -173,7 +172,11 @@ namespace Nz
 		const ShaderAst::Module* targetModule;
 		if (!states.sanitized)
 		{
-			sanitizedModule = Sanitize(module, states.optionValues);
+			ShaderAst::SanitizeVisitor::Options options = GetSanitizeOptions();
+			options.optionValues = states.optionValues;
+			options.moduleResolver = states.shaderModuleResolver;
+
+			sanitizedModule = ShaderAst::Sanitize(module, options);
 			targetModule = sanitizedModule.get();
 		}
 		else
@@ -227,11 +230,10 @@ namespace Nz
 		return s_flipYUniformName;
 	}
 
-	ShaderAst::ModulePtr GlslWriter::Sanitize(const ShaderAst::Module& module, std::unordered_map<UInt32, ShaderAst::ConstantValue> optionValues, std::string* error)
+	ShaderAst::SanitizeVisitor::Options GlslWriter::GetSanitizeOptions()
 	{
 		// Always sanitize for reserved identifiers
 		ShaderAst::SanitizeVisitor::Options options;
-		options.optionValues = std::move(optionValues);
 		options.makeVariableNameUnique = true;
 		options.reduceLoopsToWhile = true;
 		options.removeAliases = true;
@@ -246,7 +248,7 @@ namespace Nz
 			"cross", "dot", "exp", "length", "max", "min", "pow", "texture"
 		};
 
-		return ShaderAst::Sanitize(module, options, error);
+		return options;
 	}
 
 	void GlslWriter::Append(const ShaderAst::AliasType& /*aliasType*/)

src/Nazara/Shader/ShaderModuleResolver.cpp

@@ -0,0 +1,11 @@
+// Copyright (C) 2022 Jérôme "Lynix" Leclercq (lynix680@gmail.com)
+// This file is part of the "Nazara Engine - Shader module"
+// For conditions of distribution and use, see copyright notice in Config.hpp
+
+#include <Nazara/Shader/ShaderModuleResolver.hpp>
+#include <Nazara/Shader/Debug.hpp>
+
+namespace Nz
+{
+	ShaderModuleResolver::~ShaderModuleResolver() = default;
+}

src/Nazara/Shader/SpirvWriter.cpp

@@ -503,10 +503,12 @@ namespace Nz
 		if (!states.sanitized)
 		{
 			ShaderAst::SanitizeVisitor::Options options;
+			options.moduleResolver = states.shaderModuleResolver;
 			options.optionValues = states.optionValues;
 			options.reduceLoopsToWhile = true;
 			options.removeAliases = true;
 			options.removeCompoundAssignments = true;
+			options.removeConstDeclaration = true;
 			options.removeMatrixCast = true;
 			options.removeOptionDeclaration = true;
 			options.splitMultipleBranches = true;

tests/Engine/Core/VirtualDirectoryTest.cpp

@@ -0,0 +1,71 @@
+#include <Nazara/Core/VirtualDirectory.hpp>
+#include <catch2/catch.hpp>
+#include <random>
+
+TEST_CASE("VirtualDirectory", "[Core][VirtualDirectory]")
+{
+	std::shared_ptr<Nz::VirtualDirectory> virtualDir = std::make_shared<Nz::VirtualDirectory>();
+
+	WHEN("Iterating it, it only has . and ..")
+	{
+		bool failed = false;
+		virtualDir->Foreach([&](const std::string& name, const Nz::VirtualDirectory::Entry& /*entry*/)
+		{
+			if (name != "." && name != "..")
+				failed = true;
+		}, true);
+
+		CHECK_FALSE(failed);
+	}
+	AND_WHEN("Iterating it including dots, we get them")
+	{
+		bool failed = false;
+		virtualDir->Foreach([&](const std::string& name, const Nz::VirtualDirectory::Entry& /*entry*/)
+		{
+			failed = true;
+		});
+
+		CHECK_FALSE(failed);
+	}
+	AND_WHEN("We try to retrieve a file, it fails")
+	{
+		Nz::VirtualDirectory::Entry entry;
+		CHECK_FALSE(virtualDir->GetEntry("File.bin", &entry));
+		CHECK_FALSE(virtualDir->GetEntry("Foo/File.bin", &entry));
+		CHECK_FALSE(virtualDir->GetEntry("Foo/Bar/File.bin", &entry));
+
+		virtualDir->StoreDirectory("Foo/Bar", std::make_shared<Nz::VirtualDirectory>());
+
+		CHECK(virtualDir->GetEntry("Foo", &entry));
+		CHECK(std::holds_alternative<Nz::VirtualDirectory::VirtualDirectoryEntry>(entry));
+		CHECK(virtualDir->GetEntry("Foo/Bar", &entry));
+		CHECK(std::holds_alternative<Nz::VirtualDirectory::VirtualDirectoryEntry>(entry));
+		CHECK_FALSE(virtualDir->GetEntry("Foo/Bar/File.bin", &entry));
+	}
+
+	WHEN("Storing a file")
+	{
+		std::mt19937 randGen(std::random_device{}());
+		std::vector<Nz::UInt8> randomData;
+		for (std::size_t i = 0; i < 1024; ++i)
+		{
+			unsigned int data = randGen();
+			randomData.push_back((data & 0x000000FF) >>  0);
+			randomData.push_back((data & 0x0000FF00) >>  8);
+			randomData.push_back((data & 0x00FF0000) >> 16);
+			randomData.push_back((data & 0xFF000000) >> 24);
+		}
+
+		virtualDir->StoreFile("File.bin", randomData);
+
+		WHEN("We retrieve it")
+		{
+			Nz::VirtualDirectory::Entry entry;
+			REQUIRE(virtualDir->GetEntry("File.bin", &entry));
+
+			REQUIRE(std::holds_alternative<Nz::VirtualDirectory::FileContentEntry>(entry));
+			const auto& contentEntry = std::get<Nz::VirtualDirectory::FileContentEntry>(entry);
+			CHECK(std::equal(randomData.begin(), randomData.end(), contentEntry.data->begin(), contentEntry.data->end()));
+		}
+	}
+}

tests/Engine/Shader/ModuleTests.cpp

@@ -1,6 +1,7 @@
 #include <Engine/Shader/ShaderUtils.hpp>
 #include <Nazara/Core/File.hpp>
 #include <Nazara/Core/StringExt.hpp>
+#include <Nazara/Shader/DirectoryModuleResolver.hpp>
 #include <Nazara/Shader/ShaderBuilder.hpp>
 #include <Nazara/Shader/ShaderLangParser.hpp>
 #include <Nazara/Shader/Ast/SanitizeVisitor.hpp>
@@ -29,8 +30,6 @@ struct OutputData
 }
 )";
 
-		Nz::ShaderAst::ModulePtr importedModule = Nz::ShaderLang::Parse(importedSource);
-
 		std::string_view shaderSource = R"(
 [nzsl_version("1.0")]
 module;
@@ -48,14 +47,11 @@ fn main(input: InputData) -> OutputData
 
 		Nz::ShaderAst::ModulePtr shaderModule = Nz::ShaderLang::Parse(shaderSource);
 
-		Nz::ShaderAst::SanitizeVisitor::Options sanitizeOpt;
+		auto directoryModuleResolver = std::make_shared<Nz::DirectoryModuleResolver>();
-		sanitizeOpt.moduleCallback = [&](const std::vector<std::string>& modulePath) -> Nz::ShaderAst::ModulePtr
+		directoryModuleResolver->RegisterModuleFile("SimpleModule", importedSource.data(), importedSource.size());
-		{
-			REQUIRE(modulePath.size() == 1);
-			REQUIRE(modulePath[0] == "SimpleModule");
 
-			return importedModule;
+		Nz::ShaderAst::SanitizeVisitor::Options sanitizeOpt;
-		};
+		sanitizeOpt.moduleResolver = directoryModuleResolver;
 
 		REQUIRE_NOTHROW(shaderModule = Nz::ShaderAst::Sanitize(*shaderModule, sanitizeOpt));