Add support for shader hotreloading

include/Nazara/Core/StringExt.hpp

@@ -20,6 +20,8 @@ namespace Nz
 	// std::string is assumed to contains UTF-8
 	NAZARA_CORE_API std::size_t ComputeCharacterCount(const std::string_view& str);
 
+	inline bool EndsWith(const std::string_view& str, const std::string_view& s);
+
 	NAZARA_CORE_API std::string FromUtf16String(const std::u16string_view& u16str);
 	NAZARA_CORE_API std::string FromUtf32String(const std::u32string_view& u32str);
 	NAZARA_CORE_API std::string FromWideString(const std::wstring_view& str);

include/Nazara/Core/StringExt.inl

@@ -10,6 +10,20 @@
 
 namespace Nz
 {
+	inline bool EndsWith(const std::string_view& str, const std::string_view& s)
+	{
+		//FIXME: Replace with proper C++20 value once it's available
+#if __cplusplus > 201703L
+		// C++20
+		return str.ends_with(s);
+#else
+		if (s.size() > str.size())
+			return false;
+
+		return str.compare(str.size() - s.size(), s.size(), s.data()) == 0;
+#endif
+	}
+
 	inline bool IsNumber(std::string_view str)
 	{
 		if (str.empty())

include/Nazara/Graphics/MaterialPass.hpp

@@ -35,6 +35,8 @@ namespace Nz
 	{
 		public:
 			MaterialPass(std::shared_ptr<const MaterialSettings> settings);
+			MaterialPass(const MaterialPass&) = delete;
+			MaterialPass(MaterialPass&&) = delete;
 			inline ~MaterialPass();
 
 			inline void Configure(std::shared_ptr<MaterialPipeline> pipeline);
@@ -105,6 +107,9 @@ namespace Nz
 
 			void Update(RenderFrame& renderFrame, CommandBufferBuilder& builder);
 
+			MaterialPass& operator=(const MaterialPass&) = delete;
+			MaterialPass& operator=(MaterialPass&&) = delete;
+
 			// Signals:
 			NazaraSignal(OnMaterialPassInvalidated, const MaterialPass* /*materialPass*/);
 			NazaraSignal(OnMaterialPassPipelineInvalidated, const MaterialPass* /*materialPass*/);
@@ -125,6 +130,11 @@ namespace Nz
 				TextureSamplerInfo samplerInfo;
 			};
 
+			struct ShaderEntry
+			{
+				NazaraSlot(UberShader, OnShaderUpdated, onShaderUpdated);
+			};
+
 			struct UniformBuffer
 			{
 				std::shared_ptr<RenderBuffer> buffer;
@@ -135,6 +145,7 @@ namespace Nz
 			std::array<ShaderAst::ConstantValue, 64> m_optionValues;
 			std::shared_ptr<const MaterialSettings> m_settings;
 			std::vector<MaterialTexture> m_textures;
+			std::vector<ShaderEntry> m_shaders;
 			std::vector<UniformBuffer> m_uniformBuffers;
 			mutable std::shared_ptr<MaterialPipeline> m_pipeline;
 			mutable MaterialPipelineInfo m_pipelineInfo;

include/Nazara/Graphics/MaterialPipeline.hpp

@@ -67,7 +67,13 @@ namespace Nz
 			static bool Initialize();
 			static void Uninitialize();
 
+			struct UberShaderEntry
+			{
+				NazaraSlot(UberShader, OnShaderUpdated, onShaderUpdated);
+			};
+
 			mutable std::vector<std::shared_ptr<RenderPipeline>> m_renderPipelines;
+			std::vector<UberShaderEntry> m_uberShaderEntries;
 			MaterialPipelineInfo m_pipelineInfo;
 
 			using PipelineCache = std::unordered_map<MaterialPipelineInfo, std::shared_ptr<MaterialPipeline>>;

include/Nazara/Graphics/MaterialPipeline.inl

@@ -12,6 +12,15 @@ namespace Nz
 	inline MaterialPipeline::MaterialPipeline(const MaterialPipelineInfo& pipelineInfo, Token) :
 	m_pipelineInfo(pipelineInfo)
 	{
+		m_uberShaderEntries.resize(m_pipelineInfo.shaders.size());
+		for (std::size_t i = 0; i < m_uberShaderEntries.size(); ++i)
+		{
+			m_uberShaderEntries[i].onShaderUpdated.Connect(m_pipelineInfo.shaders[i].uberShader->OnShaderUpdated, [this](UberShader*)
+			{
+				// Clear cache
+				m_renderPipelines.clear();
+			});
+		}
 	}
 
 	/*!

include/Nazara/Graphics/UberShader.hpp

@@ -10,8 +10,10 @@
 #include <Nazara/Prerequisites.hpp>
 #include <Nazara/Core/Algorithm.hpp>
 #include <Nazara/Core/Bitset.hpp>
+#include <Nazara/Core/Signal.hpp>
 #include <Nazara/Graphics/Config.hpp>
 #include <Nazara/Renderer/RenderPipeline.hpp>
+#include <Nazara/Shader/ShaderModuleResolver.hpp>
 #include <Nazara/Shader/Ast/Module.hpp>
 #include <unordered_map>
 
@@ -26,6 +28,8 @@ namespace Nz
 			struct Option;
 			using ConfigCallback = std::function<void(Config& config, const std::vector<RenderPipelineInfo::VertexBufferData>& vertexBuffers)>;
 
+			UberShader(ShaderStageTypeFlags shaderStages, std::string moduleName);
+			UberShader(ShaderStageTypeFlags shaderStages, ShaderModuleResolver& moduleResolver, std::string moduleName);
 			UberShader(ShaderStageTypeFlags shaderStages, ShaderAst::ModulePtr shaderModule);
 			~UberShader() = default;
 
@@ -58,7 +62,13 @@ namespace Nz
 				UInt32 hash;
 			};
 
+			NazaraSignal(OnShaderUpdated, UberShader* /*uberShader*/);
+
 		private:
+			void Validate(ShaderAst::Module& module);
+
+			NazaraSlot(ShaderModuleResolver, OnModuleUpdated, m_onShaderModuleUpdated);
+
 			std::unordered_map<Config, std::shared_ptr<ShaderModule>, ConfigHasher, ConfigEqual> m_combinations;
 			std::unordered_map<std::string, Option> m_optionIndexByName;
 			ShaderAst::ModulePtr m_shaderModule;

include/Nazara/Shader/FilesystemModuleResolver.hpp

@@ -8,9 +8,11 @@
 #define NAZARA_SHADER_FILESYSTEMMODULERESOLVER_HPP
 
 #include <Nazara/Prerequisites.hpp>
+#include <Nazara/Core/MovablePtr.hpp>
 #include <Nazara/Shader/Config.hpp>
 #include <Nazara/Shader/ShaderModuleResolver.hpp>
 #include <filesystem>
+#include <memory>
 #include <string>
 #include <unordered_map>
 
@@ -19,23 +21,32 @@ namespace Nz
 	class NAZARA_SHADER_API FilesystemModuleResolver : public ShaderModuleResolver
 	{
 		public:
-			FilesystemModuleResolver() = default;
-			FilesystemModuleResolver(const FilesystemModuleResolver&) = default;
-			FilesystemModuleResolver(FilesystemModuleResolver&&) = default;
-			~FilesystemModuleResolver() = default;
+			FilesystemModuleResolver();
+			FilesystemModuleResolver(const FilesystemModuleResolver&) = delete;
+			FilesystemModuleResolver(FilesystemModuleResolver&&) noexcept = delete;
+			~FilesystemModuleResolver();
 
 			void RegisterModule(const std::filesystem::path& realPath);
 			void RegisterModule(std::string_view moduleSource);
 			void RegisterModule(ShaderAst::ModulePtr module);
-			void RegisterModuleDirectory(const std::filesystem::path& realPath);
+			void RegisterModuleDirectory(const std::filesystem::path& realPath, bool watchDirectory = true);
 
 			ShaderAst::ModulePtr Resolve(const std::string& moduleName) override;
 
-			FilesystemModuleResolver& operator=(const FilesystemModuleResolver&) = default;
-			FilesystemModuleResolver& operator=(FilesystemModuleResolver&&) = default;
+			FilesystemModuleResolver& operator=(const FilesystemModuleResolver&) = delete;
+			FilesystemModuleResolver& operator=(FilesystemModuleResolver&&) noexcept = delete;
+
+			static constexpr const char* ModuleExtension = ".nzsl";
 
 		private:
+			void OnFileAdded(std::string_view directory, std::string_view filename);
+			void OnFileRemoved(std::string_view directory, std::string_view filename);
+			void OnFileMoved(std::string_view directory, std::string_view filename, std::string_view oldFilename);
+			void OnFileUpdated(std::string_view directory, std::string_view filename);
+
+			std::unordered_map<std::string, std::string> m_moduleByFilepath;
 			std::unordered_map<std::string, ShaderAst::ModulePtr> m_modules;
+			MovablePtr<void> m_fileWatcher;
 	};
 }
 

include/Nazara/Shader/ShaderModuleResolver.hpp

@@ -8,6 +8,7 @@
 #define NAZARA_SHADER_SHADERMODULERESOLVER_HPP
 
 #include <Nazara/Prerequisites.hpp>
+#include <Nazara/Core/Signal.hpp>
 #include <Nazara/Shader/Config.hpp>
 #include <memory>
 #include <string>
@@ -32,6 +33,8 @@ namespace Nz
 
 			ShaderModuleResolver& operator=(const ShaderModuleResolver&) = default;
 			ShaderModuleResolver& operator=(ShaderModuleResolver&&) = default;
+
+			NazaraSignal(OnModuleUpdated, ShaderModuleResolver* /*resolver*/, const std::string& /*moduleName*/);
 	};
 }
 

src/Nazara/Graphics/BasicMaterial.cpp

@@ -229,8 +229,7 @@ namespace Nz
 
 	std::vector<std::shared_ptr<UberShader>> BasicMaterial::BuildShaders()
 	{
-		ShaderAst::ModulePtr shaderModule = Graphics::Instance()->GetShaderModuleResolver()->Resolve("BasicMaterial");
-		auto shader = std::make_shared<UberShader>(ShaderStageType::Fragment | ShaderStageType::Vertex, std::move(shaderModule));
+		auto shader = std::make_shared<UberShader>(ShaderStageType::Fragment | ShaderStageType::Vertex, "BasicMaterial");
 
 		return { std::move(shader) };
 	}

src/Nazara/Graphics/DepthMaterial.cpp

@@ -11,8 +11,7 @@ namespace Nz
 {
 	std::vector<std::shared_ptr<UberShader>> DepthMaterial::BuildShaders()
 	{
-		ShaderAst::ModulePtr shaderModule = Graphics::Instance()->GetShaderModuleResolver()->Resolve("DepthMaterial");
-		auto shader = std::make_shared<UberShader>(ShaderStageType::Fragment | ShaderStageType::Vertex, std::move(shaderModule));
+		auto shader = std::make_shared<UberShader>(ShaderStageType::Fragment | ShaderStageType::Vertex, "DepthMaterial");
 
 		return { std::move(shader) };
 	}

src/Nazara/Graphics/MaterialPass.cpp

@@ -33,10 +33,17 @@ namespace Nz
 		m_pipelineInfo.settings = m_settings;
 
 		const auto& shaders = m_settings->GetShaders();
-		for (const auto& shader : shaders)
+
+		m_shaders.resize(shaders.size());
+		for (std::size_t i = 0; i < m_shaders.size(); ++i)
 		{
 			auto& shaderData = m_pipelineInfo.shaders.emplace_back();
-			shaderData.uberShader = shader;
+			shaderData.uberShader = shaders[i];
+
+			m_shaders[i].onShaderUpdated.Connect(shaders[i]->OnShaderUpdated, [this](UberShader*)
+			{
+				InvalidatePipeline();
+			});
 		}
 
 		const auto& textureSettings = m_settings->GetTextures();

src/Nazara/Graphics/PhongLightingMaterial.cpp

@@ -310,8 +310,7 @@ namespace Nz
 
 	std::vector<std::shared_ptr<UberShader>> PhongLightingMaterial::BuildShaders()
 	{
-		ShaderAst::ModulePtr shaderModule = Graphics::Instance()->GetShaderModuleResolver()->Resolve("PhongMaterial");
-		auto shader = std::make_shared<UberShader>(ShaderStageType::Fragment | ShaderStageType::Vertex, std::move(shaderModule));
+		auto shader = std::make_shared<UberShader>(ShaderStageType::Fragment | ShaderStageType::Vertex, "PhongMaterial");
 
 		return { std::move(shader) };
 	}

src/Nazara/Graphics/UberShader.cpp

@@ -3,6 +3,7 @@
 // For conditions of distribution and use, see copyright notice in Config.hpp
 
 #include <Nazara/Graphics/UberShader.hpp>
+#include <Nazara/Core/Error.hpp>
 #include <Nazara/Graphics/Graphics.hpp>
 #include <Nazara/Renderer/RenderDevice.hpp>
 #include <Nazara/Shader/Ast/AstReflect.hpp>
@@ -13,13 +14,81 @@
 
 namespace Nz
 {
+	UberShader::UberShader(ShaderStageTypeFlags shaderStages, std::string moduleName) :
+	UberShader(shaderStages, *Graphics::Instance()->GetShaderModuleResolver(), std::move(moduleName))
+	{
+	}
+
+	UberShader::UberShader(ShaderStageTypeFlags shaderStages, ShaderModuleResolver& moduleResolver, std::string moduleName) :
+	m_shaderStages(shaderStages)
+	{
+		m_shaderModule = moduleResolver.Resolve(moduleName);
+		NazaraAssert(m_shaderModule, "invalid shader module");
+
+		Validate(*m_shaderModule);
+
+		m_onShaderModuleUpdated.Connect(moduleResolver.OnModuleUpdated, [this, name = std::move(moduleName)](ShaderModuleResolver* resolver, const std::string& updatedModuleName)
+		{
+			if (updatedModuleName != name)
+				return;
+
+			ShaderAst::ModulePtr newShaderModule = resolver->Resolve(name);
+			if (!newShaderModule)
+			{
+				NazaraError("failed to retrieve updated shader module " + name);
+				return;
+			}
+
+			try
+			{
+				// FIXME: Validate is destructive, in case of failure it can invalidate the shader
+				Validate(*newShaderModule);
+			}
+			catch (const std::exception& e)
+			{
+				NazaraError("failed to retrieve updated shader module " + name + ": " + e.what());
+				return;
+			}
+
+			m_shaderModule = std::move(newShaderModule);
+
+			// Clear cache
+			m_combinations.clear();
+
+			OnShaderUpdated(this);
+		});
+	}
+
 	UberShader::UberShader(ShaderStageTypeFlags shaderStages, ShaderAst::ModulePtr shaderModule) :
 	m_shaderModule(std::move(shaderModule)),
 	m_shaderStages(shaderStages)
 	{
-		NazaraAssert(m_shaderStages != 0, "there must be at least one shader stage");
 		NazaraAssert(m_shaderModule, "invalid shader module");
 
+		Validate(*m_shaderModule);
+	}
+
+	const std::shared_ptr<ShaderModule>& UberShader::Get(const Config& config)
+	{
+		auto it = m_combinations.find(config);
+		if (it == m_combinations.end())
+		{
+			ShaderWriter::States states;
+			states.optionValues = config.optionValues;
+			states.shaderModuleResolver = Graphics::Instance()->GetShaderModuleResolver();
+
+			std::shared_ptr<ShaderModule> stage = Graphics::Instance()->GetRenderDevice()->InstantiateShaderModule(m_shaderStages, *m_shaderModule, std::move(states));
+
+			it = m_combinations.emplace(config, std::move(stage)).first;
+		}
+
+		return it->second;
+	}
+
+	void UberShader::Validate(ShaderAst::Module& module)
+	{
+		NazaraAssert(m_shaderStages != 0, "there must be at least one shader stage");
+
 		//TODO: Try to partially sanitize shader?
 
 		std::size_t optionCount = 0;
@@ -44,26 +113,9 @@ namespace Nz
 		};
 
 		ShaderAst::AstReflect reflect;
-		reflect.Reflect(*m_shaderModule->rootNode, callbacks);
+		reflect.Reflect(*module.rootNode, callbacks);
 
 		if ((m_shaderStages & supportedStageType) != m_shaderStages)
 			throw std::runtime_error("shader doesn't support all required shader stages");
 	}
-
-	const std::shared_ptr<ShaderModule>& UberShader::Get(const Config& config)
-	{
-		auto it = m_combinations.find(config);
-		if (it == m_combinations.end())
-		{
-			ShaderWriter::States states;
-			states.optionValues = config.optionValues;
-			states.shaderModuleResolver = Graphics::Instance()->GetShaderModuleResolver();
-
-			std::shared_ptr<ShaderModule> stage = Graphics::Instance()->GetRenderDevice()->InstantiateShaderModule(m_shaderStages, *m_shaderModule, std::move(states));
-
-			it = m_combinations.emplace(config, std::move(stage)).first;
-		}
-
-		return it->second;
-	}
 }

src/Nazara/Shader/FilesystemModuleResolver.cpp

@@ -6,19 +6,44 @@
 #include <Nazara/Core/Error.hpp>
 #include <Nazara/Core/StringExt.hpp>
 #include <Nazara/Shader/ShaderLangParser.hpp>
+#include <efsw/efsw.h>
 #include <cassert>
 #include <Nazara/Shader/Debug.hpp>
 
 namespace Nz
 {
+	FilesystemModuleResolver::FilesystemModuleResolver()
+	{
+		m_fileWatcher = efsw_create(0);
+		efsw_watch(m_fileWatcher);
+	}
+
+	FilesystemModuleResolver::~FilesystemModuleResolver()
+	{
+		if (m_fileWatcher)
+			efsw_release(m_fileWatcher);
+	}
+
 	void FilesystemModuleResolver::RegisterModule(const std::filesystem::path& realPath)
 	{
-		return RegisterModule(ShaderLang::ParseFromFile(realPath));
+		ShaderAst::ModulePtr module = ShaderLang::ParseFromFile(realPath);
+		if (!module)
+			return;
+
+		std::string moduleName = module->metadata->moduleName;
+		RegisterModule(std::move(module));
+
+		std::filesystem::path canonicalPath = std::filesystem::canonical(realPath);
+		m_moduleByFilepath.emplace(canonicalPath.generic_u8string(), std::move(moduleName));
 	}
 
 	void FilesystemModuleResolver::RegisterModule(std::string_view moduleSource)
 	{
-		return RegisterModule(ShaderLang::Parse(moduleSource));
+		ShaderAst::ModulePtr module = ShaderLang::Parse(moduleSource);
+		if (!module)
+			return;
+
+		return RegisterModule(std::move(module));
 	}
 
 	void FilesystemModuleResolver::RegisterModule(ShaderAst::ModulePtr module)
@@ -29,14 +54,52 @@ namespace Nz
 		if (moduleName.empty())
 			throw std::runtime_error("cannot register anonymous module");
 
-		m_modules.insert_or_assign(std::move(moduleName), std::move(module));
+		auto it = m_modules.find(moduleName);
+		if (it != m_modules.end())
+		{
+			it->second = std::move(module);
+
+			OnModuleUpdated(this, moduleName);
+		}
+		else
+			m_modules.emplace(std::move(moduleName), std::move(module));
 	}
 
-	void FilesystemModuleResolver::RegisterModuleDirectory(const std::filesystem::path& realPath)
+	void FilesystemModuleResolver::RegisterModuleDirectory(const std::filesystem::path& realPath, bool watchDirectory)
 	{
+		if (!std::filesystem::is_directory(realPath))
+			return;
+
+		auto FileSystemCallback = [](efsw_watcher /*watcher*/, efsw_watchid /*watchid*/, const char* dir, const char* filename, efsw_action action, const char* oldFileName, void* param)
+		{
+			FilesystemModuleResolver* resolver = static_cast<FilesystemModuleResolver*>(param);
+
+			switch (action)
+			{
+				case EFSW_ADD:
+					resolver->OnFileAdded(dir, filename);
+					break;
+
+				case EFSW_DELETE:
+					resolver->OnFileRemoved(dir, filename);
+					break;
+
+				case EFSW_MODIFIED:
+					resolver->OnFileUpdated(dir, filename);
+					break;
+
+				case EFSW_MOVED:
+					resolver->OnFileMoved(dir, filename, (oldFileName) ? oldFileName : std::string_view());
+					break;
+			}
+		};
+
+		if (watchDirectory)
+			efsw_addwatch(m_fileWatcher, realPath.generic_u8string().c_str(), FileSystemCallback, 1, this);
+
 		for (const auto& entry : std::filesystem::recursive_directory_iterator(realPath))
 		{
-			if (entry.is_regular_file() && StringEqual(entry.path().extension().generic_u8string(), ".nzsl", Nz::CaseIndependent{}))
+			if (entry.is_regular_file() && StringEqual(entry.path().extension().generic_u8string(), ModuleExtension, Nz::CaseIndependent{}))
 			{
 				try
 				{
@@ -58,4 +121,53 @@ namespace Nz
 
 		return it->second;
 	}
+
+	void FilesystemModuleResolver::OnFileAdded(std::string_view directory, std::string_view filename)
+	{
+		if (!EndsWith(filename, ModuleExtension))
+			return;
+
+		RegisterModule(std::filesystem::path(directory) / filename);
+	}
+
+	void FilesystemModuleResolver::OnFileRemoved(std::string_view directory, std::string_view filename)
+	{
+		if (!EndsWith(filename, ModuleExtension))
+			return;
+
+		std::filesystem::path canonicalPath = std::filesystem::canonical(std::filesystem::path(directory) / filename);
+
+		auto it = m_moduleByFilepath.find(canonicalPath.generic_u8string());
+		if (it != m_moduleByFilepath.end())
+		{
+			m_modules.erase(it->second);
+			m_moduleByFilepath.erase(it);
+		}
+	}
+
+	void FilesystemModuleResolver::OnFileMoved(std::string_view directory, std::string_view filename, std::string_view oldFilename)
+	{
+		if (oldFilename.empty() || !EndsWith(oldFilename, ModuleExtension))
+			return;
+
+		std::filesystem::path canonicalPath = std::filesystem::canonical(std::filesystem::path(directory) / oldFilename);
+		auto it = m_moduleByFilepath.find(canonicalPath.generic_u8string());
+		if (it != m_moduleByFilepath.end())
+		{
+			std::filesystem::path newCanonicalPath = std::filesystem::canonical(std::filesystem::path(directory) / filename);
+
+			std::string moduleName = std::move(it->second);
+			m_moduleByFilepath.erase(it);
+
+			m_moduleByFilepath.emplace(newCanonicalPath.generic_u8string(), std::move(moduleName));
+		}
+	}
+
+	void FilesystemModuleResolver::OnFileUpdated(std::string_view directory, std::string_view filename)
+	{
+		if (!EndsWith(filename, ModuleExtension))
+			return;
+
+		RegisterModule(std::filesystem::path(directory) / filename);
+	}
 }

src/Nazara/Shader/ShaderLangParser.cpp

@@ -1432,11 +1432,13 @@ namespace Nz::ShaderLang
 		}
 
 		std::size_t length = static_cast<std::size_t>(file.GetSize());
+		if (length == 0)
+			return {};
 
 		std::vector<Nz::UInt8> source(length);
 		if (file.Read(&source[0], length) != length)
 		{
-			NazaraError("Failed to read program file");
+			NazaraError("Failed to read shader file");
 			return {};
 		}
 

tests/Engine/Core/StringExtTest.cpp

@@ -5,6 +5,15 @@ SCENARIO("String", "[CORE][STRING]")
 {
 	std::string unicodeString(u8"\u00E0\u00E9\u00E7\u0153\u00C2\u5B98\u46E1");
 
+	WHEN("Checking if string ends with")
+	{
+		CHECK(Nz::EndsWith("Nazara Engine", "Engine"));
+		CHECK_FALSE(Nz::EndsWith("Nazara Engine", " ngine"));
+		CHECK_FALSE(Nz::EndsWith("Nazara Engine", "NazaraEngine"));
+		CHECK_FALSE(Nz::EndsWith("Nazara Engine", "Nazara"));
+		CHECK_FALSE(Nz::EndsWith("Nazara Engine", "Sir Nazara van Engine"));
+	}
+
 	WHEN("Converting string back and forth")
 	{
 		CHECK(Nz::FromUtf16String(Nz::ToUtf16String(unicodeString)) == unicodeString);