Add shader compiler (nzslc) and use it

include/Nazara/Shader/FilesystemModuleResolver.hpp

@@ -36,6 +36,7 @@ namespace Nz
 			FilesystemModuleResolver& operator=(const FilesystemModuleResolver&) = delete;
 			FilesystemModuleResolver& operator=(FilesystemModuleResolver&&) noexcept = delete;
 
+			static constexpr const char* CompiledModuleExtension = ".nzslb";
 			static constexpr const char* ModuleExtension = ".nzsl";
 
 		private:
@@ -44,6 +45,8 @@ namespace Nz
 			void OnFileMoved(std::string_view directory, std::string_view filename, std::string_view oldFilename);
 			void OnFileUpdated(std::string_view directory, std::string_view filename);
 
+			static bool CheckExtension(std::string_view filename);
+
 			std::unordered_map<std::string, std::string> m_moduleByFilepath;
 			std::unordered_map<std::string, ShaderAst::ModulePtr> m_modules;
 			MovablePtr<void> m_fileWatcher;

src/Nazara/Graphics/Graphics.cpp

@@ -6,6 +6,8 @@
 #include <Nazara/Graphics/GuillotineTextureAtlas.hpp>
 #include <Nazara/Graphics/MaterialPipeline.hpp>
 #include <Nazara/Graphics/PredefinedShaderStructs.hpp>
+#include <Nazara/Shader/Ast/AstSerializer.hpp>
+#include <Nazara/Shader/Ast/Module.hpp>
 #include <Nazara/Utility/Font.hpp>
 #include <array>
 #include <stdexcept>
@@ -16,31 +18,31 @@ namespace Nz
 	namespace
 	{
 		const UInt8 r_textureBlitShader[] = {
-			#include <Nazara/Graphics/Resources/Shaders/TextureBlit.nzsl.h>
+			#include <Nazara/Graphics/Resources/Shaders/TextureBlit.nzslb.h>
 		};
 
 		const UInt8 r_basicMaterialShader[] = {
-			#include <Nazara/Graphics/Resources/Shaders/BasicMaterial.nzsl.h>
+			#include <Nazara/Graphics/Resources/Shaders/BasicMaterial.nzslb.h>
 		};
 
 		const UInt8 r_depthMaterialShader[] = {
-			#include <Nazara/Graphics/Resources/Shaders/DepthMaterial.nzsl.h>
+			#include <Nazara/Graphics/Resources/Shaders/DepthMaterial.nzslb.h>
 		};
 
 		const UInt8 r_phongMaterialShader[] = {
-			#include <Nazara/Graphics/Resources/Shaders/PhongMaterial.nzsl.h>
+			#include <Nazara/Graphics/Resources/Shaders/PhongMaterial.nzslb.h>
 		};
 
 		const UInt8 r_instanceDataModule[] = {
-			#include <Nazara/Graphics/Resources/Shaders/Modules/Engine/InstanceData.nzsl.h>
+			#include <Nazara/Graphics/Resources/Shaders/Modules/Engine/InstanceData.nzslb.h>
 		};
 
 		const UInt8 r_lightDataModule[] = {
-			#include <Nazara/Graphics/Resources/Shaders/Modules/Engine/LightData.nzsl.h>
+			#include <Nazara/Graphics/Resources/Shaders/Modules/Engine/LightData.nzslb.h>
 		};
 
 		const UInt8 r_viewerDataModule[] = {
-			#include <Nazara/Graphics/Resources/Shaders/Modules/Engine/ViewerData.nzsl.h>
+			#include <Nazara/Graphics/Resources/Shaders/Modules/Engine/ViewerData.nzslb.h>
 		};
 	}
 
@@ -259,7 +261,7 @@ namespace Nz
 	template<std::size_t N>
 	void Graphics::RegisterEmbedShaderModule(const UInt8(&content)[N])
 	{
-		m_shaderModuleResolver->RegisterModule(std::string_view(reinterpret_cast<const char*>(content), N));
+		m_shaderModuleResolver->RegisterModule(ShaderAst::UnserializeShader(content, N));
 	}
 
 	void Graphics::SelectDepthStencilFormats()

src/Nazara/Shader/FilesystemModuleResolver.cpp

@@ -4,8 +4,10 @@
 
 #include <Nazara/Shader/FilesystemModuleResolver.hpp>
 #include <Nazara/Core/Error.hpp>
+#include <Nazara/Core/File.hpp>
 #include <Nazara/Core/StringExt.hpp>
 #include <Nazara/Shader/ShaderLangParser.hpp>
+#include <Nazara/Shader/Ast/AstSerializer.hpp>
 #include <efsw/efsw.h>
 #include <cassert>
 #include <Nazara/Shader/Debug.hpp>
@@ -29,9 +31,25 @@ namespace Nz
 		ShaderAst::ModulePtr module;
 		try
 		{
-			module = ShaderLang::ParseFromFile(realPath);
-			if (!module)
-				return;
+			std::string ext = realPath.extension().generic_u8string();
+			if (ext == CompiledModuleExtension)
+			{
+				Nz::File file(realPath);
+				if (!file.Open(Nz::OpenMode::ReadOnly | Nz::OpenMode::Text))
+					throw std::runtime_error("failed to open " + realPath.generic_u8string());
+
+				std::size_t length = static_cast<std::size_t>(file.GetSize());
+
+				std::vector<Nz::UInt8> content(length);
+				if (length > 0 && file.Read(&content[0], length) != length)
+					throw std::runtime_error("failed to read " + realPath.generic_u8string());
+
+				module = ShaderAst::UnserializeShader(content.data(), content.size());
+			}
+			else if (ext == ModuleExtension)
+				module = ShaderLang::ParseFromFile(realPath);
+			else
+				throw std::runtime_error("unknown extension " + ext);
 		}
 		catch (const std::exception& e)
 		{
@@ -39,6 +57,9 @@ namespace Nz
 			return;
 		}
 
+		if (!module)
+			return;
+
 		std::string moduleName = module->metadata->moduleName;
 		RegisterModule(std::move(module));
 
@@ -108,7 +129,7 @@ namespace Nz
 
 		for (const auto& entry : std::filesystem::recursive_directory_iterator(realPath))
 		{
-			if (entry.is_regular_file() && StringEqual(entry.path().extension().generic_u8string(), ModuleExtension, Nz::CaseIndependent{}))
+			if (entry.is_regular_file() && CheckExtension(entry.path().generic_u8string()))
 			{
 				try
 				{
@@ -133,7 +154,7 @@ namespace Nz
 
 	void FilesystemModuleResolver::OnFileAdded(std::string_view directory, std::string_view filename)
 	{
-		if (!EndsWith(filename, ModuleExtension))
+		if (!CheckExtension(filename))
 			return;
 
 		RegisterModule(std::filesystem::path(directory) / filename);
@@ -141,7 +162,7 @@ namespace Nz
 
 	void FilesystemModuleResolver::OnFileRemoved(std::string_view directory, std::string_view filename)
 	{
-		if (!EndsWith(filename, ModuleExtension))
+		if (!CheckExtension(filename))
 			return;
 
 		std::filesystem::path canonicalPath = std::filesystem::canonical(std::filesystem::path(directory) / filename);
@@ -156,7 +177,7 @@ namespace Nz
 
 	void FilesystemModuleResolver::OnFileMoved(std::string_view directory, std::string_view filename, std::string_view oldFilename)
 	{
-		if (oldFilename.empty() || !EndsWith(oldFilename, ModuleExtension))
+		if (oldFilename.empty() || !CheckExtension(oldFilename))
 			return;
 
 		std::filesystem::path canonicalPath = std::filesystem::canonical(std::filesystem::path(directory) / oldFilename);
@@ -174,9 +195,14 @@ namespace Nz
 
 	void FilesystemModuleResolver::OnFileUpdated(std::string_view directory, std::string_view filename)
 	{
-		if (!EndsWith(filename, ModuleExtension))
+		if (!CheckExtension(filename))
 			return;
 
 		RegisterModule(std::filesystem::path(directory) / filename);
 	}
+	
+	bool FilesystemModuleResolver::CheckExtension(std::string_view filename)
+	{
+		return EndsWith(filename, ModuleExtension, Nz::CaseIndependent{}) || EndsWith(filename, CompiledModuleExtension, Nz::CaseIndependent{});
+	}
 }

src/ShaderCompiler/main.cpp

@@ -0,0 +1,224 @@
+#include <Nazara/Core/File.hpp>
+#include <Nazara/Core/StringExt.hpp>
+#include <Nazara/Shader/LangWriter.hpp>
+#include <Nazara/Shader/ShaderLangErrors.hpp>
+#include <Nazara/Shader/ShaderLangLexer.hpp>
+#include <Nazara/Shader/ShaderLangParser.hpp>
+#include <Nazara/Shader/Ast/AstSerializer.hpp>
+#include <Nazara/Shader/Ast/SanitizeVisitor.hpp>
+#include <cxxopts.hpp>
+#include <fmt/color.h>
+#include <fmt/format.h>
+#include <filesystem>
+#include <sstream>
+#include <stdexcept>
+
+std::vector<Nz::UInt8> ReadFileContent(const std::filesystem::path& filePath)
+{
+	Nz::File file(filePath);
+	if (!file.Open(Nz::OpenMode::ReadOnly | Nz::OpenMode::Text))
+		throw std::runtime_error("failed to open " + filePath.generic_u8string());
+
+	std::size_t length = static_cast<std::size_t>(file.GetSize());
+	if (length == 0)
+		return {};
+
+	std::vector<Nz::UInt8> content(length);
+	if (file.Read(&content[0], length) != length)
+		throw std::runtime_error("failed to read " + filePath.generic_u8string());
+
+	return content;
+}
+
+std::string ReadSourceFileContent(const std::filesystem::path& filePath)
+{
+	std::vector<Nz::UInt8> content = ReadFileContent(filePath);
+	return std::string(reinterpret_cast<const char*>(&content[0]), content.size());
+}
+
+void WriteFileContent(std::filesystem::path& filePath, const void* data, std::size_t size)
+{
+	Nz::File file(filePath);
+	if (!file.Open(Nz::OpenMode::WriteOnly | Nz::OpenMode::Truncate))
+		throw std::runtime_error("failed to open " + filePath.generic_u8string());
+
+	if (file.Write(data, size) != size)
+		throw std::runtime_error("failed to write " + filePath.generic_u8string());
+}
+
+int main(int argc, char* argv[])
+{
+	cxxopts::Options options("nzslc", "Tool for validating and compiling NZSL shaders");
+	options.add_options()
+		("c,compile", "Compile input shader")
+		("output-nzsl", "Output shader as NZSL to stdout")
+		("header-file", "Generate an includable header file")
+		("i,input", "Input file(s)", cxxopts::value<std::string>())
+		("o,output", "Output path", cxxopts::value<std::string>()->default_value("."), "path")
+		("p,partial", "Allow partial compilation")
+		("s,show", "Show informations about the shader (default)")
+		("h,help", "Print usage")
+	;
+
+	options.parse_positional("input");
+	options.positional_help("shader path");
+
+	try
+	{
+		auto result = options.parse(argc, argv);
+		if (result.count("help") > 0)
+		{
+			fmt::print("{}\n", options.help());
+			return EXIT_SUCCESS;
+		}
+
+		if (result.count("input") == 0)
+		{
+			fmt::print("no input file\n{}\n", options.help());
+			return EXIT_SUCCESS;
+		}
+
+		std::filesystem::path inputPath = result["input"].as<std::string>();
+		if (!std::filesystem::is_regular_file(inputPath))
+		{
+			fmt::print("{} is not a file\n", inputPath.generic_u8string());
+			return EXIT_FAILURE;
+		}
+
+		try
+		{
+			Nz::ShaderAst::ModulePtr shaderModule;
+			if (inputPath.extension() == ".nzsl")
+			{
+				std::string sourceContent = ReadSourceFileContent(inputPath);
+
+				std::vector<Nz::ShaderLang::Token> tokens = Nz::ShaderLang::Tokenize(sourceContent, inputPath.generic_u8string());
+
+				shaderModule = Nz::ShaderLang::Parse(tokens);
+			}
+			else if (inputPath.extension() == ".nzslb")
+			{
+				std::vector<Nz::UInt8> sourceContent = ReadFileContent(inputPath);
+
+				shaderModule = Nz::ShaderAst::UnserializeShader(sourceContent.data(), sourceContent.size());
+			}
+			else
+			{
+				fmt::print("{} has unknown extension\n", inputPath.generic_u8string());
+				return EXIT_FAILURE;
+			}
+
+			if (result.count("compile") > 0)
+			{
+				Nz::ShaderAst::SanitizeVisitor::Options sanitizeOptions;
+				sanitizeOptions.allowPartialSanitization = result.count("partial") > 0;
+
+				shaderModule = Nz::ShaderAst::Sanitize(*shaderModule, sanitizeOptions);
+
+				Nz::ByteArray shaderData = Nz::ShaderAst::SerializeShader(shaderModule);
+
+				std::filesystem::path outputPath = inputPath;
+				if (result.count("header-file") > 0)
+				{
+					outputPath.replace_extension(".nzslb.h");
+
+					std::stringstream ss;
+
+					bool first = true;
+					for (std::size_t i = 0; i < shaderData.size(); ++i)
+					{
+						if (!first)
+							ss << ',';
+
+						ss << +shaderData[i];
+
+						first = false;
+					}
+
+					std::string headerFile = std::move(ss).str();
+					WriteFileContent(outputPath, headerFile.data(), headerFile.size());
+				}
+				else
+				{
+					outputPath.replace_extension(".nzslb");
+					WriteFileContent(outputPath, shaderData.GetConstBuffer(), shaderData.GetSize());
+				}
+			}
+
+			if (result.count("output-nzsl") > 0)
+			{
+				Nz::LangWriter nzslWriter;
+				fmt::print("{}", nzslWriter.Generate(*shaderModule));
+			}
+		}
+		catch (const Nz::ShaderLang::Error& error)
+		{
+			fmt::print(stderr, (fmt::emphasis::bold | fg(fmt::color::red)), "{}\n", error.what());
+
+			Nz::ShaderLang::SourceLocation errorLocation = error.GetSourceLocation();
+			if (errorLocation.IsValid())
+			{
+				try
+				{
+					// Retrieve line
+					std::string sourceContent = ReadSourceFileContent(*errorLocation.file);
+
+					std::size_t lineStartOffset = 0;
+					if (errorLocation.startLine > 1)
+					{
+						lineStartOffset = sourceContent.find('\n') + 1;
+						for (std::size_t i = 0; i < errorLocation.startLine - 2; ++i) //< remember startLine is 1-based
+						{
+							lineStartOffset = sourceContent.find('\n', lineStartOffset);
+							if (lineStartOffset == std::string::npos)
+								throw std::runtime_error("file content doesn't match original source");
+
+							++lineStartOffset;
+						}
+					}
+					std::size_t lineEndOffset = sourceContent.find('\n', lineStartOffset);
+
+					std::string errorLine = sourceContent.substr(lineStartOffset, lineEndOffset - lineStartOffset);
+
+					// handle tabs
+					Nz::UInt32 startColumn = errorLocation.startColumn - 1;
+					std::size_t startPos = 0;
+					while ((startPos = errorLine.find("\t", startPos)) != std::string::npos)
+					{
+						if (startPos < startColumn)
+							startColumn += 3;
+
+						errorLine.replace(startPos, 1, "    ");
+						startPos += 4;
+					}
+
+					std::size_t columnSize;
+					if (errorLocation.startLine == errorLocation.endLine)
+						columnSize = errorLocation.endColumn - errorLocation.startColumn + 1;
+					else
+						columnSize = 1;
+
+					std::string lineStr = std::to_string(errorLocation.startLine);
+
+					fmt::print(stderr, " {} | {}\n", lineStr, errorLine);
+					fmt::print(stderr, " {} | {}", std::string(lineStr.size(), ' '), std::string(startColumn, ' '));
+					fmt::print(stderr, fg(fmt::color::green), "{}\n", std::string(columnSize, '^'));
+				}
+				catch (const std::exception& e)
+				{
+					fmt::print(stderr, "failed to print error line: {}\n", e.what());
+				}
+			}
+		}
+	}
+	catch (const cxxopts::OptionException& e)
+	{
+		fmt::print(stderr, "{}\n{}\n", e.what(), options.help());
+	}
+	catch (const std::exception& e)
+	{
+		fmt::print(stderr, "{}\n", e.what());
+	}
+
+	return EXIT_SUCCESS;
+}

tests/Engine/Math/Matrix4Test.cpp

@@ -132,7 +132,7 @@ SCENARIO("Matrix4", "[MATH][MATRIX4]")
 			{
 				transformedMatrix.MakeTransform(Nz::Vector3f::Zero(), Nz::EulerAnglesf(0.f, 0.f, Nz::DegreeAnglef(45.f)).ToQuaternion());
 				Nz::Matrix4f rotation45Z( std::sqrt(2.f) / 2.f, std::sqrt(2.f) / 2.f, 0.f, 0.f,
-				                          -std::sqrt(2.f) / 2.f, std::sqrt(2.f) / 2.f, 0.f, 0.f,
+				                         -std::sqrt(2.f) / 2.f, std::sqrt(2.f) / 2.f, 0.f, 0.f,
 				                          0.f,                  0.f,                  1.f, 0.f,
 				                          0.f,                  0.f,                  0.f, 1.f);
 

tools/nzslc.lua

@@ -0,0 +1,18 @@
+add_requires("cxxopts")
+
+target("NazaraShaderCompiler")
+    set_group("Tools")
+    set_kind("binary")
+	set_basename("nzslc")
+
+    add_deps("NazaraShader")
+    add_packages("cxxopts", "fmt")
+
+    if has_config("unitybuild") then
+        add_rules("c++.unity_build")
+    end
+
+    add_includedirs("../src")
+    add_headerfiles("../src/ShaderCompiler/**.hpp", "../src/ShaderCompiler/**.inl")
+    add_files("../src/ShaderCompiler/**.cpp")
+    set_policy("build.across_targets_in_parallel", false)

xmake.lua

@@ -195,7 +195,6 @@ for name, module in pairs(modules) do
 	set_kind("shared")
 	set_group("Modules")
 
-	add_rules("embed_resources")
 	add_rpathdirs("$ORIGIN")
 
 	if module.Deps then
@@ -236,10 +235,26 @@ for name, module in pairs(modules) do
 	add_files("src/Nazara/" .. name .. "/**.cpp")
 	add_includedirs("src")
 
-	for _, filepath in pairs(os.files("src/Nazara/" .. name .. "/Resources/**|**.h")) do
+	local embedResourceRule = false
+	for _, filepath in pairs(os.files("src/Nazara/" .. name .. "/Resources/**|**.h|**.nzsl|**.nzslb")) do
+		if not embedResourceRule then
+			add_rules("embed_resources")
+			embedResourceRule = true
+		end
+
 		add_files(filepath, {rule = "embed_resources"})
 	end
 
+	local compileShaderRule = false
+	for _, filepath in pairs(os.files("src/Nazara/" .. name .. "/Resources/**.nzsl")) do
+		if not compileShaderRule then
+			add_rules("compile_shaders")
+			compileShaderRule = true
+		end
+
+		add_files(filepath, {rule = "compile_shaders"})
+	end
+
 	-- Remove platform-specific files
 	if is_plat("windows", "mingw") then
 		for _, ext in ipairs(headerExts) do
@@ -264,7 +279,7 @@ for name, module in pairs(modules) do
 	end
 end
 
-includes("tools/xmake.lua")
-includes("tests/xmake.lua")
+includes("tools/*.lua")
+includes("tests/*.lua")
 includes("plugins/*/xmake.lua")
 includes("examples/*/xmake.lua")

xmake/rules/compile_shaders.lua

@@ -0,0 +1,22 @@
+-- Turns resources into includables headers
+rule("compile_shaders")
+	on_load(function (target)
+		target:add("deps", "NazaraShaderCompiler")
+		target:set("policy", "build.across_targets_in_parallel", false)
+	end)
+
+	before_buildcmd_file(function (target, batchcmds, shaderfile, opt)
+		local nzslc = target:dep("NazaraShaderCompiler")
+
+		-- add commands
+		batchcmds:show_progress(opt.progress, "${color.build.object}compiling shader %s", shaderfile)
+		local argv = {"--compile", "--partial", "--header-file", shaderfile}
+		batchcmds:vrunv(nzslc:targetfile(), argv, { curdir = "." })
+
+		local outputFile = path.join(path.directory(shaderfile), path.basename(shaderfile) .. ".nzslb.h")
+
+		-- add deps
+		batchcmds:add_depfiles(shaderfile)
+		batchcmds:set_depmtime(os.mtime(outputFile))
+		batchcmds:set_depcache(target:dependfile(outputFile))
+	end)