From 8784ec9b4712e2c2d8efabee59b3839243e7ed7f Mon Sep 17 00:00:00 2001 From: SirLynix Date: Wed, 6 Apr 2022 09:04:09 +0200 Subject: [PATCH] Add shader compiler (nzslc) and use it --- .../Shader/FilesystemModuleResolver.hpp | 3 + src/Nazara/Graphics/Graphics.cpp | 18 +- .../Shader/FilesystemModuleResolver.cpp | 42 +++- src/ShaderCompiler/main.cpp | 224 ++++++++++++++++++ tests/Engine/Math/Matrix4Test.cpp | 2 +- tools/nzslc.lua | 18 ++ tools/{xmake.lua => shadernodes.lua} | 0 xmake.lua | 23 +- xmake/rules/compile_shaders.lua | 22 ++ 9 files changed, 331 insertions(+), 21 deletions(-) create mode 100644 src/ShaderCompiler/main.cpp create mode 100644 tools/nzslc.lua rename tools/{xmake.lua => shadernodes.lua} (100%) create mode 100644 xmake/rules/compile_shaders.lua diff --git a/include/Nazara/Shader/FilesystemModuleResolver.hpp b/include/Nazara/Shader/FilesystemModuleResolver.hpp index 15ad52086..b5bd7dff4 100644 --- a/include/Nazara/Shader/FilesystemModuleResolver.hpp +++ b/include/Nazara/Shader/FilesystemModuleResolver.hpp @@ -36,6 +36,7 @@ namespace Nz FilesystemModuleResolver& operator=(const FilesystemModuleResolver&) = delete; FilesystemModuleResolver& operator=(FilesystemModuleResolver&&) noexcept = delete; + static constexpr const char* CompiledModuleExtension = ".nzslb"; static constexpr const char* ModuleExtension = ".nzsl"; private: @@ -44,6 +45,8 @@ namespace Nz void OnFileMoved(std::string_view directory, std::string_view filename, std::string_view oldFilename); void OnFileUpdated(std::string_view directory, std::string_view filename); + static bool CheckExtension(std::string_view filename); + std::unordered_map m_moduleByFilepath; std::unordered_map m_modules; MovablePtr m_fileWatcher; diff --git a/src/Nazara/Graphics/Graphics.cpp b/src/Nazara/Graphics/Graphics.cpp index d5063d101..b796caf85 100644 --- a/src/Nazara/Graphics/Graphics.cpp +++ b/src/Nazara/Graphics/Graphics.cpp @@ -6,6 +6,8 @@ #include #include #include +#include +#include #include #include #include @@ -16,31 +18,31 @@ namespace Nz namespace { const UInt8 r_textureBlitShader[] = { - #include + #include }; const UInt8 r_basicMaterialShader[] = { - #include + #include }; const UInt8 r_depthMaterialShader[] = { - #include + #include }; const UInt8 r_phongMaterialShader[] = { - #include + #include }; const UInt8 r_instanceDataModule[] = { - #include + #include }; const UInt8 r_lightDataModule[] = { - #include + #include }; const UInt8 r_viewerDataModule[] = { - #include + #include }; } @@ -259,7 +261,7 @@ namespace Nz template void Graphics::RegisterEmbedShaderModule(const UInt8(&content)[N]) { - m_shaderModuleResolver->RegisterModule(std::string_view(reinterpret_cast(content), N)); + m_shaderModuleResolver->RegisterModule(ShaderAst::UnserializeShader(content, N)); } void Graphics::SelectDepthStencilFormats() diff --git a/src/Nazara/Shader/FilesystemModuleResolver.cpp b/src/Nazara/Shader/FilesystemModuleResolver.cpp index cdd92daee..5350070c3 100644 --- a/src/Nazara/Shader/FilesystemModuleResolver.cpp +++ b/src/Nazara/Shader/FilesystemModuleResolver.cpp @@ -4,8 +4,10 @@ #include #include +#include #include #include +#include #include #include #include @@ -29,9 +31,25 @@ namespace Nz ShaderAst::ModulePtr module; try { - module = ShaderLang::ParseFromFile(realPath); - if (!module) - return; + std::string ext = realPath.extension().generic_u8string(); + if (ext == CompiledModuleExtension) + { + Nz::File file(realPath); + if (!file.Open(Nz::OpenMode::ReadOnly | Nz::OpenMode::Text)) + throw std::runtime_error("failed to open " + realPath.generic_u8string()); + + std::size_t length = static_cast(file.GetSize()); + + std::vector content(length); + if (length > 0 && file.Read(&content[0], length) != length) + throw std::runtime_error("failed to read " + realPath.generic_u8string()); + + module = ShaderAst::UnserializeShader(content.data(), content.size()); + } + else if (ext == ModuleExtension) + module = ShaderLang::ParseFromFile(realPath); + else + throw std::runtime_error("unknown extension " + ext); } catch (const std::exception& e) { @@ -39,6 +57,9 @@ namespace Nz return; } + if (!module) + return; + std::string moduleName = module->metadata->moduleName; RegisterModule(std::move(module)); @@ -108,7 +129,7 @@ namespace Nz for (const auto& entry : std::filesystem::recursive_directory_iterator(realPath)) { - if (entry.is_regular_file() && StringEqual(entry.path().extension().generic_u8string(), ModuleExtension, Nz::CaseIndependent{})) + if (entry.is_regular_file() && CheckExtension(entry.path().generic_u8string())) { try { @@ -133,7 +154,7 @@ namespace Nz void FilesystemModuleResolver::OnFileAdded(std::string_view directory, std::string_view filename) { - if (!EndsWith(filename, ModuleExtension)) + if (!CheckExtension(filename)) return; RegisterModule(std::filesystem::path(directory) / filename); @@ -141,7 +162,7 @@ namespace Nz void FilesystemModuleResolver::OnFileRemoved(std::string_view directory, std::string_view filename) { - if (!EndsWith(filename, ModuleExtension)) + if (!CheckExtension(filename)) return; std::filesystem::path canonicalPath = std::filesystem::canonical(std::filesystem::path(directory) / filename); @@ -156,7 +177,7 @@ namespace Nz void FilesystemModuleResolver::OnFileMoved(std::string_view directory, std::string_view filename, std::string_view oldFilename) { - if (oldFilename.empty() || !EndsWith(oldFilename, ModuleExtension)) + if (oldFilename.empty() || !CheckExtension(oldFilename)) return; std::filesystem::path canonicalPath = std::filesystem::canonical(std::filesystem::path(directory) / oldFilename); @@ -174,9 +195,14 @@ namespace Nz void FilesystemModuleResolver::OnFileUpdated(std::string_view directory, std::string_view filename) { - if (!EndsWith(filename, ModuleExtension)) + if (!CheckExtension(filename)) return; RegisterModule(std::filesystem::path(directory) / filename); } + + bool FilesystemModuleResolver::CheckExtension(std::string_view filename) + { + return EndsWith(filename, ModuleExtension, Nz::CaseIndependent{}) || EndsWith(filename, CompiledModuleExtension, Nz::CaseIndependent{}); + } } diff --git a/src/ShaderCompiler/main.cpp b/src/ShaderCompiler/main.cpp new file mode 100644 index 000000000..1ad81d835 --- /dev/null +++ b/src/ShaderCompiler/main.cpp @@ -0,0 +1,224 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +std::vector ReadFileContent(const std::filesystem::path& filePath) +{ + Nz::File file(filePath); + if (!file.Open(Nz::OpenMode::ReadOnly | Nz::OpenMode::Text)) + throw std::runtime_error("failed to open " + filePath.generic_u8string()); + + std::size_t length = static_cast(file.GetSize()); + if (length == 0) + return {}; + + std::vector content(length); + if (file.Read(&content[0], length) != length) + throw std::runtime_error("failed to read " + filePath.generic_u8string()); + + return content; +} + +std::string ReadSourceFileContent(const std::filesystem::path& filePath) +{ + std::vector content = ReadFileContent(filePath); + return std::string(reinterpret_cast(&content[0]), content.size()); +} + +void WriteFileContent(std::filesystem::path& filePath, const void* data, std::size_t size) +{ + Nz::File file(filePath); + if (!file.Open(Nz::OpenMode::WriteOnly | Nz::OpenMode::Truncate)) + throw std::runtime_error("failed to open " + filePath.generic_u8string()); + + if (file.Write(data, size) != size) + throw std::runtime_error("failed to write " + filePath.generic_u8string()); +} + +int main(int argc, char* argv[]) +{ + cxxopts::Options options("nzslc", "Tool for validating and compiling NZSL shaders"); + options.add_options() + ("c,compile", "Compile input shader") + ("output-nzsl", "Output shader as NZSL to stdout") + ("header-file", "Generate an includable header file") + ("i,input", "Input file(s)", cxxopts::value()) + ("o,output", "Output path", cxxopts::value()->default_value("."), "path") + ("p,partial", "Allow partial compilation") + ("s,show", "Show informations about the shader (default)") + ("h,help", "Print usage") + ; + + options.parse_positional("input"); + options.positional_help("shader path"); + + try + { + auto result = options.parse(argc, argv); + if (result.count("help") > 0) + { + fmt::print("{}\n", options.help()); + return EXIT_SUCCESS; + } + + if (result.count("input") == 0) + { + fmt::print("no input file\n{}\n", options.help()); + return EXIT_SUCCESS; + } + + std::filesystem::path inputPath = result["input"].as(); + if (!std::filesystem::is_regular_file(inputPath)) + { + fmt::print("{} is not a file\n", inputPath.generic_u8string()); + return EXIT_FAILURE; + } + + try + { + Nz::ShaderAst::ModulePtr shaderModule; + if (inputPath.extension() == ".nzsl") + { + std::string sourceContent = ReadSourceFileContent(inputPath); + + std::vector tokens = Nz::ShaderLang::Tokenize(sourceContent, inputPath.generic_u8string()); + + shaderModule = Nz::ShaderLang::Parse(tokens); + } + else if (inputPath.extension() == ".nzslb") + { + std::vector sourceContent = ReadFileContent(inputPath); + + shaderModule = Nz::ShaderAst::UnserializeShader(sourceContent.data(), sourceContent.size()); + } + else + { + fmt::print("{} has unknown extension\n", inputPath.generic_u8string()); + return EXIT_FAILURE; + } + + if (result.count("compile") > 0) + { + Nz::ShaderAst::SanitizeVisitor::Options sanitizeOptions; + sanitizeOptions.allowPartialSanitization = result.count("partial") > 0; + + shaderModule = Nz::ShaderAst::Sanitize(*shaderModule, sanitizeOptions); + + Nz::ByteArray shaderData = Nz::ShaderAst::SerializeShader(shaderModule); + + std::filesystem::path outputPath = inputPath; + if (result.count("header-file") > 0) + { + outputPath.replace_extension(".nzslb.h"); + + std::stringstream ss; + + bool first = true; + for (std::size_t i = 0; i < shaderData.size(); ++i) + { + if (!first) + ss << ','; + + ss << +shaderData[i]; + + first = false; + } + + std::string headerFile = std::move(ss).str(); + WriteFileContent(outputPath, headerFile.data(), headerFile.size()); + } + else + { + outputPath.replace_extension(".nzslb"); + WriteFileContent(outputPath, shaderData.GetConstBuffer(), shaderData.GetSize()); + } + } + + if (result.count("output-nzsl") > 0) + { + Nz::LangWriter nzslWriter; + fmt::print("{}", nzslWriter.Generate(*shaderModule)); + } + } + catch (const Nz::ShaderLang::Error& error) + { + fmt::print(stderr, (fmt::emphasis::bold | fg(fmt::color::red)), "{}\n", error.what()); + + Nz::ShaderLang::SourceLocation errorLocation = error.GetSourceLocation(); + if (errorLocation.IsValid()) + { + try + { + // Retrieve line + std::string sourceContent = ReadSourceFileContent(*errorLocation.file); + + std::size_t lineStartOffset = 0; + if (errorLocation.startLine > 1) + { + lineStartOffset = sourceContent.find('\n') + 1; + for (std::size_t i = 0; i < errorLocation.startLine - 2; ++i) //< remember startLine is 1-based + { + lineStartOffset = sourceContent.find('\n', lineStartOffset); + if (lineStartOffset == std::string::npos) + throw std::runtime_error("file content doesn't match original source"); + + ++lineStartOffset; + } + } + std::size_t lineEndOffset = sourceContent.find('\n', lineStartOffset); + + std::string errorLine = sourceContent.substr(lineStartOffset, lineEndOffset - lineStartOffset); + + // handle tabs + Nz::UInt32 startColumn = errorLocation.startColumn - 1; + std::size_t startPos = 0; + while ((startPos = errorLine.find("\t", startPos)) != std::string::npos) + { + if (startPos < startColumn) + startColumn += 3; + + errorLine.replace(startPos, 1, " "); + startPos += 4; + } + + std::size_t columnSize; + if (errorLocation.startLine == errorLocation.endLine) + columnSize = errorLocation.endColumn - errorLocation.startColumn + 1; + else + columnSize = 1; + + std::string lineStr = std::to_string(errorLocation.startLine); + + fmt::print(stderr, " {} | {}\n", lineStr, errorLine); + fmt::print(stderr, " {} | {}", std::string(lineStr.size(), ' '), std::string(startColumn, ' ')); + fmt::print(stderr, fg(fmt::color::green), "{}\n", std::string(columnSize, '^')); + } + catch (const std::exception& e) + { + fmt::print(stderr, "failed to print error line: {}\n", e.what()); + } + } + } + } + catch (const cxxopts::OptionException& e) + { + fmt::print(stderr, "{}\n{}\n", e.what(), options.help()); + } + catch (const std::exception& e) + { + fmt::print(stderr, "{}\n", e.what()); + } + + return EXIT_SUCCESS; +} diff --git a/tests/Engine/Math/Matrix4Test.cpp b/tests/Engine/Math/Matrix4Test.cpp index a1e57e3ea..879ce21d6 100644 --- a/tests/Engine/Math/Matrix4Test.cpp +++ b/tests/Engine/Math/Matrix4Test.cpp @@ -132,7 +132,7 @@ SCENARIO("Matrix4", "[MATH][MATRIX4]") { transformedMatrix.MakeTransform(Nz::Vector3f::Zero(), Nz::EulerAnglesf(0.f, 0.f, Nz::DegreeAnglef(45.f)).ToQuaternion()); Nz::Matrix4f rotation45Z( std::sqrt(2.f) / 2.f, std::sqrt(2.f) / 2.f, 0.f, 0.f, - -std::sqrt(2.f) / 2.f, std::sqrt(2.f) / 2.f, 0.f, 0.f, + -std::sqrt(2.f) / 2.f, std::sqrt(2.f) / 2.f, 0.f, 0.f, 0.f, 0.f, 1.f, 0.f, 0.f, 0.f, 0.f, 1.f); diff --git a/tools/nzslc.lua b/tools/nzslc.lua new file mode 100644 index 000000000..c8c860f78 --- /dev/null +++ b/tools/nzslc.lua @@ -0,0 +1,18 @@ +add_requires("cxxopts") + +target("NazaraShaderCompiler") + set_group("Tools") + set_kind("binary") + set_basename("nzslc") + + add_deps("NazaraShader") + add_packages("cxxopts", "fmt") + + if has_config("unitybuild") then + add_rules("c++.unity_build") + end + + add_includedirs("../src") + add_headerfiles("../src/ShaderCompiler/**.hpp", "../src/ShaderCompiler/**.inl") + add_files("../src/ShaderCompiler/**.cpp") + set_policy("build.across_targets_in_parallel", false) diff --git a/tools/xmake.lua b/tools/shadernodes.lua similarity index 100% rename from tools/xmake.lua rename to tools/shadernodes.lua diff --git a/xmake.lua b/xmake.lua index 2999bb827..e0dd2b1d8 100644 --- a/xmake.lua +++ b/xmake.lua @@ -195,7 +195,6 @@ for name, module in pairs(modules) do set_kind("shared") set_group("Modules") - add_rules("embed_resources") add_rpathdirs("$ORIGIN") if module.Deps then @@ -236,10 +235,26 @@ for name, module in pairs(modules) do add_files("src/Nazara/" .. name .. "/**.cpp") add_includedirs("src") - for _, filepath in pairs(os.files("src/Nazara/" .. name .. "/Resources/**|**.h")) do + local embedResourceRule = false + for _, filepath in pairs(os.files("src/Nazara/" .. name .. "/Resources/**|**.h|**.nzsl|**.nzslb")) do + if not embedResourceRule then + add_rules("embed_resources") + embedResourceRule = true + end + add_files(filepath, {rule = "embed_resources"}) end + local compileShaderRule = false + for _, filepath in pairs(os.files("src/Nazara/" .. name .. "/Resources/**.nzsl")) do + if not compileShaderRule then + add_rules("compile_shaders") + compileShaderRule = true + end + + add_files(filepath, {rule = "compile_shaders"}) + end + -- Remove platform-specific files if is_plat("windows", "mingw") then for _, ext in ipairs(headerExts) do @@ -264,7 +279,7 @@ for name, module in pairs(modules) do end end -includes("tools/xmake.lua") -includes("tests/xmake.lua") +includes("tools/*.lua") +includes("tests/*.lua") includes("plugins/*/xmake.lua") includes("examples/*/xmake.lua") diff --git a/xmake/rules/compile_shaders.lua b/xmake/rules/compile_shaders.lua new file mode 100644 index 000000000..402a90aba --- /dev/null +++ b/xmake/rules/compile_shaders.lua @@ -0,0 +1,22 @@ +-- Turns resources into includables headers +rule("compile_shaders") + on_load(function (target) + target:add("deps", "NazaraShaderCompiler") + target:set("policy", "build.across_targets_in_parallel", false) + end) + + before_buildcmd_file(function (target, batchcmds, shaderfile, opt) + local nzslc = target:dep("NazaraShaderCompiler") + + -- add commands + batchcmds:show_progress(opt.progress, "${color.build.object}compiling shader %s", shaderfile) + local argv = {"--compile", "--partial", "--header-file", shaderfile} + batchcmds:vrunv(nzslc:targetfile(), argv, { curdir = "." }) + + local outputFile = path.join(path.directory(shaderfile), path.basename(shaderfile) .. ".nzslb.h") + + -- add deps + batchcmds:add_depfiles(shaderfile) + batchcmds:set_depmtime(os.mtime(outputFile)) + batchcmds:set_depcache(target:dependfile(outputFile)) + end)