Shader: Add module resolver + use modules for engine shaders

This commit is contained in:
Jérôme Leclercq
2022-03-10 21:00:10 +01:00
parent 98bd04e35a
commit db0c1e6e8c
30 changed files with 737 additions and 106 deletions

View File

@@ -94,5 +94,6 @@
#include <Nazara/Core/Unicode.hpp>
#include <Nazara/Core/Updatable.hpp>
#include <Nazara/Core/Uuid.hpp>
#include <Nazara/Core/VirtualDirectory.hpp>
#endif // NAZARA_GLOBAL_CORE_HPP

View File

@@ -0,0 +1,83 @@
// Copyright (C) 2022 Jérôme "Lynix" Leclercq (lynix680@gmail.com)
// This file is part of the "Nazara Engine - Core module"
// For conditions of distribution and use, see copyright notice in Config.hpp
#pragma once
#ifndef NAZARA_CORE_VIRTUALDIRECTORY_HPP
#define NAZARA_CORE_VIRTUALDIRECTORY_HPP
#include <Nazara/Prerequisites.hpp>
#include <filesystem>
#include <map>
#include <memory>
#include <optional>
#include <string>
#include <variant>
#include <vector>
namespace Nz
{
class VirtualDirectory : public std::enable_shared_from_this<VirtualDirectory>
{
public:
struct DataPointerEntry;
struct FileContentEntry;
using PhysicalFileEntry = std::filesystem::path;
using VirtualDirectoryEntry = std::shared_ptr<VirtualDirectory>;
using Entry = std::variant<DataPointerEntry, FileContentEntry, PhysicalFileEntry, VirtualDirectoryEntry>;
inline VirtualDirectory(VirtualDirectoryEntry parentDirectory = nullptr);
inline VirtualDirectory(std::filesystem::path physicalPath, VirtualDirectoryEntry parentDirectory = nullptr);
VirtualDirectory(const VirtualDirectory&) = delete;
VirtualDirectory(VirtualDirectory&&) = delete;
~VirtualDirectory() = default;
template<typename F> void Foreach(F&& cb, bool includeDots = false);
inline bool GetEntry(std::string_view path, Entry* entry);
inline VirtualDirectoryEntry& StoreDirectory(std::string_view path, VirtualDirectoryEntry directory);
inline VirtualDirectoryEntry& StoreDirectory(std::string_view path, std::filesystem::path directoryPath);
inline FileContentEntry& StoreFile(std::string_view path, std::vector<UInt8> file);
inline PhysicalFileEntry& StoreFile(std::string_view path, std::filesystem::path filePath);
inline DataPointerEntry& StoreFile(std::string_view path, const void* data, std::size_t size);
VirtualDirectory& operator=(const VirtualDirectory&) = delete;
VirtualDirectory& operator=(VirtualDirectory&&) = delete;
struct DataPointerEntry
{
const void* data;
std::size_t size;
};
struct FileContentEntry
{
std::shared_ptr<std::vector<UInt8>> data;
};
private:
inline void EnsureDots();
inline bool GetEntryInternal(std::string_view name, Entry* entry);
inline bool RetrieveDirectory(std::string_view path, bool allowCreation, std::shared_ptr<VirtualDirectory>& directory, std::string_view& entryName);
inline VirtualDirectoryEntry& StoreDirectoryInternal(std::string name, std::filesystem::path directoryPath);
inline VirtualDirectoryEntry& StoreDirectoryInternal(std::string name, VirtualDirectoryEntry directory);
inline FileContentEntry& StoreFileInternal(std::string name, std::vector<UInt8> fileContent);
inline PhysicalFileEntry& StoreFileInternal(std::string name, std::filesystem::path filePath);
inline DataPointerEntry& StoreFileInternal(std::string name, const void* data, std::size_t size);
template<typename F1, typename F2> static bool SplitPath(std::string_view path, F1&& dirCB, F2&& fileCB);
std::map<std::string /*name*/, Entry, std::less<>> m_content;
std::optional<std::filesystem::path> m_physicalPath;
VirtualDirectoryEntry m_parent;
bool m_wereDotRegistered;
};
}
#include <Nazara/Core/VirtualDirectory.inl>
#endif // NAZARA_CORE_VIRTUALDIRECTORY_HPP

View File

@@ -0,0 +1,266 @@
// Copyright (C) 2022 Jérôme "Lynix" Leclercq (lynix680@gmail.com)
// This file is part of the "Nazara Engine - Core module"
// For conditions of distribution and use, see copyright notice in Config.hpp
#include <Nazara/Core/VirtualDirectory.hpp>
#include <cassert>
#include <Nazara/Core/Debug.hpp>
namespace Nz
{
inline VirtualDirectory::VirtualDirectory(VirtualDirectoryEntry parentDirectory) :
m_parent(std::move(parentDirectory)),
m_wereDotRegistered(false)
{
}
inline VirtualDirectory::VirtualDirectory(std::filesystem::path physicalPath, VirtualDirectoryEntry parentDirectory) :
m_physicalPath(std::move(physicalPath)),
m_parent(std::move(parentDirectory)),
m_wereDotRegistered(false)
{
}
template<typename F>
void VirtualDirectory::Foreach(F&& cb, bool includeDots)
{
if (includeDots)
EnsureDots();
for (auto&& pair : m_content)
{
if (!includeDots && (pair.first == "." || pair.first == ".."))
continue;
cb(pair.first, pair.second);
}
if (m_physicalPath)
{
for (auto&& physicalEntry : std::filesystem::directory_iterator(*m_physicalPath))
{
Entry entry;
std::string filename = physicalEntry.path().filename().generic_u8string();
if (m_content.find(filename) != m_content.end())
continue; //< Physical file/directory has been overridden by a virtual one
if (physicalEntry.is_regular_file())
entry.emplace<PhysicalFileEntry>(physicalEntry.path());
else if (physicalEntry.is_directory())
{
// FIXME: Allocating a shared_ptr on iteration is bad, not sure about a workaround
entry.emplace<VirtualDirectoryEntry>(std::make_shared<VirtualDirectory>(physicalEntry.path(), shared_from_this()));
}
else
continue;
cb(physicalEntry.path().filename().generic_u8string(), entry);
}
}
}
inline bool VirtualDirectory::GetEntry(std::string_view path, Entry* entry)
{
std::shared_ptr<VirtualDirectory> dir;
std::string_view entryName;
if (!RetrieveDirectory(path, false, dir, entryName))
return false;
if (!dir->GetEntryInternal(entryName, entry))
return false;
return true;
}
inline auto VirtualDirectory::StoreDirectory(std::string_view path, VirtualDirectoryEntry directory) -> VirtualDirectoryEntry&
{
std::shared_ptr<VirtualDirectory> dir;
std::string_view entryName;
if (!RetrieveDirectory(path, true, dir, entryName))
throw std::runtime_error("invalid path");
return dir->StoreDirectoryInternal(std::string(entryName), std::move(directory));
}
inline auto VirtualDirectory::StoreDirectory(std::string_view path, std::filesystem::path directoryPath) -> VirtualDirectoryEntry&
{
std::shared_ptr<VirtualDirectory> dir;
std::string_view entryName;
if (!RetrieveDirectory(path, true, dir, entryName))
throw std::runtime_error("invalid path");
return dir->StoreDirectoryInternal(std::string(entryName), std::move(directoryPath));
}
inline auto VirtualDirectory::StoreFile(std::string_view path, std::vector<UInt8> file) -> FileContentEntry&
{
std::shared_ptr<VirtualDirectory> dir;
std::string_view entryName;
if (!RetrieveDirectory(path, true, dir, entryName))
throw std::runtime_error("invalid path");
return dir->StoreFileInternal(std::string(entryName), std::move(file));
}
inline auto VirtualDirectory::StoreFile(std::string_view path, std::filesystem::path filePath) -> PhysicalFileEntry&
{
std::shared_ptr<VirtualDirectory> dir;
std::string_view entryName;
if (!RetrieveDirectory(path, true, dir, entryName))
throw std::runtime_error("invalid path");
return dir->StoreFileInternal(std::string(entryName), std::move(filePath));
}
inline auto VirtualDirectory::StoreFile(std::string_view path, const void* data, std::size_t size) -> DataPointerEntry&
{
std::shared_ptr<VirtualDirectory> dir;
std::string_view entryName;
if (!RetrieveDirectory(path, true, dir, entryName))
throw std::runtime_error("invalid path");
return dir->StoreFileInternal(std::string(entryName), data, size);
}
inline void VirtualDirectory::EnsureDots()
{
if (!m_wereDotRegistered)
{
StoreDirectoryInternal(".", shared_from_this());
if (m_parent)
StoreDirectoryInternal("..", m_parent);
else
StoreDirectoryInternal("..", shared_from_this());
m_wereDotRegistered = true;
}
}
inline bool VirtualDirectory::RetrieveDirectory(std::string_view path, bool allowCreation, std::shared_ptr<VirtualDirectory>& directory, std::string_view& entryName)
{
directory = shared_from_this();
return SplitPath(path, [&](std::string_view dirName)
{
Entry entry;
if (directory->GetEntryInternal(dirName, &entry))
{
if (auto dir = std::get_if<VirtualDirectoryEntry>(&entry))
directory = *dir;
else
return false;
}
else
{
if (allowCreation)
{
auto newDirectory = std::make_shared<VirtualDirectory>(directory);
directory = directory->StoreDirectoryInternal(std::string(dirName), newDirectory);
}
else
return false;
}
return true;
},
[&](std::string_view name)
{
entryName = name;
});
}
inline bool VirtualDirectory::GetEntryInternal(std::string_view name, Entry* entry)
{
EnsureDots();
auto it = m_content.find(name);
if (it != m_content.end())
{
*entry = it->second;
return true;
}
else
{
if (m_physicalPath)
{
std::filesystem::path entryPath = *m_physicalPath / name;
if (std::filesystem::is_regular_file(entryPath))
entry->emplace<PhysicalFileEntry>(entryPath);
else if (std::filesystem::is_directory(entryPath))
{
// FIXME: Allocating a shared_ptr on iteration is bad, not sure about a workaround
*entry = StoreDirectoryInternal(std::string(name), entryPath);
}
else
return false;
return true;
}
return false;
}
}
inline auto VirtualDirectory::StoreDirectoryInternal(std::string name, std::filesystem::path directoryPath) -> VirtualDirectoryEntry&
{
assert(name.find_first_of("\\/:") == name.npos);
auto it = m_content.insert_or_assign(std::move(name), std::make_shared<VirtualDirectory>(directoryPath, shared_from_this())).first;
return std::get<VirtualDirectoryEntry>(it->second);
}
inline auto VirtualDirectory::StoreDirectoryInternal(std::string name, VirtualDirectoryEntry directory) -> VirtualDirectoryEntry&
{
assert(name.find_first_of("\\/:") == name.npos);
auto it = m_content.insert_or_assign(std::move(name), std::move(directory)).first;
return std::get<VirtualDirectoryEntry>(it->second);
}
inline auto VirtualDirectory::StoreFileInternal(std::string name, std::vector<UInt8> fileContent) -> FileContentEntry&
{
assert(name.find_first_of("\\/:") == name.npos);
FileContentEntry fileEntry;
fileEntry.data = std::make_shared<std::vector<UInt8>>(std::move(fileContent));
auto it = m_content.insert_or_assign(std::move(name), std::move(fileEntry)).first;
return std::get<FileContentEntry>(it->second);
}
inline auto VirtualDirectory::StoreFileInternal(std::string name, std::filesystem::path filePath) -> PhysicalFileEntry&
{
assert(name.find_first_of("\\/:") == name.npos);
auto it = m_content.insert_or_assign(std::move(name), std::move(filePath)).first;
return std::get<PhysicalFileEntry>(it->second);
}
inline auto VirtualDirectory::StoreFileInternal(std::string name, const void* data, std::size_t size) -> DataPointerEntry&
{
assert(name.find_first_of("\\/:") == name.npos);
auto it = m_content.insert_or_assign(std::move(name), DataPointerEntry{ data, size }).first;
return std::get<DataPointerEntry>(it->second);
}
template<typename F1, typename F2>
inline bool VirtualDirectory::SplitPath(std::string_view path, F1&& dirCB, F2&& lastCB)
{
std::size_t pos;
while ((pos = path.find_first_of("\\/:")) != std::string::npos)
{
if (!dirCB(path.substr(0, pos)))
return false;
path = path.substr(pos + 1);
}
lastCB(path);
return true;
}
}
#include <Nazara/Core/DebugOff.hpp>

View File

@@ -15,6 +15,7 @@
#include <Nazara/Renderer/RenderPassCache.hpp>
#include <Nazara/Renderer/RenderPipelineLayout.hpp>
#include <Nazara/Renderer/Renderer.hpp>
#include <Nazara/Shader/DirectoryModuleResolver.hpp>
#include <optional>
namespace Nz
@@ -45,6 +46,7 @@ namespace Nz
inline const std::shared_ptr<RenderDevice>& GetRenderDevice() const;
inline const RenderPassCache& GetRenderPassCache() const;
inline TextureSamplerCache& GetSamplerCache();
inline const std::shared_ptr<DirectoryModuleResolver>& GetShaderModuleResolver() const;
struct Config
{
@@ -62,10 +64,12 @@ namespace Nz
void BuildDefaultTextures();
void BuildFullscreenVertexBuffer();
void RegisterMaterialPasses();
void RegisterShaderModules();
void SelectDepthStencilFormats();
std::optional<RenderPassCache> m_renderPassCache;
std::optional<TextureSamplerCache> m_samplerCache;
std::shared_ptr<DirectoryModuleResolver> m_shaderModuleResolver;
std::shared_ptr<RenderBuffer> m_fullscreenVertexBuffer;
std::shared_ptr<RenderDevice> m_renderDevice;
std::shared_ptr<RenderPipeline> m_blitPipeline;

View File

@@ -62,6 +62,11 @@ namespace Nz
{
return *m_samplerCache;
}
inline const std::shared_ptr<DirectoryModuleResolver>& Graphics::GetShaderModuleResolver() const
{
return m_shaderModuleResolver;
}
}
#include <Nazara/Graphics/DebugOff.hpp>

View File

@@ -30,12 +30,14 @@
#define NAZARA_GLOBAL_SHADER_HPP
#include <Nazara/Shader/Config.hpp>
#include <Nazara/Shader/DirectoryModuleResolver.hpp>
#include <Nazara/Shader/GlslWriter.hpp>
#include <Nazara/Shader/LangWriter.hpp>
#include <Nazara/Shader/Shader.hpp>
#include <Nazara/Shader/ShaderBuilder.hpp>
#include <Nazara/Shader/ShaderLangLexer.hpp>
#include <Nazara/Shader/ShaderLangParser.hpp>
#include <Nazara/Shader/ShaderModuleResolver.hpp>
#include <Nazara/Shader/ShaderWriter.hpp>
#include <Nazara/Shader/SpirvAstVisitor.hpp>
#include <Nazara/Shader/SpirvBlock.hpp>

View File

@@ -4,8 +4,8 @@
#pragma once
#ifndef NAZARA_SHADER_AST_INDEXREMAPPER_HPP
#define NAZARA_SHADER_AST_INDEXREMAPPER_HPP
#ifndef NAZARA_SHADER_AST_INDEXREMAPPERVISITOR_HPP
#define NAZARA_SHADER_AST_INDEXREMAPPERVISITOR_HPP
#include <Nazara/Prerequisites.hpp>
#include <Nazara/Shader/Config.hpp>
@@ -60,4 +60,4 @@ namespace Nz::ShaderAst
#include <Nazara/Shader/Ast/IndexRemapperVisitor.inl>
#endif // NAZARA_SHADER_AST_INDEXREMAPPER_HPP
#endif // NAZARA_SHADER_AST_INDEXREMAPPERVISITOR_HPP

View File

@@ -10,6 +10,7 @@
#include <Nazara/Prerequisites.hpp>
#include <Nazara/Core/Bitset.hpp>
#include <Nazara/Shader/Config.hpp>
#include <Nazara/Shader/ShaderModuleResolver.hpp>
#include <Nazara/Shader/Ast/AstCloner.hpp>
#include <Nazara/Shader/Ast/AstTypes.hpp>
#include <Nazara/Shader/Ast/Module.hpp>
@@ -42,7 +43,7 @@ namespace Nz::ShaderAst
struct Options
{
std::function<ModulePtr(const std::vector<std::string>& /*modulePath*/)> moduleCallback;
std::shared_ptr<ShaderModuleResolver> moduleResolver;
std::unordered_set<std::string> reservedIdentifiers;
std::unordered_map<UInt32, ConstantValue> optionValues;
bool makeVariableNameUnique = false;

View File

@@ -0,0 +1,45 @@
// Copyright (C) 2022 Jérôme "Lynix" Leclercq (lynix680@gmail.com)
// This file is part of the "Nazara Engine - Shader module"
// For conditions of distribution and use, see copyright notice in Config.hpp
#pragma once
#ifndef NAZARA_SHADER_DIRECTORYMODULERESOLVER_HPP
#define NAZARA_SHADER_DIRECTORYMODULERESOLVER_HPP
#include <Nazara/Prerequisites.hpp>
#include <Nazara/Core/VirtualDirectory.hpp>
#include <Nazara/Shader/Config.hpp>
#include <Nazara/Shader/ShaderModuleResolver.hpp>
#include <string>
#include <unordered_map>
namespace Nz
{
class NAZARA_SHADER_API DirectoryModuleResolver : public ShaderModuleResolver
{
public:
inline DirectoryModuleResolver();
DirectoryModuleResolver(const DirectoryModuleResolver&) = delete;
DirectoryModuleResolver(DirectoryModuleResolver&&) = delete;
~DirectoryModuleResolver() = default;
inline void RegisterModuleDirectory(std::string_view path, std::filesystem::path realPath);
inline void RegisterModuleFile(std::string_view path, std::filesystem::path realPath);
inline void RegisterModuleFile(std::string_view path, std::vector<UInt8> fileContent);
inline void RegisterModuleFile(std::string_view path, const void* staticData, std::size_t size);
ShaderAst::ModulePtr Resolve(const std::vector<std::string>& modulePath) override;
DirectoryModuleResolver& operator=(const DirectoryModuleResolver&) = delete;
DirectoryModuleResolver& operator=(DirectoryModuleResolver&&) = delete;
private:
std::shared_ptr<VirtualDirectory> m_searchDirectory;
std::unordered_map<std::string, ShaderAst::ModulePtr> m_knownModules;
};
}
#include <Nazara/Shader/DirectoryModuleResolver.inl>
#endif // NAZARA_SHADER_DIRECTORYMODULERESOLVER_HPP

View File

@@ -0,0 +1,36 @@
// Copyright (C) 2022 Jérôme "Lynix" Leclercq (lynix680@gmail.com)
// This file is part of the "Nazara Engine - Shader module"
// For conditions of distribution and use, see copyright notice in Config.hpp
#include <Nazara/Shader/DirectoryModuleResolver.hpp>
#include <Nazara/Shader/Debug.hpp>
namespace Nz
{
inline DirectoryModuleResolver::DirectoryModuleResolver() :
m_searchDirectory(std::make_shared<VirtualDirectory>())
{
}
inline void DirectoryModuleResolver::RegisterModuleDirectory(std::string_view path, std::filesystem::path realPath)
{
m_searchDirectory->StoreDirectory(path, realPath);
}
inline void DirectoryModuleResolver::RegisterModuleFile(std::string_view path, std::filesystem::path realPath)
{
m_searchDirectory->StoreFile(path, realPath);
}
inline void DirectoryModuleResolver::RegisterModuleFile(std::string_view path, std::vector<UInt8> fileContent)
{
m_searchDirectory->StoreFile(path, std::move(fileContent));
}
inline void DirectoryModuleResolver::RegisterModuleFile(std::string_view path, const void* staticData, std::size_t size)
{
m_searchDirectory->StoreFile(path, staticData, size);
}
}
#include <Nazara/Shader/DebugOff.hpp>

View File

@@ -13,6 +13,7 @@
#include <Nazara/Shader/Ast/AstExpressionVisitorExcept.hpp>
#include <Nazara/Shader/Ast/AstStatementVisitorExcept.hpp>
#include <Nazara/Shader/Ast/Module.hpp>
#include <Nazara/Shader/Ast/SanitizeVisitor.hpp>
#include <set>
#include <sstream>
#include <string>
@@ -48,7 +49,7 @@ namespace Nz
};
static const char* GetFlipYUniformName();
static ShaderAst::ModulePtr Sanitize(const ShaderAst::Module& module, std::unordered_map<UInt32, ShaderAst::ConstantValue> optionValues, std::string* error = nullptr);
static ShaderAst::SanitizeVisitor::Options GetSanitizeOptions();
private:
void Append(const ShaderAst::AliasType& aliasType);

View File

@@ -0,0 +1,40 @@
// Copyright (C) 2022 Jérôme "Lynix" Leclercq (lynix680@gmail.com)
// This file is part of the "Nazara Engine - Shader module"
// For conditions of distribution and use, see copyright notice in Config.hpp
#pragma once
#ifndef NAZARA_SHADER_SHADERMODULERESOLVER_HPP
#define NAZARA_SHADER_SHADERMODULERESOLVER_HPP
#include <Nazara/Prerequisites.hpp>
#include <Nazara/Shader/Config.hpp>
#include <memory>
#include <string>
#include <vector>
namespace Nz
{
namespace ShaderAst
{
using ModulePtr = std::shared_ptr<class Module>;
}
class NAZARA_SHADER_API ShaderModuleResolver
{
public:
ShaderModuleResolver() = default;
ShaderModuleResolver(const ShaderModuleResolver&) = default;
ShaderModuleResolver(ShaderModuleResolver&&) = default;
virtual ~ShaderModuleResolver();
virtual ShaderAst::ModulePtr Resolve(const std::vector<std::string>& /*modulePath*/) = 0;
ShaderModuleResolver& operator=(const ShaderModuleResolver&) = default;
ShaderModuleResolver& operator=(ShaderModuleResolver&&) = default;
};
}
#include <Nazara/Shader/ShaderModuleResolver.inl>
#endif // NAZARA_SHADER_SHADERMODULERESOLVER_HPP

View File

@@ -0,0 +1,12 @@
// Copyright (C) 2022 Jérôme "Lynix" Leclercq (lynix680@gmail.com)
// This file is part of the "Nazara Engine - Shader module"
// For conditions of distribution and use, see copyright notice in Config.hpp
#include <Nazara/Shader/ShaderModuleResolver.hpp>
#include <Nazara/Shader/Debug.hpp>
namespace Nz
{
}
#include <Nazara/Shader/DebugOff.hpp>

View File

@@ -10,11 +10,14 @@
#include <Nazara/Prerequisites.hpp>
#include <Nazara/Shader/Config.hpp>
#include <Nazara/Shader/Ast/ConstantValue.hpp>
#include <memory>
#include <string>
#include <unordered_map>
namespace Nz
{
class ShaderModuleResolver;
class NAZARA_SHADER_API ShaderWriter
{
public:
@@ -27,6 +30,7 @@ namespace Nz
struct States
{
std::shared_ptr<ShaderModuleResolver> shaderModuleResolver;
std::unordered_map<UInt32, ShaderAst::ConstantValue> optionValues;
bool optimize = false;
bool sanitized = false;