sid
/
NazaraEngine
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Initial shaderlang commit

This commit is contained in:
Jérôme Leclercq 2021-02-24 23:51:24 +01:00
parent d59423afca
commit 9a0f201433
7 changed files with 623 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

56
include/Nazara/Shader/ShaderLangLexer.hpp Normal file
View File

@ -0,0 +1,56 @@
// Copyright (C) 2020 Jérôme Leclercq
// This file is part of the "Nazara Engine - Renderer module"
// For conditions of distribution and use, see copyright notice in Config.hpp
#pragma once
#ifndef NAZARA_SHADER_LANGLEXER_HPP
#define NAZARA_SHADER_LANGLEXER_HPP
#include <Nazara/Prerequisites.hpp>
#include <Nazara/Shader/Config.hpp>
#include <stdexcept>
#include <string>
#include <variant>
#include <vector>
namespace Nz::ShaderLang
{
enum class TokenType
{
#define NAZARA_SHADERLANG_TOKEN(X) X,
#include <Nazara/Shader/ShaderLangTokenList.hpp>
};
struct Token
{
unsigned int column;
unsigned int line;
TokenType type;
std::variant<double, long long, std::string> data;
};
class BadNumber : public std::exception
{
using exception::exception;
};
class NumberOutOfRange : public std::exception
{
using exception::exception;
};
class UnrecognizedToken : public std::exception
{
using exception::exception;
};
NAZARA_SHADER_API std::vector<Token> Tokenize(const std::string_view& str);
NAZARA_SHADER_API const char* ToString(TokenType tokenType);
NAZARA_SHADER_API std::string ToString(const std::vector<Token>& tokens, bool pretty = true);
}
#include <Nazara/Shader/ShaderLangLexer.inl>
#endif

12
include/Nazara/Shader/ShaderLangLexer.inl Normal file
View File

@ -0,0 +1,12 @@
// Copyright (C) 2020 Jérôme Leclercq
// This file is part of the "Nazara Engine - Shader generator"
// For conditions of distribution and use, see copyright notice in Config.hpp
#include <Nazara/Shader/ShaderLangLexer.hpp>
#include <Nazara/Shader/Debug.hpp>
namespace Nz
{
}
#include <Nazara/Shader/DebugOff.hpp>

58
include/Nazara/Shader/ShaderLangParser.hpp Normal file
View File

@ -0,0 +1,58 @@
// Copyright (C) 2020 Jérôme Leclercq
// This file is part of the "Nazara Engine - Renderer module"
// For conditions of distribution and use, see copyright notice in Config.hpp
#pragma once
#ifndef NAZARA_SHADER_LANGPARSER_HPP
#define NAZARA_SHADER_LANGPARSER_HPP
#include <Nazara/Prerequisites.hpp>
#include <Nazara/Shader/Config.hpp>
#include <Nazara/Shader/ShaderLangLexer.hpp>
namespace Nz::ShaderLang
{
class ExpectedToken : public std::exception
{
public:
using exception::exception;
};
class UnexpectedToken : public std::exception
{
public:
using exception::exception;
};
class NAZARA_SHADER_API Parser
{
public:
inline Parser();
~Parser() = default;
void Parse(const std::vector<Token>& tokens);
private:
const Token& Advance();
void Expect(const Token& token, TokenType type);
void ExpectNext(TokenType type);
void ParseFunctionBody();
void ParseFunctionDeclaration();
void ParseFunctionParameter();
const Token& PeekNext();
struct Context
{
std::size_t tokenCount;
std::size_t tokenIndex = 0;
const Token* tokens;
};
Context* m_context;
};
}
#include <Nazara/Shader/ShaderLangParser.inl>
#endif

16
include/Nazara/Shader/ShaderLangParser.inl Normal file
View File

@ -0,0 +1,16 @@
// Copyright (C) 2020 Jérôme Leclercq
// This file is part of the "Nazara Engine - Shader generator"
// For conditions of distribution and use, see copyright notice in Config.hpp
#include <Nazara/Shader/ShaderLangParser.hpp>
#include <Nazara/Shader/Debug.hpp>
namespace Nz::ShaderLang
{
inline Parser::Parser() :
m_context(nullptr)
{
}
}
#include <Nazara/Shader/DebugOff.hpp>

36
include/Nazara/Shader/ShaderLangTokenList.hpp Normal file
View File

@ -0,0 +1,36 @@
// Copyright (C) 2020 Jérôme Leclercq
// This file is part of the "Nazara Engine - Renderer module"
// For conditions of distribution and use, see copyright notice in Config.hpp
#if !defined(NAZARA_SHADERLANG_TOKEN)
#error You must define NAZARA_SHADERLANG_TOKEN before including this file
#endif
#ifndef NAZARA_SHADERLANG_TOKENT_LAST
#define NAZARA_SHADERLANG_TOKEN_LAST(X) NAZARA_SHADERLANG_TOKEN(X)
#endif
NAZARA_SHADERLANG_TOKEN(Add)
NAZARA_SHADERLANG_TOKEN(BoolFalse)
NAZARA_SHADERLANG_TOKEN(BoolTrue)
NAZARA_SHADERLANG_TOKEN(ClosingParenthesis)
NAZARA_SHADERLANG_TOKEN(ClosingCurlyBracket)
NAZARA_SHADERLANG_TOKEN(Colon)
NAZARA_SHADERLANG_TOKEN(Comma)
NAZARA_SHADERLANG_TOKEN(Divide)
NAZARA_SHADERLANG_TOKEN(Dot)
NAZARA_SHADERLANG_TOKEN(FloatingPointValue)
NAZARA_SHADERLANG_TOKEN(EndOfStream)
NAZARA_SHADERLANG_TOKEN(FunctionDeclaration)
NAZARA_SHADERLANG_TOKEN(FunctionReturn)
NAZARA_SHADERLANG_TOKEN(IntegerValue)
NAZARA_SHADERLANG_TOKEN(Identifier)
NAZARA_SHADERLANG_TOKEN(Multiply)
NAZARA_SHADERLANG_TOKEN(OpenCurlyBracket)
NAZARA_SHADERLANG_TOKEN(OpenParenthesis)
NAZARA_SHADERLANG_TOKEN(Semicolon)
NAZARA_SHADERLANG_TOKEN(Return)
NAZARA_SHADERLANG_TOKEN(Subtract)
#undef NAZARA_SHADERLANG_TOKEN
#undef NAZARA_SHADERLANG_TOKEN_LAST

327
src/Nazara/Shader/ShaderLangLexer.cpp Normal file
View File

@ -0,0 +1,327 @@
// Copyright (C) 2020 Jérôme Leclercq
// This file is part of the "Nazara Engine - Shader generator"
// For conditions of distribution and use, see copyright notice in Config.hpp
#include <Nazara/Shader/ShaderLangLexer.hpp>
#include <cctype>
#include <charconv>
#include <locale>
#include <optional>
#include <sstream>
#include <stdexcept>
#include <unordered_map>
#include <Nazara/Shader/Debug.hpp>
namespace Nz::ShaderLang
{
namespace
{
class ForceCLocale
{
public:
ForceCLocale()
{
m_previousLocale = std::locale::global(std::locale::classic());
}
~ForceCLocale()
{
std::locale::global(m_previousLocale);
}
private:
std::locale m_previousLocale;
};
}
std::vector<Token> Tokenize(const std::string_view& str)
{
// Can't use std::from_chars for double thanks to libc++ and libstdc++ developers being lazy
ForceCLocale forceCLocale;
std::unordered_map<std::string, TokenType> reservedKeywords = {
{ "false", TokenType::BoolFalse },
{ "fn", TokenType::FunctionDeclaration },
{ "return", TokenType::Return },
{ "true", TokenType::BoolTrue }
};
std::size_t currentPos = 0;
auto Peek = [&](std::size_t advance = 1) -> char
{
if (currentPos + advance < str.size())
return str[currentPos + advance];
else
return char(-1);
};
auto IsAlphaNum = [&](const char c)
{
return std::isalnum(c) || c == '_';
};
unsigned int lineNumber = 0;
std::size_t lastLineFeed = 0;
std::vector<Token> tokens;
for (;;)
{
char c = Peek(0);
Token token;
token.column = currentPos - lastLineFeed;
token.line = lineNumber;
if (c == -1)
{
token.type = TokenType::EndOfStream;
tokens.push_back(std::move(token));
break;
}
std::optional<TokenType> tokenType;
switch (c)
{
case ' ':
case '\t':
case '\r':
break; //< Ignore blank spaces
case '\n':
{
lineNumber++;
lastLineFeed = currentPos;
break;
}
case '-':
{
if (Peek() == '>')
{
tokenType = TokenType::FunctionReturn;
break;
}
tokenType = TokenType::Subtract;
break;
}
case '/':
{
char next = Peek();
if (next == '/')
{
// Line comment
do
{
currentPos++;
next = Peek();
}
while (next != -1 && next != '\n');
}
else if (next == '*')
{
// Block comment
do
{
currentPos++;
next = Peek();
if (next == '*')
{
currentPos++;
if (Peek() == '/')
break;
}
else if (next == '\n')
{
lineNumber++;
lastLineFeed = currentPos + 1;
}
}
while (next != -1);
}
else
tokenType == TokenType::Divide;
break;
}
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9':
{
bool floatingPoint = false;
std::size_t start = currentPos;
char next = Peek();
if (next == 'x' || next == 'X')
currentPos++;
for (;;)
{
next = Peek();
if (!isdigit(next))
{
if (next != '.')
break;
if (floatingPoint)
break;
floatingPoint = true;
}
currentPos++;
}
if (floatingPoint)
{
tokenType = TokenType::FloatingPointValue;
std::string valueStr(str.substr(start, currentPos - start + 1));
char* end;
double value = std::strtod(valueStr.c_str(), &end);
if (end != &str[currentPos])
throw BadNumber{};
token.data = value;
}
else
{
tokenType = TokenType::IntegerValue;
long long value;
std::from_chars_result r = std::from_chars(&str[start], &str[currentPos + 1], value);
if (r.ptr != &str[currentPos])
{
if (r.ec == std::errc::result_out_of_range)
throw NumberOutOfRange{};
throw BadNumber{};
}
token.data = value;
}
break;
}
case '+': tokenType = TokenType::Add; break;
case '*': tokenType = TokenType::Multiply; break;
case ':': tokenType = TokenType::Colon; break;
case ';': tokenType = TokenType::Semicolon; break;
case '.': tokenType = TokenType::Dot; break;
case ',': tokenType = TokenType::Comma; break;
case '{': tokenType = TokenType::OpenCurlyBracket; break;
case '}': tokenType = TokenType::ClosingCurlyBracket; break;
case '(': tokenType = TokenType::OpenParenthesis; break;
case ')': tokenType = TokenType::ClosingParenthesis; break;
default: break;
}
if (!tokenType)
{
if (IsAlphaNum(c))
{
std::size_t start = currentPos;
while (IsAlphaNum(Peek()))
currentPos++;
std::string identifier(str.substr(start, currentPos - start + 1));
if (auto it = reservedKeywords.find(identifier); it == reservedKeywords.end())
{
tokenType = TokenType::Identifier;
token.data = std::move(identifier);
}
else
tokenType = it->second;
}
}
if (tokenType)
{
token.type = *tokenType;
tokens.push_back(std::move(token));
}
currentPos++;
}
return tokens;
}
const char* ToString(TokenType tokenType)
{
switch (tokenType)
{
#define NAZARA_SHADERLANG_TOKEN(X) case TokenType:: X: return #X;
#include <Nazara/Shader/ShaderLangTokenList.hpp>
}
return "<Error>";
}
std::string ToString(const std::vector<Token>& tokens, bool pretty)
{
if (tokens.empty())
return {};
unsigned int lastLineNumber = tokens.front().line;
std::stringstream ss;
bool empty = true;
for (const Token& token : tokens)
{
if (token.line != lastLineNumber && pretty)
{
lastLineNumber = token.line;
if (!empty)
ss << '\n';
}
else if (!empty)
ss << ' ';
ss << ToString(token.type);
switch (token.type)
{
case TokenType::FloatingPointValue:
ss << "(" << std::get<double>(token.data) << ")";
break;
case TokenType::Identifier:
ss << "(" << std::get<std::string>(token.data) << ")";
break;
case TokenType::IntegerValue:
ss << "(" << std::get<long long>(token.data) << ")";
break;
default:
break;
}
empty = false;
}
ss << '\n';
return std::move(ss).str();
}
}

118
src/Nazara/Shader/ShaderLangParser.cpp Normal file
View File

@ -0,0 +1,118 @@
// Copyright (C) 2020 Jérôme Leclercq
// This file is part of the "Nazara Engine - Shader generator"
// For conditions of distribution and use, see copyright notice in Config.hpp
#include <Nazara/Shader/ShaderLangParser.hpp>
#include <cassert>
#include <Nazara/Shader/Debug.hpp>
namespace Nz::ShaderLang
{
void Parser::Parse(const std::vector<Token>& tokens)
{
Context context;
context.tokenCount = tokens.size();
context.tokens = tokens.data();
m_context = &context;
for (const Token& token : tokens)
{
switch (token.type)
{
case TokenType::FunctionDeclaration:
ParseFunctionDeclaration();
break;
default:
throw UnexpectedToken{};
}
}
}
const Token& Parser::Advance()
{
assert(m_context->tokenIndex + 1 < m_context->tokenCount);
return m_context->tokens[++m_context->tokenIndex];
}
void Parser::Expect(const Token& token, TokenType type)
{
if (token.type != type)
throw ExpectedToken{};
}
void Parser::ExpectNext(TokenType type)
{
Expect(m_context->tokens[m_context->tokenIndex + 1], type);
}
void Parser::ParseFunctionBody()
{
}
void Parser::ParseFunctionDeclaration()
{
ExpectNext(TokenType::Identifier);
std::string functionName = std::get<std::string>(Advance().data);
ExpectNext(TokenType::OpenParenthesis);
Advance();
bool firstParameter = true;
for (;;)
{
const Token& t = PeekNext();
if (t.type == TokenType::ClosingParenthesis)
break;
if (!firstParameter)
{
Expect(t, TokenType::Comma);
Advance();
}
ParseFunctionParameter();
firstParameter = false;
}
ExpectNext(TokenType::ClosingParenthesis);
Advance();
if (PeekNext().type == TokenType::FunctionReturn)
{
Advance();
std::string returnType = std::get<std::string>(Advance().data);
}
ExpectNext(TokenType::OpenCurlyBracket);
Advance();
ParseFunctionBody();
ExpectNext(TokenType::ClosingCurlyBracket);
Advance();
}
void Parser::ParseFunctionParameter()
{
ExpectNext(TokenType::Identifier);
std::string parameterName = std::get<std::string>(Advance().data);
ExpectNext(TokenType::Colon);
Advance();
ExpectNext(TokenType::Identifier);
std::string parameterType = std::get<std::string>(Advance().data);
}
const Token& Parser::PeekNext()
{
assert(m_context->tokenIndex + 1 < m_context->tokenCount);
return m_context->tokens[m_context->tokenIndex + 1];
}
}