diff --git a/include/Nazara/Shader/ShaderLangLexer.hpp b/include/Nazara/Shader/ShaderLangLexer.hpp new file mode 100644 index 000000000..10c63d31a --- /dev/null +++ b/include/Nazara/Shader/ShaderLangLexer.hpp @@ -0,0 +1,56 @@ +// Copyright (C) 2020 Jérôme Leclercq +// This file is part of the "Nazara Engine - Renderer module" +// For conditions of distribution and use, see copyright notice in Config.hpp + +#pragma once + +#ifndef NAZARA_SHADER_LANGLEXER_HPP +#define NAZARA_SHADER_LANGLEXER_HPP + +#include +#include +#include +#include +#include +#include + +namespace Nz::ShaderLang +{ + enum class TokenType + { +#define NAZARA_SHADERLANG_TOKEN(X) X, + +#include + }; + + struct Token + { + unsigned int column; + unsigned int line; + TokenType type; + std::variant data; + }; + + class BadNumber : public std::exception + { + using exception::exception; + }; + + class NumberOutOfRange : public std::exception + { + using exception::exception; + }; + + class UnrecognizedToken : public std::exception + { + using exception::exception; + }; + + NAZARA_SHADER_API std::vector Tokenize(const std::string_view& str); + NAZARA_SHADER_API const char* ToString(TokenType tokenType); + NAZARA_SHADER_API std::string ToString(const std::vector& tokens, bool pretty = true); +} + +#include + +#endif diff --git a/include/Nazara/Shader/ShaderLangLexer.inl b/include/Nazara/Shader/ShaderLangLexer.inl new file mode 100644 index 000000000..6767d0351 --- /dev/null +++ b/include/Nazara/Shader/ShaderLangLexer.inl @@ -0,0 +1,12 @@ +// Copyright (C) 2020 Jérôme Leclercq +// This file is part of the "Nazara Engine - Shader generator" +// For conditions of distribution and use, see copyright notice in Config.hpp + +#include +#include + +namespace Nz +{ +} + +#include diff --git a/include/Nazara/Shader/ShaderLangParser.hpp b/include/Nazara/Shader/ShaderLangParser.hpp new file mode 100644 index 000000000..be6da0a35 --- /dev/null +++ b/include/Nazara/Shader/ShaderLangParser.hpp @@ -0,0 +1,58 @@ +// Copyright (C) 2020 Jérôme Leclercq +// This file is part of the "Nazara Engine - Renderer module" +// For conditions of distribution and use, see copyright notice in Config.hpp + +#pragma once + +#ifndef NAZARA_SHADER_LANGPARSER_HPP +#define NAZARA_SHADER_LANGPARSER_HPP + +#include +#include +#include + +namespace Nz::ShaderLang +{ + class ExpectedToken : public std::exception + { + public: + using exception::exception; + }; + + class UnexpectedToken : public std::exception + { + public: + using exception::exception; + }; + + class NAZARA_SHADER_API Parser + { + public: + inline Parser(); + ~Parser() = default; + + void Parse(const std::vector& tokens); + + private: + const Token& Advance(); + void Expect(const Token& token, TokenType type); + void ExpectNext(TokenType type); + void ParseFunctionBody(); + void ParseFunctionDeclaration(); + void ParseFunctionParameter(); + const Token& PeekNext(); + + struct Context + { + std::size_t tokenCount; + std::size_t tokenIndex = 0; + const Token* tokens; + }; + + Context* m_context; + }; +} + +#include + +#endif diff --git a/include/Nazara/Shader/ShaderLangParser.inl b/include/Nazara/Shader/ShaderLangParser.inl new file mode 100644 index 000000000..12db8d17d --- /dev/null +++ b/include/Nazara/Shader/ShaderLangParser.inl @@ -0,0 +1,16 @@ +// Copyright (C) 2020 Jérôme Leclercq +// This file is part of the "Nazara Engine - Shader generator" +// For conditions of distribution and use, see copyright notice in Config.hpp + +#include +#include + +namespace Nz::ShaderLang +{ + inline Parser::Parser() : + m_context(nullptr) + { + } +} + +#include diff --git a/include/Nazara/Shader/ShaderLangTokenList.hpp b/include/Nazara/Shader/ShaderLangTokenList.hpp new file mode 100644 index 000000000..b968c590e --- /dev/null +++ b/include/Nazara/Shader/ShaderLangTokenList.hpp @@ -0,0 +1,36 @@ +// Copyright (C) 2020 Jérôme Leclercq +// This file is part of the "Nazara Engine - Renderer module" +// For conditions of distribution and use, see copyright notice in Config.hpp + +#if !defined(NAZARA_SHADERLANG_TOKEN) +#error You must define NAZARA_SHADERLANG_TOKEN before including this file +#endif + +#ifndef NAZARA_SHADERLANG_TOKENT_LAST +#define NAZARA_SHADERLANG_TOKEN_LAST(X) NAZARA_SHADERLANG_TOKEN(X) +#endif + +NAZARA_SHADERLANG_TOKEN(Add) +NAZARA_SHADERLANG_TOKEN(BoolFalse) +NAZARA_SHADERLANG_TOKEN(BoolTrue) +NAZARA_SHADERLANG_TOKEN(ClosingParenthesis) +NAZARA_SHADERLANG_TOKEN(ClosingCurlyBracket) +NAZARA_SHADERLANG_TOKEN(Colon) +NAZARA_SHADERLANG_TOKEN(Comma) +NAZARA_SHADERLANG_TOKEN(Divide) +NAZARA_SHADERLANG_TOKEN(Dot) +NAZARA_SHADERLANG_TOKEN(FloatingPointValue) +NAZARA_SHADERLANG_TOKEN(EndOfStream) +NAZARA_SHADERLANG_TOKEN(FunctionDeclaration) +NAZARA_SHADERLANG_TOKEN(FunctionReturn) +NAZARA_SHADERLANG_TOKEN(IntegerValue) +NAZARA_SHADERLANG_TOKEN(Identifier) +NAZARA_SHADERLANG_TOKEN(Multiply) +NAZARA_SHADERLANG_TOKEN(OpenCurlyBracket) +NAZARA_SHADERLANG_TOKEN(OpenParenthesis) +NAZARA_SHADERLANG_TOKEN(Semicolon) +NAZARA_SHADERLANG_TOKEN(Return) +NAZARA_SHADERLANG_TOKEN(Subtract) + +#undef NAZARA_SHADERLANG_TOKEN +#undef NAZARA_SHADERLANG_TOKEN_LAST diff --git a/src/Nazara/Shader/ShaderLangLexer.cpp b/src/Nazara/Shader/ShaderLangLexer.cpp new file mode 100644 index 000000000..a93cefa85 --- /dev/null +++ b/src/Nazara/Shader/ShaderLangLexer.cpp @@ -0,0 +1,327 @@ +// Copyright (C) 2020 Jérôme Leclercq +// This file is part of the "Nazara Engine - Shader generator" +// For conditions of distribution and use, see copyright notice in Config.hpp + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace Nz::ShaderLang +{ + namespace + { + class ForceCLocale + { + public: + ForceCLocale() + { + m_previousLocale = std::locale::global(std::locale::classic()); + } + + ~ForceCLocale() + { + std::locale::global(m_previousLocale); + } + + private: + std::locale m_previousLocale; + }; + } + + std::vector Tokenize(const std::string_view& str) + { + // Can't use std::from_chars for double thanks to libc++ and libstdc++ developers being lazy + ForceCLocale forceCLocale; + + std::unordered_map reservedKeywords = { + { "false", TokenType::BoolFalse }, + { "fn", TokenType::FunctionDeclaration }, + { "return", TokenType::Return }, + { "true", TokenType::BoolTrue } + }; + + std::size_t currentPos = 0; + + auto Peek = [&](std::size_t advance = 1) -> char + { + if (currentPos + advance < str.size()) + return str[currentPos + advance]; + else + return char(-1); + }; + + auto IsAlphaNum = [&](const char c) + { + return std::isalnum(c) || c == '_'; + }; + + unsigned int lineNumber = 0; + std::size_t lastLineFeed = 0; + std::vector tokens; + + for (;;) + { + char c = Peek(0); + + Token token; + token.column = currentPos - lastLineFeed; + token.line = lineNumber; + + if (c == -1) + { + token.type = TokenType::EndOfStream; + tokens.push_back(std::move(token)); + break; + } + + std::optional tokenType; + switch (c) + { + case ' ': + case '\t': + case '\r': + break; //< Ignore blank spaces + + case '\n': + { + lineNumber++; + lastLineFeed = currentPos; + break; + } + + case '-': + { + if (Peek() == '>') + { + tokenType = TokenType::FunctionReturn; + break; + } + + tokenType = TokenType::Subtract; + break; + } + + case '/': + { + char next = Peek(); + if (next == '/') + { + // Line comment + do + { + currentPos++; + next = Peek(); + } + while (next != -1 && next != '\n'); + } + else if (next == '*') + { + // Block comment + do + { + currentPos++; + next = Peek(); + + if (next == '*') + { + currentPos++; + if (Peek() == '/') + break; + } + else if (next == '\n') + { + lineNumber++; + lastLineFeed = currentPos + 1; + } + } + while (next != -1); + } + else + tokenType == TokenType::Divide; + + break; + } + + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + { + bool floatingPoint = false; + + std::size_t start = currentPos; + char next = Peek(); + + if (next == 'x' || next == 'X') + currentPos++; + + for (;;) + { + next = Peek(); + + if (!isdigit(next)) + { + if (next != '.') + break; + + if (floatingPoint) + break; + + floatingPoint = true; + } + + currentPos++; + } + + if (floatingPoint) + { + tokenType = TokenType::FloatingPointValue; + + std::string valueStr(str.substr(start, currentPos - start + 1)); + + char* end; + double value = std::strtod(valueStr.c_str(), &end); + if (end != &str[currentPos]) + throw BadNumber{}; + + token.data = value; + } + else + { + tokenType = TokenType::IntegerValue; + + long long value; + std::from_chars_result r = std::from_chars(&str[start], &str[currentPos + 1], value); + if (r.ptr != &str[currentPos]) + { + if (r.ec == std::errc::result_out_of_range) + throw NumberOutOfRange{}; + + throw BadNumber{}; + } + + token.data = value; + } + + break; + } + + case '+': tokenType = TokenType::Add; break; + case '*': tokenType = TokenType::Multiply; break; + case ':': tokenType = TokenType::Colon; break; + case ';': tokenType = TokenType::Semicolon; break; + case '.': tokenType = TokenType::Dot; break; + case ',': tokenType = TokenType::Comma; break; + case '{': tokenType = TokenType::OpenCurlyBracket; break; + case '}': tokenType = TokenType::ClosingCurlyBracket; break; + case '(': tokenType = TokenType::OpenParenthesis; break; + case ')': tokenType = TokenType::ClosingParenthesis; break; + + default: break; + } + + if (!tokenType) + { + if (IsAlphaNum(c)) + { + std::size_t start = currentPos; + + while (IsAlphaNum(Peek())) + currentPos++; + + std::string identifier(str.substr(start, currentPos - start + 1)); + if (auto it = reservedKeywords.find(identifier); it == reservedKeywords.end()) + { + tokenType = TokenType::Identifier; + token.data = std::move(identifier); + } + else + tokenType = it->second; + } + } + + if (tokenType) + { + token.type = *tokenType; + + tokens.push_back(std::move(token)); + } + + currentPos++; + } + + return tokens; + } + + const char* ToString(TokenType tokenType) + { + switch (tokenType) + { +#define NAZARA_SHADERLANG_TOKEN(X) case TokenType:: X: return #X; + +#include + } + + return ""; + } + + std::string ToString(const std::vector& tokens, bool pretty) + { + if (tokens.empty()) + return {}; + + unsigned int lastLineNumber = tokens.front().line; + + std::stringstream ss; + bool empty = true; + + for (const Token& token : tokens) + { + if (token.line != lastLineNumber && pretty) + { + lastLineNumber = token.line; + if (!empty) + ss << '\n'; + } + else if (!empty) + ss << ' '; + + ss << ToString(token.type); + switch (token.type) + { + case TokenType::FloatingPointValue: + ss << "(" << std::get(token.data) << ")"; + break; + + case TokenType::Identifier: + ss << "(" << std::get(token.data) << ")"; + break; + + case TokenType::IntegerValue: + ss << "(" << std::get(token.data) << ")"; + break; + + default: + break; + } + + empty = false; + } + + ss << '\n'; + + return std::move(ss).str(); + } +} diff --git a/src/Nazara/Shader/ShaderLangParser.cpp b/src/Nazara/Shader/ShaderLangParser.cpp new file mode 100644 index 000000000..482afd007 --- /dev/null +++ b/src/Nazara/Shader/ShaderLangParser.cpp @@ -0,0 +1,118 @@ +// Copyright (C) 2020 Jérôme Leclercq +// This file is part of the "Nazara Engine - Shader generator" +// For conditions of distribution and use, see copyright notice in Config.hpp + +#include +#include +#include + +namespace Nz::ShaderLang +{ + void Parser::Parse(const std::vector& tokens) + { + Context context; + context.tokenCount = tokens.size(); + context.tokens = tokens.data(); + + m_context = &context; + + for (const Token& token : tokens) + { + switch (token.type) + { + case TokenType::FunctionDeclaration: + ParseFunctionDeclaration(); + break; + + default: + throw UnexpectedToken{}; + } + } + } + + const Token& Parser::Advance() + { + assert(m_context->tokenIndex + 1 < m_context->tokenCount); + return m_context->tokens[++m_context->tokenIndex]; + } + + void Parser::Expect(const Token& token, TokenType type) + { + if (token.type != type) + throw ExpectedToken{}; + } + + void Parser::ExpectNext(TokenType type) + { + Expect(m_context->tokens[m_context->tokenIndex + 1], type); + } + + void Parser::ParseFunctionBody() + { + + } + + void Parser::ParseFunctionDeclaration() + { + ExpectNext(TokenType::Identifier); + + std::string functionName = std::get(Advance().data); + + ExpectNext(TokenType::OpenParenthesis); + Advance(); + + bool firstParameter = true; + for (;;) + { + const Token& t = PeekNext(); + if (t.type == TokenType::ClosingParenthesis) + break; + + if (!firstParameter) + { + Expect(t, TokenType::Comma); + Advance(); + } + + ParseFunctionParameter(); + firstParameter = false; + } + + ExpectNext(TokenType::ClosingParenthesis); + Advance(); + + if (PeekNext().type == TokenType::FunctionReturn) + { + Advance(); + + std::string returnType = std::get(Advance().data); + } + + + ExpectNext(TokenType::OpenCurlyBracket); + Advance(); + + ParseFunctionBody(); + + ExpectNext(TokenType::ClosingCurlyBracket); + Advance(); + } + + void Parser::ParseFunctionParameter() + { + ExpectNext(TokenType::Identifier); + std::string parameterName = std::get(Advance().data); + + ExpectNext(TokenType::Colon); + Advance(); + + ExpectNext(TokenType::Identifier); + std::string parameterType = std::get(Advance().data); + } + + const Token& Parser::PeekNext() + { + assert(m_context->tokenIndex + 1 < m_context->tokenCount); + return m_context->tokens[m_context->tokenIndex + 1]; + } +}