Initial shaderlang commit

include/Nazara/Shader/ShaderLangLexer.hpp

@@ -0,0 +1,56 @@
+// Copyright (C) 2020 Jérôme Leclercq
+// This file is part of the "Nazara Engine - Renderer module"
+// For conditions of distribution and use, see copyright notice in Config.hpp
+
+#pragma once
+
+#ifndef NAZARA_SHADER_LANGLEXER_HPP
+#define NAZARA_SHADER_LANGLEXER_HPP
+
+#include <Nazara/Prerequisites.hpp>
+#include <Nazara/Shader/Config.hpp>
+#include <stdexcept>
+#include <string>
+#include <variant>
+#include <vector>
+
+namespace Nz::ShaderLang
+{
+	enum class TokenType
+	{
+#define NAZARA_SHADERLANG_TOKEN(X) X,
+
+#include <Nazara/Shader/ShaderLangTokenList.hpp>
+	};
+
+	struct Token
+	{
+		unsigned int column;
+		unsigned int line;
+		TokenType type;
+		std::variant<double, long long, std::string> data;
+	};
+
+	class BadNumber : public std::exception
+	{
+		using exception::exception;
+	};
+
+	class NumberOutOfRange : public std::exception
+	{
+		using exception::exception;
+	};
+
+	class UnrecognizedToken : public std::exception
+	{
+		using exception::exception;
+	};
+
+	NAZARA_SHADER_API std::vector<Token> Tokenize(const std::string_view& str);
+	NAZARA_SHADER_API const char* ToString(TokenType tokenType);
+	NAZARA_SHADER_API std::string ToString(const std::vector<Token>& tokens, bool pretty = true);
+}
+
+#include <Nazara/Shader/ShaderLangLexer.inl>
+
+#endif

include/Nazara/Shader/ShaderLangLexer.inl

@@ -0,0 +1,12 @@
+// Copyright (C) 2020 Jérôme Leclercq
+// This file is part of the "Nazara Engine - Shader generator"
+// For conditions of distribution and use, see copyright notice in Config.hpp
+
+#include <Nazara/Shader/ShaderLangLexer.hpp>
+#include <Nazara/Shader/Debug.hpp>
+
+namespace Nz
+{
+}
+
+#include <Nazara/Shader/DebugOff.hpp>

include/Nazara/Shader/ShaderLangParser.hpp

@@ -0,0 +1,58 @@
+// Copyright (C) 2020 Jérôme Leclercq
+// This file is part of the "Nazara Engine - Renderer module"
+// For conditions of distribution and use, see copyright notice in Config.hpp
+
+#pragma once
+
+#ifndef NAZARA_SHADER_LANGPARSER_HPP
+#define NAZARA_SHADER_LANGPARSER_HPP
+
+#include <Nazara/Prerequisites.hpp>
+#include <Nazara/Shader/Config.hpp>
+#include <Nazara/Shader/ShaderLangLexer.hpp>
+
+namespace Nz::ShaderLang
+{
+	class ExpectedToken : public std::exception
+	{
+		public:
+			using exception::exception;
+	};
+
+	class UnexpectedToken : public std::exception
+	{
+		public:
+			using exception::exception;
+	};
+
+	class NAZARA_SHADER_API Parser
+	{
+		public:
+			inline Parser();
+			~Parser() = default;
+
+			void Parse(const std::vector<Token>& tokens);
+
+		private:
+			const Token& Advance();
+			void Expect(const Token& token, TokenType type);
+			void ExpectNext(TokenType type);
+			void ParseFunctionBody();
+			void ParseFunctionDeclaration();
+			void ParseFunctionParameter();
+			const Token& PeekNext();
+
+			struct Context
+			{
+				std::size_t tokenCount;
+				std::size_t tokenIndex = 0;
+				const Token* tokens;
+			};
+
+			Context* m_context;
+	};
+}
+
+#include <Nazara/Shader/ShaderLangParser.inl>
+
+#endif

include/Nazara/Shader/ShaderLangParser.inl

@@ -0,0 +1,16 @@
+// Copyright (C) 2020 Jérôme Leclercq
+// This file is part of the "Nazara Engine - Shader generator"
+// For conditions of distribution and use, see copyright notice in Config.hpp
+
+#include <Nazara/Shader/ShaderLangParser.hpp>
+#include <Nazara/Shader/Debug.hpp>
+
+namespace Nz::ShaderLang
+{
+	inline Parser::Parser() :
+	m_context(nullptr)
+	{
+	}
+}
+
+#include <Nazara/Shader/DebugOff.hpp>

include/Nazara/Shader/ShaderLangTokenList.hpp

@@ -0,0 +1,36 @@
+// Copyright (C) 2020 Jérôme Leclercq
+// This file is part of the "Nazara Engine - Renderer module"
+// For conditions of distribution and use, see copyright notice in Config.hpp
+
+#if !defined(NAZARA_SHADERLANG_TOKEN)
+#error You must define NAZARA_SHADERLANG_TOKEN before including this file
+#endif
+
+#ifndef NAZARA_SHADERLANG_TOKENT_LAST
+#define NAZARA_SHADERLANG_TOKEN_LAST(X) NAZARA_SHADERLANG_TOKEN(X)
+#endif
+
+NAZARA_SHADERLANG_TOKEN(Add)
+NAZARA_SHADERLANG_TOKEN(BoolFalse)
+NAZARA_SHADERLANG_TOKEN(BoolTrue)
+NAZARA_SHADERLANG_TOKEN(ClosingParenthesis)
+NAZARA_SHADERLANG_TOKEN(ClosingCurlyBracket)
+NAZARA_SHADERLANG_TOKEN(Colon)
+NAZARA_SHADERLANG_TOKEN(Comma)
+NAZARA_SHADERLANG_TOKEN(Divide)
+NAZARA_SHADERLANG_TOKEN(Dot)
+NAZARA_SHADERLANG_TOKEN(FloatingPointValue)
+NAZARA_SHADERLANG_TOKEN(EndOfStream)
+NAZARA_SHADERLANG_TOKEN(FunctionDeclaration)
+NAZARA_SHADERLANG_TOKEN(FunctionReturn)
+NAZARA_SHADERLANG_TOKEN(IntegerValue)
+NAZARA_SHADERLANG_TOKEN(Identifier)
+NAZARA_SHADERLANG_TOKEN(Multiply)
+NAZARA_SHADERLANG_TOKEN(OpenCurlyBracket)
+NAZARA_SHADERLANG_TOKEN(OpenParenthesis)
+NAZARA_SHADERLANG_TOKEN(Semicolon)
+NAZARA_SHADERLANG_TOKEN(Return)
+NAZARA_SHADERLANG_TOKEN(Subtract)
+
+#undef NAZARA_SHADERLANG_TOKEN
+#undef NAZARA_SHADERLANG_TOKEN_LAST

src/Nazara/Shader/ShaderLangLexer.cpp

@@ -0,0 +1,327 @@
+// Copyright (C) 2020 Jérôme Leclercq
+// This file is part of the "Nazara Engine - Shader generator"
+// For conditions of distribution and use, see copyright notice in Config.hpp
+
+#include <Nazara/Shader/ShaderLangLexer.hpp>
+#include <cctype>
+#include <charconv>
+#include <locale>
+#include <optional>
+#include <sstream>
+#include <stdexcept>
+#include <unordered_map>
+#include <Nazara/Shader/Debug.hpp>
+
+namespace Nz::ShaderLang
+{
+	namespace
+	{
+		class ForceCLocale
+		{
+			public:
+				ForceCLocale()
+				{
+					m_previousLocale = std::locale::global(std::locale::classic());
+				}
+
+				~ForceCLocale()
+				{
+					std::locale::global(m_previousLocale);
+				}
+
+			private:
+				std::locale m_previousLocale;
+		};
+	}
+
+	std::vector<Token> Tokenize(const std::string_view& str)
+	{
+		// Can't use std::from_chars for double thanks to libc++ and libstdc++ developers being lazy
+		ForceCLocale forceCLocale;
+
+		std::unordered_map<std::string, TokenType> reservedKeywords = {
+			{ "false",  TokenType::BoolFalse },
+			{ "fn",     TokenType::FunctionDeclaration },
+			{ "return", TokenType::Return },
+			{ "true",   TokenType::BoolTrue }
+		};
+
+		std::size_t currentPos = 0;
+
+		auto Peek = [&](std::size_t advance = 1) -> char
+		{
+			if (currentPos + advance < str.size())
+				return str[currentPos + advance];
+			else
+				return char(-1);
+		};
+
+		auto IsAlphaNum = [&](const char c)
+		{
+			return std::isalnum(c) || c == '_';
+		};
+
+		unsigned int lineNumber = 0;
+		std::size_t lastLineFeed = 0;
+		std::vector<Token> tokens;
+
+		for (;;)
+		{
+			char c = Peek(0);
+
+			Token token;
+			token.column = currentPos - lastLineFeed;
+			token.line = lineNumber;
+
+			if (c == -1)
+			{
+				token.type = TokenType::EndOfStream;
+				tokens.push_back(std::move(token));
+				break;
+			}
+
+			std::optional<TokenType> tokenType;
+			switch (c)
+			{
+				case ' ':
+				case '\t':
+				case '\r':
+					break; //< Ignore blank spaces
+
+				case '\n':
+				{
+					lineNumber++;
+					lastLineFeed = currentPos;
+					break;
+				}
+
+				case '-':
+				{
+					if (Peek() == '>')
+					{
+						tokenType = TokenType::FunctionReturn;
+						break;
+					}
+
+					tokenType = TokenType::Subtract;
+					break;
+				}
+
+				case '/':
+				{
+					char next = Peek();
+					if (next == '/')
+					{
+						// Line comment
+						do
+						{
+							currentPos++;
+							next = Peek();
+						}
+						while (next != -1 && next != '\n');
+					}
+					else if (next == '*')
+					{
+						// Block comment
+						do
+						{
+							currentPos++;
+							next = Peek();
+
+							if (next == '*')
+							{
+								currentPos++;
+								if (Peek() == '/')
+									break;
+							}
+							else if (next == '\n')
+							{
+								lineNumber++;
+								lastLineFeed = currentPos + 1;
+							}
+						}
+						while (next != -1);
+					}
+					else
+						tokenType == TokenType::Divide;
+
+					break;
+				}
+
+				case '0':
+				case '1':
+				case '2':
+				case '3':
+				case '4':
+				case '5':
+				case '6':
+				case '7':
+				case '8':
+				case '9':
+				{
+					bool floatingPoint = false;
+
+					std::size_t start = currentPos;
+					char next = Peek();
+
+					if (next == 'x' || next == 'X')
+						currentPos++;
+
+					for (;;)
+					{
+						next = Peek();
+
+						if (!isdigit(next))
+						{
+							if (next != '.')
+								break;
+
+							if (floatingPoint)
+								break;
+
+							floatingPoint = true;
+						}
+
+						currentPos++;
+					}
+
+					if (floatingPoint)
+					{
+						tokenType = TokenType::FloatingPointValue;
+
+						std::string valueStr(str.substr(start, currentPos - start + 1));
+
+						char* end;
+						double value = std::strtod(valueStr.c_str(), &end);
+						if (end != &str[currentPos])
+							throw BadNumber{};
+
+						token.data = value;
+					}
+					else
+					{
+						tokenType = TokenType::IntegerValue;
+
+						long long value;
+						std::from_chars_result r = std::from_chars(&str[start], &str[currentPos + 1], value);
+						if (r.ptr != &str[currentPos])
+						{
+							if (r.ec == std::errc::result_out_of_range)
+								throw NumberOutOfRange{};
+
+							throw BadNumber{};
+						}
+
+						token.data = value;
+					}
+
+					break;
+				}
+
+				case '+': tokenType = TokenType::Add; break;
+				case '*': tokenType = TokenType::Multiply; break;
+				case ':': tokenType = TokenType::Colon; break;
+				case ';': tokenType = TokenType::Semicolon; break;
+				case '.': tokenType = TokenType::Dot; break;
+				case ',': tokenType = TokenType::Comma; break;
+				case '{': tokenType = TokenType::OpenCurlyBracket; break;
+				case '}': tokenType = TokenType::ClosingCurlyBracket; break;
+				case '(': tokenType = TokenType::OpenParenthesis; break;
+				case ')': tokenType = TokenType::ClosingParenthesis; break;
+
+				default: break;
+			}
+
+			if (!tokenType)
+			{
+				if (IsAlphaNum(c))
+				{
+					std::size_t start = currentPos;
+
+					while (IsAlphaNum(Peek()))
+						currentPos++;
+
+					std::string identifier(str.substr(start, currentPos - start + 1));
+					if (auto it = reservedKeywords.find(identifier); it == reservedKeywords.end())
+					{
+						tokenType = TokenType::Identifier;
+						token.data = std::move(identifier);
+					}
+					else
+						tokenType = it->second;
+				}
+			}
+
+			if (tokenType)
+			{
+				token.type = *tokenType;
+
+				tokens.push_back(std::move(token));
+			}
+
+			currentPos++;
+		}
+
+		return tokens;
+	}
+
+	const char* ToString(TokenType tokenType)
+	{
+		switch (tokenType)
+		{
+#define NAZARA_SHADERLANG_TOKEN(X) case TokenType:: X: return #X;
+
+#include <Nazara/Shader/ShaderLangTokenList.hpp>
+		}
+
+		return "<Error>";
+	}
+
+	std::string ToString(const std::vector<Token>& tokens, bool pretty)
+	{
+		if (tokens.empty())
+			return {};
+
+		unsigned int lastLineNumber = tokens.front().line;
+
+		std::stringstream ss;
+		bool empty = true;
+
+		for (const Token& token : tokens)
+		{
+			if (token.line != lastLineNumber && pretty)
+			{
+				lastLineNumber = token.line;
+				if (!empty)
+					ss << '\n';
+			}
+			else if (!empty)
+				ss << ' ';
+
+			ss << ToString(token.type);
+			switch (token.type)
+			{
+				case TokenType::FloatingPointValue:
+					ss << "(" << std::get<double>(token.data) << ")";
+					break;
+
+				case TokenType::Identifier:
+					ss << "(" << std::get<std::string>(token.data) << ")";
+					break;
+
+				case TokenType::IntegerValue:
+					ss << "(" << std::get<long long>(token.data) << ")";
+					break;
+
+				default:
+					break;
+			}
+
+			empty = false;
+		}
+
+		ss << '\n';
+
+		return std::move(ss).str();
+	}
+}

src/Nazara/Shader/ShaderLangParser.cpp

@@ -0,0 +1,118 @@
+// Copyright (C) 2020 Jérôme Leclercq
+// This file is part of the "Nazara Engine - Shader generator"
+// For conditions of distribution and use, see copyright notice in Config.hpp
+
+#include <Nazara/Shader/ShaderLangParser.hpp>
+#include <cassert>
+#include <Nazara/Shader/Debug.hpp>
+
+namespace Nz::ShaderLang
+{
+	void Parser::Parse(const std::vector<Token>& tokens)
+	{
+		Context context;
+		context.tokenCount = tokens.size();
+		context.tokens = tokens.data();
+
+		m_context = &context;
+
+		for (const Token& token : tokens)
+		{
+			switch (token.type)
+			{
+				case TokenType::FunctionDeclaration:
+					ParseFunctionDeclaration();
+					break;
+
+				default:
+					throw UnexpectedToken{};
+			}
+		}
+	}
+
+	const Token& Parser::Advance()
+	{
+		assert(m_context->tokenIndex + 1 < m_context->tokenCount);
+		return m_context->tokens[++m_context->tokenIndex];
+	}
+
+	void Parser::Expect(const Token& token, TokenType type)
+	{
+		if (token.type != type)
+			throw ExpectedToken{};
+	}
+
+	void Parser::ExpectNext(TokenType type)
+	{
+		Expect(m_context->tokens[m_context->tokenIndex + 1], type);
+	}
+
+	void Parser::ParseFunctionBody()
+	{
+
+	}
+
+	void Parser::ParseFunctionDeclaration()
+	{
+		ExpectNext(TokenType::Identifier);
+
+		std::string functionName = std::get<std::string>(Advance().data);
+
+		ExpectNext(TokenType::OpenParenthesis);
+		Advance();
+
+		bool firstParameter = true;
+		for (;;)
+		{
+			const Token& t = PeekNext();
+			if (t.type == TokenType::ClosingParenthesis)
+				break;
+
+			if (!firstParameter)
+			{
+				Expect(t, TokenType::Comma);
+				Advance();
+			}
+
+			ParseFunctionParameter();
+			firstParameter = false;
+		}
+
+		ExpectNext(TokenType::ClosingParenthesis);
+		Advance();
+
+		if (PeekNext().type == TokenType::FunctionReturn)
+		{
+			Advance();
+
+			std::string returnType = std::get<std::string>(Advance().data);
+		}
+
+
+		ExpectNext(TokenType::OpenCurlyBracket);
+		Advance();
+
+		ParseFunctionBody();
+
+		ExpectNext(TokenType::ClosingCurlyBracket);
+		Advance();
+	}
+
+	void Parser::ParseFunctionParameter()
+	{
+		ExpectNext(TokenType::Identifier);
+		std::string parameterName = std::get<std::string>(Advance().data);
+
+		ExpectNext(TokenType::Colon);
+		Advance();
+
+		ExpectNext(TokenType::Identifier);
+		std::string parameterType = std::get<std::string>(Advance().data);
+	}
+
+	const Token& Parser::PeekNext()
+	{
+		assert(m_context->tokenIndex + 1 < m_context->tokenCount);
+		return m_context->tokens[m_context->tokenIndex + 1];
+	}
+}