InputCommon: Make input expression multiline-comment tokenizing less hacky.

This commit is contained in:
Jordan Woyak 2024-11-07 19:34:55 -06:00
parent 2c83a256ae
commit c94ec85460
1 changed files with 15 additions and 19 deletions

View File

@ -181,7 +181,19 @@ Token Lexer::NextToken()
case '*':
return Token(TOK_MUL);
case '/':
{
// Handle /* */ style comments.
if (it != expr.end() && *it == '*')
{
++it;
const auto end_of_comment = expr.find("*/", it - expr.begin());
if (end_of_comment == std::string::npos)
return Token(TOK_INVALID);
it = expr.begin() + end_of_comment + 2;
return Token(TOK_COMMENT);
}
return Token(TOK_DIV);
}
case '%':
return Token(TOK_MOD);
case '=':
@ -214,26 +226,10 @@ ParseStatus Lexer::Tokenize(std::vector<Token>& tokens)
{
while (true)
{
const std::size_t string_position = it - expr.begin();
const std::string::iterator prev_it = it;
Token tok = NextToken();
tok.string_position = string_position;
tok.string_length = it - expr.begin();
// Handle /* */ style comments.
if (tok.type == TOK_DIV && PeekToken().type == TOK_MUL)
{
const auto end_of_comment = expr.find("*/", it - expr.begin());
if (end_of_comment == std::string::npos)
return ParseStatus::SyntaxError;
tok.type = TOK_COMMENT;
tok.string_length = end_of_comment + 4;
it = expr.begin() + end_of_comment + 2;
}
tok.string_position = prev_it - expr.begin();
tok.string_length = it - prev_it;
tokens.push_back(tok);
if (tok.type == TOK_INVALID)