From 307e557e5b8442a0f8ec94ff35b07f6b686f9834 Mon Sep 17 00:00:00 2001 From: Jonathan Turner Date: Mon, 1 Jun 2009 13:40:24 +0000 Subject: [PATCH] Added comment lexing. Removed previous optimization. Flattened lex grammar. --- langkit/langkit_lexer.cpp | 201 ++++++++++++++++++-------------------- langkit/langkit_lexer.hpp | 32 ++++-- langkit/main.cpp | 17 ++-- 3 files changed, 127 insertions(+), 123 deletions(-) diff --git a/langkit/langkit_lexer.cpp b/langkit/langkit_lexer.cpp index ee686f4f..af60ffb4 100644 --- a/langkit/langkit_lexer.cpp +++ b/langkit/langkit_lexer.cpp @@ -10,53 +10,112 @@ Lexer Lexer::operator<<(const Pattern &p) { } std::vector Lexer::lex(const std::string &input, const char *filename) { - std::vector::iterator iter, end, iter2; + std::vector::iterator iter, end, iter2, end2; std::vector retval; bool found; - std::string::const_iterator input_iter = input.begin(); + std::string::const_iterator input_iter = input.begin(), input_end = input.end(); int current_col = 0; int current_line = 0; + boost::match_results what; - std::string master_lex_pattern; - - unsigned int i = 0; - for (iter = lex_patterns.begin(), end = lex_patterns.end(); iter != end; ++iter) { - if (i > 0) { - master_lex_pattern += "|"; - } - master_lex_pattern += "(" + iter->regex_string + ")"; - ++i; - } - - boost::regex lex_regex(master_lex_pattern); - - while (input_iter != input.end()) { + while (input_iter != input_end) { found = false; - /* - for (iter = lex_patterns.begin(), end = lex_patterns.end(); iter != end; ++iter) { - boost::match_results what; - if (regex_search(input_iter, input.end(), what, iter->regex, boost::match_continuous)) { - TokenPtr t(new Token(what[0], iter->identifier, filename)); + if (regex_search(input_iter, input_end, what, singleline_comment_pattern.regex, boost::match_continuous)) { + std::string comment_start(what[0]); + input_iter += comment_start.size(); + + bool found_eol = false; + + while ((!found_eol) && (input_iter != input_end)) { + boost::match_results eol_delim; + if (regex_search(input_iter, input_end, eol_delim, line_sep_pattern.regex, boost::match_continuous)) { + std::string comment_end(eol_delim[0]); + input_iter += comment_end.size(); + ++current_line; + current_col = 0; + found_eol = true; + break; + } + if ((!found_eol) && (input_iter != input_end)) { + ++input_iter; + } + } + } + else if (regex_search(input_iter, input_end, what, multiline_comment_start_pattern.regex, boost::match_continuous)) { + std::string comment_start(what[0]); + input_iter += comment_start.size(); + + bool found_eoc = false; + + while ((!found_eoc) && (input_iter != input_end)) { + boost::match_results eol_delim; + if (regex_search(input_iter, input_end, eol_delim, line_sep_pattern.regex, boost::match_continuous)) { + std::string comment_end(eol_delim[0]); + input_iter += comment_end.size(); + ++current_line; + current_col = 0; + break; + } + boost::match_results eoc_delim; + if (regex_search(input_iter, input_end, eoc_delim, multiline_comment_end_pattern.regex, boost::match_continuous)) { + std::string comment_end(eoc_delim[0]); + input_iter += comment_end.size(); + current_col += comment_end.size(); + found_eoc = true; + break; + } + if ((!found_eoc) && (input_iter != input_end)) { + ++input_iter; + } + } + + if (!found_eoc) { + std::cout << "Incomplete comment block! Add exceptions!" << std::endl; + return retval; + } + } + else if (regex_search(input_iter, input_end, what, skip_pattern.regex, boost::match_continuous)) { + std::string whitespace(what[0]); + input_iter += whitespace.size(); + current_col += whitespace.size(); + found = true; + } + else if (regex_search(input_iter, input_end, what, line_sep_pattern.regex, boost::match_continuous)) { + const std::string cr(what[0]); + + boost::match_results if_delim; + if (regex_search(cr.begin(), cr.end(), if_delim, command_sep_pattern.regex, boost::match_continuous)) { + TokenPtr t(new Token(if_delim[0], command_sep_pattern.identifier, filename)); t->start.column = current_col; t->start.line = current_line; current_col += t->text.size(); t->end.column = current_col; t->end.line = current_line; retval.push_back(t); - input_iter += t->text.size(); - found = true; - break; } - } - */ - boost::match_results what; - if (regex_search(input_iter, input.end(), what, lex_regex, boost::match_continuous)) { - for (i = 0; i < lex_patterns.size(); ++i) { - if (!(std::string(what[i+1])).empty()) { - TokenPtr t(new Token(what[i+1], lex_patterns[i].identifier, filename)); + input_iter += cr.size(); + ++current_line; + current_col = 0; + found = true; + } + else if (regex_search(input_iter, input_end, what, command_sep_pattern.regex, boost::match_continuous)) { + TokenPtr t(new Token(what[0], command_sep_pattern.identifier, filename)); + t->start.column = current_col; + t->start.line = current_line; + current_col += t->text.size(); + t->end.column = current_col; + t->end.line = current_line; + retval.push_back(t); + input_iter += t->text.size(); + found = true; + } + else { + for (iter = lex_patterns.begin(), end = lex_patterns.end(); iter != end; ++iter) { + if (regex_search(input_iter, input_end, what, iter->regex, boost::match_continuous)) { + TokenPtr t(new Token(what[0], iter->identifier, filename)); t->start.column = current_col; t->start.line = current_line; current_col += t->text.size(); @@ -68,87 +127,13 @@ std::vector Lexer::lex(const std::string &input, const char *filename) break; } } - } - - - if (!found) { - for (iter = skip_patterns.begin(), end = skip_patterns.end(); iter != end; ++iter) { - boost::match_results what; - if (regex_search(input_iter, input.end(), what, iter->regex, boost::match_continuous)) { - std::string whitespace(what[0]); - input_iter += whitespace.size(); - current_col += whitespace.size(); - found = true; - break; - } - } if (!found) { - for (iter = line_sep_patterns.begin(), end = line_sep_patterns.end(); iter != end; ++iter) { - boost::match_results what; - if (regex_search(input_iter, input.end(), what, iter->regex, boost::match_continuous)) { - const std::string cr(what[0]); - - for (iter2 = command_sep_patterns.begin(), end = command_sep_patterns.end(); iter2 != end; ++iter2) { - boost::match_results if_delim; - if (regex_search(cr.begin(), cr.end(), if_delim, iter2->regex, boost::match_continuous)) { - TokenPtr t(new Token(if_delim[0], iter2->identifier, filename)); - t->start.column = current_col; - t->start.line = current_line; - current_col += t->text.size(); - t->end.column = current_col; - t->end.line = current_line; - retval.push_back(t); - break; - } - } - - input_iter += cr.size(); - found = true; - ++current_line; - current_col = 0; - - break; - } - } - - if (!found) { - for (iter = command_sep_patterns.begin(), end = command_sep_patterns.end(); iter != end; ++iter) { - boost::match_results what; - if (regex_search(input_iter, input.end(), what, iter->regex, boost::match_continuous)) { - TokenPtr t(new Token(what[0], iter->identifier, filename)); - t->start.column = current_col; - t->start.line = current_line; - current_col += t->text.size(); - t->end.column = current_col; - t->end.line = current_line; - retval.push_back(t); - input_iter += t->text.size(); - found = true; - break; - } - } - - if (!found) { - const std::string err(input_iter, input.end()); - std::cout << "Unknown string at: " << err << std::endl; - return retval; - } - } + const std::string err(input_iter, input_end); + std::cout << "Unknown string at: " << err << std::endl; + return retval; } } } return retval; } - -void Lexer::set_skip(const Pattern &p) { - skip_patterns.push_back(p); -} - -void Lexer::set_line_sep(const Pattern &p) { - line_sep_patterns.push_back(p); -} - -void Lexer::set_command_sep(const Pattern &p) { - command_sep_patterns.push_back(p); -} diff --git a/langkit/langkit_lexer.hpp b/langkit/langkit_lexer.hpp index 98d5bdac..7d8d7bc6 100644 --- a/langkit/langkit_lexer.hpp +++ b/langkit/langkit_lexer.hpp @@ -20,10 +20,10 @@ struct File_Position { struct Pattern { boost::regex regex; - std::string regex_string; int identifier; - Pattern(const std::string ®exp, int id) : regex(regexp), regex_string(regexp), identifier(id) { } + Pattern() { } + Pattern(const std::string ®exp, int id) : regex(regexp), identifier(id) { } }; typedef std::tr1::shared_ptr TokenPtr; @@ -41,16 +41,32 @@ struct Token { struct Lexer { std::vector lex_patterns; - std::vector skip_patterns; - std::vector command_sep_patterns; - std::vector line_sep_patterns; + Pattern skip_pattern; + Pattern command_sep_pattern; + Pattern line_sep_pattern; + Pattern multiline_comment_start_pattern; + Pattern multiline_comment_end_pattern; + Pattern singleline_comment_pattern; Lexer operator<<(const Pattern &p); std::vector lex(const std::string &input, const char *fname); - void set_skip(const Pattern &p); - void set_line_sep(const Pattern &p); - void set_command_sep(const Pattern &p); + void set_skip(const Pattern &p) { + skip_pattern = p; + } + void set_line_sep(const Pattern &p) { + line_sep_pattern = p; + } + void set_command_sep(const Pattern &p) { + command_sep_pattern = p; + } + void set_multiline_comment(const Pattern &start, const Pattern &end) { + multiline_comment_start_pattern = start; + multiline_comment_end_pattern = end; + } + void set_singleline_comment(const Pattern &p) { + singleline_comment_pattern = p; + } }; diff --git a/langkit/main.cpp b/langkit/main.cpp index b4ea3a7a..2ec2642c 100644 --- a/langkit/main.cpp +++ b/langkit/main.cpp @@ -12,11 +12,11 @@ class TokenType { public: enum Type { Whitespace, Identifier, Number, Operator, Parens_Open, Parens_Close, Square_Open, Square_Close, Curly_Open, Curly_Close, Comma, Quoted_String, Single_Quoted_String, Carriage_Return, Semicolon, - Function_Def, Scoped_Block, Statement, Equation, Return, Add}; }; + Function_Def, Scoped_Block, Statement, Equation, Return, Add, Comment}; }; void debug_print(TokenPtr token, std::string prepend) { - std::cout << prepend << "Token: " << token->text << "(" << token->identifier << ") @ " << token->filename << ": (" << token->start.column - << ", " << token->start.line << ") to (" << token->end.column << ", " << token->end.line << ") " << std::endl; + std::cout << prepend << "Token: " << token->text << "(" << token->identifier << ") @ " << token->filename << ": (" << token->start.line + << ", " << token->start.column << ") to (" << token->end.line << ", " << token->end.column << ") " << std::endl; for (unsigned int i = 0; i < token->children.size(); ++i) { debug_print(token->children[i], prepend + " "); @@ -111,13 +111,14 @@ void parse(std::vector &tokens) { std::pair results = rule(iter, end, parent); - /* + while (results.second) { results = rule(results.first + 1, end, parent); //debug_print(parent, ""); } - */ + + /* if (results.second) { std::cout << "Parse successful: " << std::endl; debug_print(parent, ""); @@ -126,7 +127,7 @@ void parse(std::vector &tokens) { std::cout << "Parse failed: " << std::endl; debug_print(parent, ""); } - + */ } @@ -137,10 +138,12 @@ int main(int argc, char *argv[]) { lexer.set_skip(Pattern("[ \\t]+", TokenType::Whitespace)); lexer.set_line_sep(Pattern("\\n|\\r\\n", TokenType::Carriage_Return)); lexer.set_command_sep(Pattern(";|\\r\\n|\\n", TokenType::Semicolon)); + lexer.set_multiline_comment(Pattern("/\\*", TokenType::Comment), Pattern("\\*/", TokenType::Comment)); + lexer.set_singleline_comment(Pattern("//", TokenType::Comment)); lexer << Pattern("[A-Za-z]+", TokenType::Identifier); lexer << Pattern("[0-9]+(\\.[0-9]+)?", TokenType::Number); - lexer << Pattern("[!@#$%^&*\\-+=/<>]+", TokenType::Operator); + lexer << Pattern("[!@#$%^&*\\-+=<>/]+", TokenType::Operator); lexer << Pattern("\\(", TokenType::Parens_Open); lexer << Pattern("\\)", TokenType::Parens_Close); lexer << Pattern("\\[", TokenType::Square_Open);