diff --git a/chaiscript/chaiscript_engine.hpp b/chaiscript/chaiscript_engine.hpp index 1e0c2261..01962f3f 100644 --- a/chaiscript/chaiscript_engine.hpp +++ b/chaiscript/chaiscript_engine.hpp @@ -10,8 +10,8 @@ template class ChaiScript_System { - Lexer lexer; - Rule parser; + langkit::Lexer lexer; + langkit::Rule parser; Eval_Engine engine; public: @@ -30,10 +30,10 @@ public: val = Cast_Helper()(vals[0]); } catch (std::exception &e) { - throw EvalError("Can not evaluate string: " + val, TokenPtr()); + throw EvalError("Can not evaluate string: " + val, langkit::TokenPtr()); } catch (EvalError &ee) { - throw EvalError("Can not evaluate string: " + val + " reason: " + ee.reason, TokenPtr()); + throw EvalError("Can not evaluate string: " + val + " reason: " + ee.reason, langkit::TokenPtr()); } return evaluate_string(val); } @@ -57,7 +57,8 @@ public: return ret_val; } - void debug_print(TokenPtr token, std::string prepend) { + void debug_print(langkit::TokenPtr token, std::string prepend) { + using namespace langkit; std::cout << prepend << "Token: " << token->text << "(" << tokentype_to_string(token->identifier) << ") @ " << token->filename << ": (" << token->start.line << ", " << token->start.column << ") to (" << token->end.line << ", " << token->end.column << ") " << std::endl; @@ -67,13 +68,15 @@ public: } } - void debug_print(std::vector &tokens) { + void debug_print(std::vector &tokens) { + using namespace langkit; for (unsigned int i = 0; i < tokens.size(); ++i) { debug_print(tokens[i], ""); } } - Lexer build_lexer() { + langkit::Lexer build_lexer() { + using namespace langkit; Lexer lexer; lexer.set_skip(Pattern("[ \\t]+", TokenType::Whitespace)); lexer.set_line_sep(Pattern("\\n|\\r\\n", TokenType::Carriage_Return)); @@ -98,7 +101,8 @@ public: return lexer; } - Rule build_parser_rules() { + langkit::Rule build_parser_rules() { + using namespace langkit; Rule params; Rule block(TokenType::Scoped_Block); Rule fundef(TokenType::Function_Def); @@ -184,7 +188,8 @@ public: } - Eval_Engine build_eval_system(Lexer &lexer, Rule &parser) { + Eval_Engine build_eval_system(langkit::Lexer &lexer, langkit::Rule &parser) { + using namespace langkit; Eval_Engine ss; Bootstrap::bootstrap(ss); bootstrap_vector >(ss, "Vector"); @@ -197,7 +202,8 @@ public: return ss; } - TokenPtr parse(Rule &rule, std::vector &tokens, const char *filename) { + langkit::TokenPtr parse(langkit::Rule &rule, std::vector &tokens, const char *filename) { + using namespace langkit; Token_Iterator iter = tokens.begin(), end = tokens.end(); TokenPtr parent(new Token("Root", TokenType::File, filename)); @@ -215,6 +221,7 @@ public: } Boxed_Value evaluate_string(const std::string &input, const char *filename = "__EVAL__") { + using namespace langkit; std::vector tokens = lexer.lex(input, filename); Boxed_Value value; diff --git a/chaiscript/chaiscript_eval.hpp b/chaiscript/chaiscript_eval.hpp index 316588b9..9b37598d 100644 --- a/chaiscript/chaiscript_eval.hpp +++ b/chaiscript/chaiscript_eval.hpp @@ -8,33 +8,33 @@ struct ParserError { std::string reason; - TokenPtr location; + langkit::TokenPtr location; - ParserError(const std::string &why, const TokenPtr where) : reason(why), location(where){ } + ParserError(const std::string &why, const langkit::TokenPtr where) : reason(why), location(where){ } }; struct EvalError { std::string reason; - TokenPtr location; + langkit::TokenPtr location; - EvalError(const std::string &why, const TokenPtr where) : reason(why), location(where) { } + EvalError(const std::string &why, const langkit::TokenPtr where) : reason(why), location(where) { } }; struct ReturnValue { Boxed_Value retval; - TokenPtr location; + langkit::TokenPtr location; - ReturnValue(const Boxed_Value &return_value, const TokenPtr where) : retval(return_value), location(where) { } + ReturnValue(const Boxed_Value &return_value, const langkit::TokenPtr where) : retval(return_value), location(where) { } }; struct BreakLoop { - TokenPtr location; + langkit::TokenPtr location; - BreakLoop(const TokenPtr where) : location(where) { } + BreakLoop(const langkit::TokenPtr where) : location(where) { } }; template -const Boxed_Value eval_function (Eval_System &ss, TokenPtr node, const std::vector ¶m_names, const std::vector &vals) { +const Boxed_Value eval_function (Eval_System &ss, langkit::TokenPtr node, const std::vector ¶m_names, const std::vector &vals) { ss.new_scope(); for (unsigned int i = 0; i < param_names.size(); ++i) { @@ -47,7 +47,7 @@ const Boxed_Value eval_function (Eval_System &ss, TokenPtr node, const std::vect } template -Boxed_Value eval_token(Eval_System &ss, TokenPtr node) { +Boxed_Value eval_token(Eval_System &ss, langkit::TokenPtr node) { Boxed_Value retval; unsigned int i, j; diff --git a/langkit/langkit_lexer.hpp b/langkit/langkit_lexer.hpp index a1b4659d..2cf7e87f 100644 --- a/langkit/langkit_lexer.hpp +++ b/langkit/langkit_lexer.hpp @@ -8,197 +8,199 @@ #include #include -struct File_Position { - int line; - int column; +namespace langkit +{ + struct File_Position { + int line; + int column; - File_Position(int file_line, int file_column) - : line(file_line), column(file_column) { } + File_Position(int file_line, int file_column) + : line(file_line), column(file_column) { } - File_Position() : line(0), column(0) { } -}; + File_Position() : line(0), column(0) { } + }; -struct Pattern { - boost::regex regex; - int identifier; + struct Pattern { + boost::regex regex; + int identifier; - Pattern() { } - Pattern(const std::string ®exp, int id) : regex(regexp), identifier(id) { } -}; + Pattern() { } + Pattern(const std::string ®exp, int id) : regex(regexp), identifier(id) { } + }; -typedef std::tr1::shared_ptr TokenPtr; + typedef std::tr1::shared_ptr TokenPtr; -struct Token { - std::string text; - int identifier; - const char *filename; - File_Position start, end; + struct Token { + std::string text; + int identifier; + const char *filename; + File_Position start, end; - std::vector children; + std::vector children; - Token(const std::string &token_text, int id, const char *fname) : text(token_text), identifier(id), filename(fname) { } -}; + Token(const std::string &token_text, int id, const char *fname) : text(token_text), identifier(id), filename(fname) { } + }; -struct Lexer { - std::vector lex_patterns; - Pattern skip_pattern; - Pattern command_sep_pattern; - Pattern line_sep_pattern; - Pattern multiline_comment_start_pattern; - Pattern multiline_comment_end_pattern; - Pattern singleline_comment_pattern; + struct Lexer { + std::vector lex_patterns; + Pattern skip_pattern; + Pattern command_sep_pattern; + Pattern line_sep_pattern; + Pattern multiline_comment_start_pattern; + Pattern multiline_comment_end_pattern; + Pattern singleline_comment_pattern; - Lexer operator<<(const Pattern &p) { - lex_patterns.push_back(p); - return *this; - } + Lexer operator<<(const Pattern &p) { + lex_patterns.push_back(p); + return *this; + } - std::vector lex(const std::string &input, const char *filename) { - std::vector::iterator iter, end, iter2, end2; - std::vector retval; - bool found; - std::string::const_iterator input_iter = input.begin(), input_end = input.end(); + std::vector lex(const std::string &input, const char *filename) { + std::vector::iterator iter, end, iter2, end2; + std::vector retval; + bool found; + std::string::const_iterator input_iter = input.begin(), input_end = input.end(); - int current_col = 0; - int current_line = 0; - boost::match_results what; + int current_col = 0; + int current_line = 0; + boost::match_results what; - while (input_iter != input_end) { - found = false; + while (input_iter != input_end) { + found = false; - if (regex_search(input_iter, input_end, what, singleline_comment_pattern.regex, boost::match_continuous)) { - std::string comment_start(what[0]); - input_iter += comment_start.size(); + if (regex_search(input_iter, input_end, what, singleline_comment_pattern.regex, boost::match_continuous)) { + std::string comment_start(what[0]); + input_iter += comment_start.size(); - bool found_eol = false; + bool found_eol = false; - while ((!found_eol) && (input_iter != input_end)) { - boost::match_results eol_delim; - if (regex_search(input_iter, input_end, eol_delim, line_sep_pattern.regex, boost::match_continuous)) { - std::string comment_end(eol_delim[0]); - input_iter += comment_end.size(); - ++current_line; - current_col = 0; - found_eol = true; - break; - } - if ((!found_eol) && (input_iter != input_end)) { - ++input_iter; + while ((!found_eol) && (input_iter != input_end)) { + boost::match_results eol_delim; + if (regex_search(input_iter, input_end, eol_delim, line_sep_pattern.regex, boost::match_continuous)) { + std::string comment_end(eol_delim[0]); + input_iter += comment_end.size(); + ++current_line; + current_col = 0; + found_eol = true; + break; + } + if ((!found_eol) && (input_iter != input_end)) { + ++input_iter; + } } } - } - else if (regex_search(input_iter, input_end, what, multiline_comment_start_pattern.regex, boost::match_continuous)) { - std::string comment_start(what[0]); - input_iter += comment_start.size(); + else if (regex_search(input_iter, input_end, what, multiline_comment_start_pattern.regex, boost::match_continuous)) { + std::string comment_start(what[0]); + input_iter += comment_start.size(); - bool found_eoc = false; + bool found_eoc = false; - while ((!found_eoc) && (input_iter != input_end)) { - boost::match_results eol_delim; - if (regex_search(input_iter, input_end, eol_delim, line_sep_pattern.regex, boost::match_continuous)) { - std::string comment_end(eol_delim[0]); - input_iter += comment_end.size(); - ++current_line; - current_col = 0; + while ((!found_eoc) && (input_iter != input_end)) { + boost::match_results eol_delim; + if (regex_search(input_iter, input_end, eol_delim, line_sep_pattern.regex, boost::match_continuous)) { + std::string comment_end(eol_delim[0]); + input_iter += comment_end.size(); + ++current_line; + current_col = 0; + } + boost::match_results eoc_delim; + if (regex_search(input_iter, input_end, eoc_delim, multiline_comment_end_pattern.regex, boost::match_continuous)) { + std::string comment_end(eoc_delim[0]); + input_iter += comment_end.size(); + current_col += comment_end.size(); + found_eoc = true; + break; + } + if ((!found_eoc) && (input_iter != input_end)) { + ++input_iter; + ++current_col; + } } - boost::match_results eoc_delim; - if (regex_search(input_iter, input_end, eoc_delim, multiline_comment_end_pattern.regex, boost::match_continuous)) { - std::string comment_end(eoc_delim[0]); - input_iter += comment_end.size(); - current_col += comment_end.size(); - found_eoc = true; - break; - } - if ((!found_eoc) && (input_iter != input_end)) { - ++input_iter; - ++current_col; + + if (!found_eoc) { + std::cout << "Incomplete comment block! Add exceptions!" << std::endl; + return retval; } } - - if (!found_eoc) { - std::cout << "Incomplete comment block! Add exceptions!" << std::endl; - return retval; + else if (regex_search(input_iter, input_end, what, skip_pattern.regex, boost::match_continuous)) { + std::string whitespace(what[0]); + input_iter += whitespace.size(); + current_col += whitespace.size(); + found = true; } - } - else if (regex_search(input_iter, input_end, what, skip_pattern.regex, boost::match_continuous)) { - std::string whitespace(what[0]); - input_iter += whitespace.size(); - current_col += whitespace.size(); - found = true; - } - else if (regex_search(input_iter, input_end, what, line_sep_pattern.regex, boost::match_continuous)) { - const std::string cr(what[0]); + else if (regex_search(input_iter, input_end, what, line_sep_pattern.regex, boost::match_continuous)) { + const std::string cr(what[0]); - boost::match_results if_delim; - if (regex_search(cr.begin(), cr.end(), if_delim, command_sep_pattern.regex, boost::match_continuous)) { - TokenPtr t(new Token(if_delim[0], command_sep_pattern.identifier, filename)); - t->start.column = current_col; - t->start.line = current_line; - current_col += t->text.size(); - t->end.column = current_col; - t->end.line = current_line; - retval.push_back(t); - } - - input_iter += cr.size(); - ++current_line; - current_col = 0; - found = true; - } - else if (regex_search(input_iter, input_end, what, command_sep_pattern.regex, boost::match_continuous)) { - TokenPtr t(new Token(what[0], command_sep_pattern.identifier, filename)); - t->start.column = current_col; - t->start.line = current_line; - current_col += t->text.size(); - t->end.column = current_col; - t->end.line = current_line; - retval.push_back(t); - input_iter += t->text.size(); - found = true; - } - else { - for (iter = lex_patterns.begin(), end = lex_patterns.end(); iter != end; ++iter) { - if (regex_search(input_iter, input_end, what, iter->regex, boost::match_continuous)) { - TokenPtr t(new Token(what[0], iter->identifier, filename)); + boost::match_results if_delim; + if (regex_search(cr.begin(), cr.end(), if_delim, command_sep_pattern.regex, boost::match_continuous)) { + TokenPtr t(new Token(if_delim[0], command_sep_pattern.identifier, filename)); t->start.column = current_col; t->start.line = current_line; current_col += t->text.size(); t->end.column = current_col; t->end.line = current_line; retval.push_back(t); - input_iter += t->text.size(); - found = true; - break; + } + + input_iter += cr.size(); + ++current_line; + current_col = 0; + found = true; + } + else if (regex_search(input_iter, input_end, what, command_sep_pattern.regex, boost::match_continuous)) { + TokenPtr t(new Token(what[0], command_sep_pattern.identifier, filename)); + t->start.column = current_col; + t->start.line = current_line; + current_col += t->text.size(); + t->end.column = current_col; + t->end.line = current_line; + retval.push_back(t); + input_iter += t->text.size(); + found = true; + } + else { + for (iter = lex_patterns.begin(), end = lex_patterns.end(); iter != end; ++iter) { + if (regex_search(input_iter, input_end, what, iter->regex, boost::match_continuous)) { + TokenPtr t(new Token(what[0], iter->identifier, filename)); + t->start.column = current_col; + t->start.line = current_line; + current_col += t->text.size(); + t->end.column = current_col; + t->end.line = current_line; + retval.push_back(t); + input_iter += t->text.size(); + found = true; + break; + } + } + + if (!found) { + const std::string err(input_iter, input_end); + std::cout << "Unknown string at: " << err << std::endl; + return retval; } } - - if (!found) { - const std::string err(input_iter, input_end); - std::cout << "Unknown string at: " << err << std::endl; - return retval; - } } + return retval; } - return retval; - } - - void set_skip(const Pattern &p) { - skip_pattern = p; - } - void set_line_sep(const Pattern &p) { - line_sep_pattern = p; - } - void set_command_sep(const Pattern &p) { - command_sep_pattern = p; - } - void set_multiline_comment(const Pattern &start, const Pattern &end) { - multiline_comment_start_pattern = start; - multiline_comment_end_pattern = end; - } - void set_singleline_comment(const Pattern &p) { - singleline_comment_pattern = p; - } -}; + void set_skip(const Pattern &p) { + skip_pattern = p; + } + void set_line_sep(const Pattern &p) { + line_sep_pattern = p; + } + void set_command_sep(const Pattern &p) { + command_sep_pattern = p; + } + void set_multiline_comment(const Pattern &start, const Pattern &end) { + multiline_comment_start_pattern = start; + multiline_comment_end_pattern = end; + } + void set_singleline_comment(const Pattern &p) { + singleline_comment_pattern = p; + } + }; +} #endif /* LANGKIT_LEXER_HPP_ */ diff --git a/langkit/langkit_parser.hpp b/langkit/langkit_parser.hpp index efa060f5..68b15625 100644 --- a/langkit/langkit_parser.hpp +++ b/langkit/langkit_parser.hpp @@ -8,96 +8,76 @@ #include "langkit_lexer.hpp" -struct RuleImpl; +namespace langkit +{ + struct RuleImpl; -typedef std::vector::iterator Token_Iterator; -typedef boost::function(Token_Iterator, Token_Iterator, TokenPtr, bool, int)> RuleFun; -typedef std::tr1::shared_ptr RuleImplPtr; + typedef std::vector::iterator Token_Iterator; + typedef boost::function(Token_Iterator, Token_Iterator, TokenPtr, bool, int)> RuleFun; + typedef std::tr1::shared_ptr RuleImplPtr; -struct RuleImpl { - RuleFun rule; - bool keep; - int new_id; + struct RuleImpl { + RuleFun rule; + bool keep; + int new_id; - RuleImpl() : keep(true), new_id(-1) {} - RuleImpl(int id) : keep(true), new_id(id) {} - RuleImpl(RuleFun fun) : rule(fun), keep(true), new_id(-1) {} - RuleImpl(RuleFun fun, bool keep_match) : rule(fun), keep(keep_match), new_id(-1) {} + RuleImpl() : keep(true), new_id(-1) {} + RuleImpl(int id) : keep(true), new_id(id) {} + RuleImpl(RuleFun fun) : rule(fun), keep(true), new_id(-1) {} + RuleImpl(RuleFun fun, bool keep_match) : rule(fun), keep(keep_match), new_id(-1) {} - std::pair operator()(Token_Iterator iter, Token_Iterator end, TokenPtr parent) { - return rule(iter, end, parent, keep, new_id); - } -}; - -//struct Rule; - -template -std::pair String_Rule(T_Iter iter, T_Iter end, TokenPtr parent, bool keep, int new_id, const std::string &val) { - if (iter != end) { - if ((*iter)->text == val) { - if (keep) { - parent->children.push_back(*iter); - } - return std::pair(++iter, true); + std::pair operator()(Token_Iterator iter, Token_Iterator end, TokenPtr parent) { + return rule(iter, end, parent, keep, new_id); } - } + }; - return std::pair(iter, false); -} + //struct Rule; -template -std::pair Id_Rule(T_Iter iter, T_Iter end, TokenPtr parent, bool keep, int new_id, const int val) { - if (iter != end) { - if ((*iter)->identifier == val) { - if (keep) { - parent->children.push_back(*iter); - } - return std::pair(++iter, true); - } - } - - return std::pair(iter, false); -} - -template -std::pair Or_Rule(T_Iter iter, T_Iter end, TokenPtr parent, bool keep, int new_id, R_Type lhs, R_Type rhs) { - T_Iter new_iter; - unsigned int prev_size; - TokenPtr prev_parent = parent; - - if (new_id != -1) { - parent = TokenPtr(new Token("", new_id, parent->filename)); - } - - prev_size = parent->children.size(); - - if (iter != end) { - std::pair result = lhs(iter, end, parent); - - if (result.second) { - if (new_id != -1) { - parent->filename = (*iter)->filename; - parent->start = (*iter)->start; - if (result.first == iter) { - parent->end = (*iter)->start; + template + std::pair String_Rule(T_Iter iter, T_Iter end, TokenPtr parent, bool keep, int new_id, const std::string &val) { + if (iter != end) { + if ((*iter)->text == val) { + if (keep) { + parent->children.push_back(*iter); } - else { - parent->end = (*(result.first - 1))->end; - } - prev_parent->children.push_back(parent); + return std::pair(++iter, true); } - return std::pair(result.first, true); } - else { - if (parent->children.size() != prev_size) { - //Clear out the partial matches - parent->children.erase(parent->children.begin() + prev_size, parent->children.end()); - } - result = rhs(iter, end, parent); + return std::pair(iter, false); + } + + template + std::pair Id_Rule(T_Iter iter, T_Iter end, TokenPtr parent, bool keep, int new_id, const int val) { + if (iter != end) { + if ((*iter)->identifier == val) { + if (keep) { + parent->children.push_back(*iter); + } + return std::pair(++iter, true); + } + } + + return std::pair(iter, false); + } + + template + std::pair Or_Rule(T_Iter iter, T_Iter end, TokenPtr parent, bool keep, int new_id, R_Type lhs, R_Type rhs) { + T_Iter new_iter; + unsigned int prev_size; + TokenPtr prev_parent = parent; + + if (new_id != -1) { + parent = TokenPtr(new Token("", new_id, parent->filename)); + } + + prev_size = parent->children.size(); + + if (iter != end) { + std::pair result = lhs(iter, end, parent); + if (result.second) { if (new_id != -1) { - parent->filename = (*iter)->filename; parent->start = (*iter)->start; if (result.first == iter) { @@ -106,129 +86,105 @@ std::pair Or_Rule(T_Iter iter, T_Iter end, TokenPtr parent, bool k else { parent->end = (*(result.first - 1))->end; } - prev_parent->children.push_back(parent); } return std::pair(result.first, true); } - } - } - - if (parent->children.size() != prev_size) { - //Clear out the partial matches - parent->children.erase(parent->children.begin() + prev_size, parent->children.end()); - } - - return std::pair(iter, false); -} - -template -std::pair And_Rule(T_Iter iter, T_Iter end, TokenPtr parent, bool keep, int new_id, R_Type lhs, R_Type rhs) { - T_Iter lhs_iter, rhs_iter; - unsigned int prev_size; - TokenPtr prev_parent = parent; - - if (new_id != -1) { - parent = TokenPtr(new Token("", new_id, parent->filename)); - } - - prev_size = parent->children.size(); - - if (iter != end) { - std::pair result = lhs(iter, end, parent); - - if (result.second) { - result = rhs(result.first, end, parent); - if (result.second) { - if (new_id != -1) { - - parent->filename = (*iter)->filename; - parent->start = (*iter)->start; - if (result.first == iter) { - parent->end = (*iter)->start; - } - else { - parent->end = (*(result.first - 1))->end; - } - - prev_parent->children.push_back(parent); + else { + if (parent->children.size() != prev_size) { + //Clear out the partial matches + parent->children.erase(parent->children.begin() + prev_size, parent->children.end()); + } + + result = rhs(iter, end, parent); + if (result.second) { + if (new_id != -1) { + + parent->filename = (*iter)->filename; + parent->start = (*iter)->start; + if (result.first == iter) { + parent->end = (*iter)->start; + } + else { + parent->end = (*(result.first - 1))->end; + } + + prev_parent->children.push_back(parent); + } + return std::pair(result.first, true); } - return std::pair(result.first, true); } } + + if (parent->children.size() != prev_size) { + //Clear out the partial matches + parent->children.erase(parent->children.begin() + prev_size, parent->children.end()); + } + + return std::pair(iter, false); } - if (parent->children.size() != prev_size) { - //Clear out the partial matches - parent->children.erase(parent->children.begin() + prev_size, parent->children.end()); - } + template + std::pair And_Rule(T_Iter iter, T_Iter end, TokenPtr parent, bool keep, int new_id, R_Type lhs, R_Type rhs) { + T_Iter lhs_iter, rhs_iter; + unsigned int prev_size; + TokenPtr prev_parent = parent; - return std::pair(iter, false); -} - -template -std::pair Kleene_Rule - (T_Iter iter, T_Iter end, TokenPtr parent, bool keep, int new_id, R_Type rule) { - - TokenPtr prev_parent = parent; - std::pair result; - T_Iter new_iter = iter; - - if (iter != end) { if (new_id != -1) { parent = TokenPtr(new Token("", new_id, parent->filename)); } - result.second = true; - while (result.second == true) { - result = rule(new_iter, end, parent); - new_iter = result.first; + prev_size = parent->children.size(); + + if (iter != end) { + std::pair result = lhs(iter, end, parent); + + if (result.second) { + result = rhs(result.first, end, parent); + if (result.second) { + if (new_id != -1) { + + parent->filename = (*iter)->filename; + parent->start = (*iter)->start; + if (result.first == iter) { + parent->end = (*iter)->start; + } + else { + parent->end = (*(result.first - 1))->end; + } + + prev_parent->children.push_back(parent); + } + return std::pair(result.first, true); + } + } } - if (new_id != -1) { - - parent->filename = (*iter)->filename; - parent->start = (*iter)->start; - if (result.first == iter) { - parent->end = (*iter)->start; - } - else { - parent->end = (*(result.first - 1))->end; - } - - prev_parent->children.push_back(parent); + if (parent->children.size() != prev_size) { + //Clear out the partial matches + parent->children.erase(parent->children.begin() + prev_size, parent->children.end()); } - return std::pair(result.first, true); - } - else { - return std::pair(iter, true); - } -} -template -std::pair Plus_Rule - (T_Iter iter, T_Iter end, TokenPtr parent, bool keep, int new_id, R_Type rule) { - - unsigned int prev_size; - TokenPtr prev_parent = parent; - T_Iter loop_iter = iter; - - if (new_id != -1) { - parent = TokenPtr(new Token("", new_id, parent->filename)); + return std::pair(iter, false); } - prev_size = parent->children.size(); + template + std::pair Kleene_Rule + (T_Iter iter, T_Iter end, TokenPtr parent, bool keep, int new_id, R_Type rule) { - if (iter != end) { + TokenPtr prev_parent = parent; std::pair result; - result = rule(loop_iter, end, parent); + T_Iter new_iter = iter; + + if (iter != end) { + if (new_id != -1) { + parent = TokenPtr(new Token("", new_id, parent->filename)); + } - if (result.second == true) { - loop_iter = result.first; result.second = true; - while ((loop_iter != end) && (result.second == true)) { - result = rule(loop_iter, end, parent); - loop_iter = result.first; + while (result.second == true) { + result = rule(new_iter, end, parent); + new_iter = result.first; } if (new_id != -1) { @@ -244,40 +200,122 @@ std::pair Plus_Rule prev_parent->children.push_back(parent); } - return std::pair(result.first, true); } + else { + return std::pair(iter, true); + } } - if (parent->children.size() != prev_size) { - //Clear out the partial matches - parent->children.erase(parent->children.begin() + prev_size, parent->children.end()); + template + std::pair Plus_Rule + (T_Iter iter, T_Iter end, TokenPtr parent, bool keep, int new_id, R_Type rule) { + + unsigned int prev_size; + TokenPtr prev_parent = parent; + T_Iter loop_iter = iter; + + if (new_id != -1) { + parent = TokenPtr(new Token("", new_id, parent->filename)); + } + + prev_size = parent->children.size(); + + if (iter != end) { + std::pair result; + result = rule(loop_iter, end, parent); + + if (result.second == true) { + loop_iter = result.first; + result.second = true; + while ((loop_iter != end) && (result.second == true)) { + result = rule(loop_iter, end, parent); + loop_iter = result.first; + } + + if (new_id != -1) { + + parent->filename = (*iter)->filename; + parent->start = (*iter)->start; + if (result.first == iter) { + parent->end = (*iter)->start; + } + else { + parent->end = (*(result.first - 1))->end; + } + + prev_parent->children.push_back(parent); + } + + return std::pair(result.first, true); + } + } + + if (parent->children.size() != prev_size) { + //Clear out the partial matches + parent->children.erase(parent->children.begin() + prev_size, parent->children.end()); + } + + return std::pair(iter, false); } - return std::pair(iter, false); -} + template + std::pair Optional_Rule + (T_Iter iter, T_Iter end, TokenPtr parent, bool keep, int new_id, R_Type rule) { -template -std::pair Optional_Rule - (T_Iter iter, T_Iter end, TokenPtr parent, bool keep, int new_id, R_Type rule) { + TokenPtr prev_parent = parent; + T_Iter new_iter = iter; - TokenPtr prev_parent = parent; - T_Iter new_iter = iter; + if (iter != end) { + if (new_id != -1) { + parent = TokenPtr(new Token("", new_id, parent->filename)); + } + + std::pair result; + result.second = true; + if ((new_iter != end) && (result.second == true)) { + result = rule(new_iter, end, parent); + new_iter = result.first; + } + + if (new_id != -1) { + + parent->filename = (*iter)->filename; + parent->start = (*iter)->start; + if (result.first == iter) { + parent->end = (*iter)->start; + } + else { + parent->end = (*(result.first - 1))->end; + } + + prev_parent->children.push_back(parent); + } + return std::pair(result.first, true); + } + else { + return std::pair(iter, true); + } + } + + template + std::pair Epsilon_Rule + (T_Iter iter, T_Iter end, TokenPtr parent, bool keep, int new_id, R_Type rule) { + + TokenPtr prev_parent = parent; + T_Iter new_iter = iter; - if (iter != end) { if (new_id != -1) { parent = TokenPtr(new Token("", new_id, parent->filename)); } std::pair result; - result.second = true; - if ((new_iter != end) && (result.second == true)) { + if ((new_iter != end)) { result = rule(new_iter, end, parent); new_iter = result.first; } if (new_id != -1) { - parent->filename = (*iter)->filename; parent->start = (*iter)->start; if (result.first == iter) { @@ -289,158 +327,123 @@ std::pair Optional_Rule prev_parent->children.push_back(parent); } - return std::pair(result.first, true); - } - else { - return std::pair(iter, true); - } -} -template -std::pair Epsilon_Rule - (T_Iter iter, T_Iter end, TokenPtr parent, bool keep, int new_id, R_Type rule) { - - TokenPtr prev_parent = parent; - T_Iter new_iter = iter; - - if (new_id != -1) { - parent = TokenPtr(new Token("", new_id, parent->filename)); + return std::pair(iter, result.second); } - std::pair result; - if ((new_iter != end)) { - result = rule(new_iter, end, parent); - new_iter = result.first; - } + template + std::pair Wrap_Rule + (T_Iter iter, T_Iter end, TokenPtr parent, bool keep, int new_id, R_Type rule) { - if (new_id != -1) { - parent->filename = (*iter)->filename; - parent->start = (*iter)->start; - if (result.first == iter) { - parent->end = (*iter)->start; - } - else { - parent->end = (*(result.first - 1))->end; + TokenPtr prev_parent = parent; + T_Iter new_iter = iter; + + if (new_id != -1) { + parent = TokenPtr(new Token("", new_id, parent->filename)); } - prev_parent->children.push_back(parent); - } - - return std::pair(iter, result.second); -} - -template -std::pair Wrap_Rule - (T_Iter iter, T_Iter end, TokenPtr parent, bool keep, int new_id, R_Type rule) { - - TokenPtr prev_parent = parent; - T_Iter new_iter = iter; - - if (new_id != -1) { - parent = TokenPtr(new Token("", new_id, parent->filename)); - } - - std::pair result; - if ((new_iter != end)) { - result = rule(new_iter, end, parent); - new_iter = result.first; - } - - if (new_id != -1) { - parent->filename = (*iter)->filename; - parent->start = (*iter)->start; - if (result.first == iter) { - parent->end = (*iter)->start; - } - else { - parent->end = (*(result.first - 1))->end; + std::pair result; + if ((new_iter != end)) { + result = rule(new_iter, end, parent); + new_iter = result.first; } - prev_parent->children.push_back(parent); + if (new_id != -1) { + parent->filename = (*iter)->filename; + parent->start = (*iter)->start; + if (result.first == iter) { + parent->end = (*iter)->start; + } + else { + parent->end = (*(result.first - 1))->end; + } + + prev_parent->children.push_back(parent); + } + + return std::pair(result.first, result.second); } - return std::pair(result.first, result.second); -} + template + std::pair Ignore_Rule + (T_Iter iter, T_Iter end, TokenPtr parent, bool keep, int new_id, R_Type rule) { -template -std::pair Ignore_Rule - (T_Iter iter, T_Iter end, TokenPtr parent, bool keep, int new_id, R_Type rule) { + rule.impl->keep = false; - rule.impl->keep = false; - - return rule(iter, end, parent); -} - -struct Rule { - RuleImplPtr impl; - - Rule() : impl(new RuleImpl()) {} - Rule(int id) : impl(new RuleImpl(id)) {} - Rule(RuleFun fun) : impl(new RuleImpl(fun)) {} - Rule(RuleFun fun, bool keep) : impl(new RuleImpl(fun, keep)) {} - - std::pair operator()(Token_Iterator iter, Token_Iterator end, TokenPtr parent) { - return (*impl)(iter, end, parent); + return rule(iter, end, parent); } - Rule &operator=(const Rule &rule) { - int prev_id = impl->new_id; - *impl = *(rule.impl); - impl->new_id = prev_id; + struct Rule { + RuleImplPtr impl; - return *this; + Rule() : impl(new RuleImpl()) {} + Rule(int id) : impl(new RuleImpl(id)) {} + Rule(RuleFun fun) : impl(new RuleImpl(fun)) {} + Rule(RuleFun fun, bool keep) : impl(new RuleImpl(fun, keep)) {} + + std::pair operator()(Token_Iterator iter, Token_Iterator end, TokenPtr parent) { + return (*impl)(iter, end, parent); + } + + Rule &operator=(const Rule &rule) { + int prev_id = impl->new_id; + *impl = *(rule.impl); + impl->new_id = prev_id; + + return *this; + } + + }; + + inline Rule operator>>(const Rule &lhs, const Rule &rhs) { + return Rule(boost::bind(And_Rule, _1, _2, _3, _4, _5, lhs, rhs)); } -}; + inline Rule operator|(const Rule &lhs, const Rule &rhs) { + return Rule(boost::bind(Or_Rule, _1, _2, _3, _4, _5, lhs, rhs)); + } -inline Rule operator>>(const Rule &lhs, const Rule &rhs) { - return Rule(boost::bind(And_Rule, _1, _2, _3, _4, _5, lhs, rhs)); -} + inline Rule operator*(const Rule &operand) { + return Rule(boost::bind(Kleene_Rule, _1, _2, _3, _4, _5, operand)); + } -inline Rule operator|(const Rule &lhs, const Rule &rhs) { - return Rule(boost::bind(Or_Rule, _1, _2, _3, _4, _5, lhs, rhs)); -} + inline Rule operator+(const Rule &operand) { + return Rule(boost::bind(Plus_Rule, _1, _2, _3, _4, _5, operand)); + } -inline Rule operator*(const Rule &operand) { - return Rule(boost::bind(Kleene_Rule, _1, _2, _3, _4, _5, operand)); -} - -inline Rule operator+(const Rule &operand) { - return Rule(boost::bind(Plus_Rule, _1, _2, _3, _4, _5, operand)); -} - -inline Rule operator~(const Rule &operand) { - return Rule(boost::bind(Optional_Rule, _1, _2, _3, _4, _5, operand)); -} + inline Rule operator~(const Rule &operand) { + return Rule(boost::bind(Optional_Rule, _1, _2, _3, _4, _5, operand)); + } -template (*Function)(ItrType, ItrType, TokenPtr, bool, int, ParamType)> -struct Rule_Builder -{ - Rule_Builder(ParamType p, bool t_keep = true) - : m_p(p), m_keep(t_keep) + template (*Function)(ItrType, ItrType, TokenPtr, bool, int, ParamType)> + struct Rule_Builder { + Rule_Builder(ParamType p, bool t_keep = true) + : m_p(p), m_keep(t_keep) + { - } + } - // Auto conversion operator is the glue here. - // In one sense this option cleans up the impl quite a bit, with much fewer code - // repeats in all the rule builders. - // In another sense, it might take a couple of tries to get it right. - operator Rule() { - return Rule(boost::bind(Function, _1, _2, _3, _4, _5, m_p), m_keep); - } + // Auto conversion operator is the glue here. + // In one sense this option cleans up the impl quite a bit, with much fewer code + // repeats in all the rule builders. + // In another sense, it might take a couple of tries to get it right. + operator Rule() { + return Rule(boost::bind(Function, _1, _2, _3, _4, _5, m_p), m_keep); + } - ParamType m_p; - bool m_keep; -}; + ParamType m_p; + bool m_keep; + }; -typedef Rule_Builder > Epsilon; -typedef Rule_Builder > Wrap; -typedef Rule_Builder > Ign; -typedef Rule_Builder > Id; -typedef Rule_Builder > Str; + typedef Rule_Builder > Epsilon; + typedef Rule_Builder > Wrap; + typedef Rule_Builder > Ign; + typedef Rule_Builder > Id; + typedef Rule_Builder > Str; +} #endif /* LANGKIT_PARSER_HPP_ */ diff --git a/langkit/main.cpp b/langkit/main.cpp index 19dff702..ff34425d 100644 --- a/langkit/main.cpp +++ b/langkit/main.cpp @@ -24,7 +24,7 @@ const char *tokentype_to_string(int tokentype) { return token_types[tokentype]; } -void debug_print(TokenPtr token, std::string prepend) { +void debug_print(langkit::TokenPtr token, std::string prepend) { std::cout << prepend << "Token: " << token->text << "(" << tokentype_to_string(token->identifier) << ") @ " << token->filename << ": (" << token->start.line << ", " << token->start.column << ") to (" << token->end.line << ", " << token->end.column << ") " << std::endl; @@ -34,7 +34,7 @@ void debug_print(TokenPtr token, std::string prepend) { } } -void debug_print(std::vector &tokens) { +void debug_print(std::vector &tokens) { for (unsigned int i = 0; i < tokens.size(); ++i) { debug_print(tokens[i], ""); } @@ -59,7 +59,8 @@ std::string load_file(const char *filename) { return ret_val; } -void parse(std::vector &tokens, const char *filename) { +void parse(std::vector &tokens, const char *filename) { + using namespace langkit; /* Rule lhs; @@ -161,6 +162,7 @@ void parse(std::vector &tokens, const char *filename) { int main(int argc, char *argv[]) { + using namespace langkit; std::string input; Lexer lexer;