diff --git a/langkit/langkit_lexer.cpp b/langkit/langkit_lexer.cpp index c0394eb3..0550f88e 100644 --- a/langkit/langkit_lexer.cpp +++ b/langkit/langkit_lexer.cpp @@ -9,7 +9,7 @@ Lexer Lexer::operator<<(const Pattern &p) { return *this; } -std::vector Lexer::lex(const std::string &input) { +std::vector Lexer::lex(const std::string &input, const char *filename) { std::vector::iterator iter, end; std::vector retval; bool found; @@ -23,7 +23,7 @@ std::vector Lexer::lex(const std::string &input) { for (iter = lex_patterns.begin(), end = lex_patterns.end(); iter != end; ++iter) { boost::match_results what; if (regex_search(input_iter, input.end(), what, iter->regex, boost::match_continuous)) { - Token t(what[0], iter->identifier); + Token t(what[0], iter->identifier, filename); t.start.column = current_col; t.start.line = current_line; current_col += t.text.size(); diff --git a/langkit/langkit_lexer.hpp b/langkit/langkit_lexer.hpp index 4c8a959e..72c5251b 100644 --- a/langkit/langkit_lexer.hpp +++ b/langkit/langkit_lexer.hpp @@ -10,12 +10,11 @@ struct File_Position { int line; int column; - char *filename; - File_Position(int file_line, int file_column, char *fname) - : line(file_line), column(file_column), filename(fname) { } + File_Position(int file_line, int file_column) + : line(file_line), column(file_column) { } - File_Position() : line(0), column(0), filename(NULL) { } + File_Position() : line(0), column(0) { } }; struct Pattern { @@ -28,11 +27,12 @@ struct Pattern { struct Token { std::string text; int identifier; + const char *filename; File_Position start, end; std::vector children; - Token(const std::string &token_text, int id) : text(token_text), identifier(id) { } + Token(const std::string &token_text, int id, const char *fname) : text(token_text), identifier(id), filename(fname) { } }; struct Lexer { @@ -42,7 +42,7 @@ struct Lexer { std::vector line_sep_patterns; Lexer operator<<(const Pattern &p); - std::vector lex(const std::string &input); + std::vector lex(const std::string &input, const char *fname); void set_skip(const Pattern &p); void set_line_sep(const Pattern &p); diff --git a/langkit/main.cpp b/langkit/main.cpp index 0ac5ff0b..67d9be16 100644 --- a/langkit/main.cpp +++ b/langkit/main.cpp @@ -14,7 +14,7 @@ class TokenType { public: enum Type { Whitespace, Identifier, Number, Operator, Square_Open, Square_Close, Curly_Open, Curly_Close, Comma, Quoted_String, Single_Quoted_String, Carriage_Return, Semicolon }; }; void debug_print(Token &token) { - std::cout << "Token: " << token.text << "(" << token.identifier << ") @ (" << token.start.column + std::cout << "Token: " << token.text << "(" << token.identifier << ") @ " << token.filename << ": (" << token.start.column << ", " << token.start.line << ") to (" << token.end.column << ", " << token.end.line << ") " << std::endl; } void debug_print(std::vector &tokens) { @@ -84,7 +84,7 @@ int main(int argc, char *argv[]) { std::cout << "Expression> "; std::getline(std::cin, input); while (input != "quit") { - std::vector tokens = lexer.lex(input); + std::vector tokens = lexer.lex(input, "INPUT"); debug_print(tokens); parse(tokens); @@ -93,7 +93,7 @@ int main(int argc, char *argv[]) { } } else { - std::vector tokens = lexer.lex(load_file(argv[1])); + std::vector tokens = lexer.lex(load_file(argv[1]), argv[1]); debug_print(tokens); } }