From e49df4c54df5f93e9eaec985b6e61835cb277f86 Mon Sep 17 00:00:00 2001 From: Jason Turner Date: Fri, 25 Aug 2017 11:17:47 -0600 Subject: [PATCH] Move the parser to string_view --- include/chaiscript/chaiscript_defines.hpp | 10 +++ .../chaiscript/dispatchkit/dispatchkit.hpp | 10 --- .../chaiscript/language/chaiscript_common.hpp | 29 +++++++++ .../chaiscript/language/chaiscript_parser.hpp | 61 ++++++++++--------- include/chaiscript/utility/fnv1a.hpp | 24 ++++++-- include/chaiscript/utility/static_string.hpp | 24 ++++++++ 6 files changed, 114 insertions(+), 44 deletions(-) diff --git a/include/chaiscript/chaiscript_defines.hpp b/include/chaiscript/chaiscript_defines.hpp index 44fa86e4..ef937620 100644 --- a/include/chaiscript/chaiscript_defines.hpp +++ b/include/chaiscript/chaiscript_defines.hpp @@ -210,6 +210,16 @@ namespace chaiscript { return final_value(t, base, exponent, neg_exponent); } + struct str_less { + bool operator()(const std::string &t_lhs, const std::string &t_rhs) const noexcept { + return t_lhs < t_rhs; + } + template + bool operator()(const LHS &t_lhs, const RHS &t_rhs) const noexcept { + return std::lexicographical_compare(t_lhs.begin(), t_lhs.end(), t_rhs.begin(), t_rhs.end()); + } + struct is_transparent{}; + }; enum class Options { diff --git a/include/chaiscript/dispatchkit/dispatchkit.hpp b/include/chaiscript/dispatchkit/dispatchkit.hpp index ccaa03c0..f51aa0cf 100644 --- a/include/chaiscript/dispatchkit/dispatchkit.hpp +++ b/include/chaiscript/dispatchkit/dispatchkit.hpp @@ -437,16 +437,6 @@ namespace chaiscript /// of the object stack, functions and registered types. class Dispatch_Engine { - struct str_less { - bool operator()(const std::string &t_lhs, const std::string &t_rhs) const noexcept { - return t_lhs < t_rhs; - } - template - bool operator()(const LHS &t_lhs, const RHS &t_rhs) const noexcept { - return std::lexicographical_compare(t_lhs.begin(), t_lhs.end(), t_rhs.begin(), t_rhs.end()); - } - struct is_transparent{}; - }; public: typedef std::map Type_Name_Map; diff --git a/include/chaiscript/language/chaiscript_common.hpp b/include/chaiscript/language/chaiscript_common.hpp index da25af22..82aa478b 100644 --- a/include/chaiscript/language/chaiscript_common.hpp +++ b/include/chaiscript/language/chaiscript_common.hpp @@ -45,6 +45,19 @@ namespace chaiscript return words.count(name) == 1; } + static bool is_reserved_word(const std::string_view &name) noexcept + { + static const std::unordered_set words { + "def", "fun", "while", "for", "if", "else", + "&&", "||", ",", "auto", "return", "break", + "true", "false", "class", "attr", "var", "global", + "GLOBAL", "_", + "__LINE__", "__FILE__", "__FUNC__", "__CLASS__" + }; + + return words.count(name) == 1; + } + static bool valid_object_name(const std::string &name) noexcept { return name.find("::") == std::string::npos && !is_reserved_word(name); @@ -60,6 +73,22 @@ namespace chaiscript throw exception::illegal_name_error(name); } } + + static bool valid_object_name(const std::string_view &name) noexcept + { + return name.find("::") == std::string::npos && !is_reserved_word(name); + } + + static void validate_object_name(const std::string_view &name) + { + if (is_reserved_word(name)) { + throw exception::reserved_word_error(std::string(name)); + } + + if (name.find("::") != std::string::npos) { + throw exception::illegal_name_error(std::string(name)); + } + } }; /// Signature of module entry point that all binary loadable modules must implement. diff --git a/include/chaiscript/language/chaiscript_parser.hpp b/include/chaiscript/language/chaiscript_parser.hpp index 71112298..3c679639 100644 --- a/include/chaiscript/language/chaiscript_parser.hpp +++ b/include/chaiscript/language/chaiscript_parser.hpp @@ -210,7 +210,7 @@ namespace chaiscript std::array m_10 {{SS("*"), SS("/"), SS("%")}}; std::array m_11 {{SS("++"), SS("--"), SS("-"), SS("+"), SS("!"), SS("~")}}; - bool is_match(const std::string &t_str) const noexcept { + bool is_match(const std::string_view &t_str) const noexcept { constexpr std::array groups{{0,1,2,3,4,5,6,7,8,9,10,11}}; return std::any_of(groups.begin(), groups.end(), [&t_str, this](const std::size_t group){ return is_match(group, t_str); }); } @@ -239,9 +239,9 @@ namespace chaiscript } } - bool is_match(const std::size_t t_group, const std::string &t_str) const noexcept { + constexpr bool is_match(const std::size_t t_group, const std::string_view &t_str) const noexcept { auto match = [&t_str](const auto &array) { - return std::any_of(array.begin(), array.end(), [&t_str](const auto &v){ return v.c_str() == t_str; }); + return std::any_of(array.begin(), array.end(), [&t_str](const auto &v){ return v == t_str; }); }; switch (t_group) { @@ -325,9 +325,9 @@ namespace chaiscript { } - static std::string str(const Position &t_begin, const Position &t_end) noexcept { + static std::string_view str(const Position &t_begin, const Position &t_end) noexcept { if (t_begin.m_pos != nullptr && t_end.m_pos != nullptr) { - return std::string(t_begin.m_pos, t_end.m_pos); + return std::string_view(t_begin.m_pos, std::distance(t_begin.m_pos, t_end.m_pos)); } else { return {}; } @@ -423,10 +423,10 @@ namespace chaiscript Tracer m_tracer; Optimizer m_optimizer; - void validate_object_name(const std::string &name) const + void validate_object_name(const std::string_view &name) const { if (!Name_Validator::valid_object_name(name)) { - throw exception::eval_error("Invalid Object Name: " + name, File_Position(m_position.line, m_position.col), *m_filename); + throw exception::eval_error("Invalid Object Name: " + std::string(name), File_Position(m_position.line, m_position.col), *m_filename); } } @@ -720,7 +720,7 @@ namespace chaiscript } /// Parses a floating point value and returns a Boxed_Value representation of it - static Boxed_Value buildFloat(const std::string &t_val) + static Boxed_Value buildFloat(const std::string_view &t_val) { bool float_ = false; bool long_ = false; @@ -753,7 +753,7 @@ namespace chaiscript - static Boxed_Value buildInt(const int base, const std::string &t_val, const bool prefixed) + static Boxed_Value buildInt(const int base, std::string_view t_val, const bool prefixed) { bool unsigned_ = false; bool long_ = false; @@ -780,7 +780,7 @@ namespace chaiscript } } - const auto val = prefixed?std::string(t_val.begin()+2,t_val.end()):t_val; + if (prefixed) { t_val.remove_prefix(2); }; #ifdef __GNUC__ #pragma GCC diagnostic push @@ -793,7 +793,8 @@ namespace chaiscript #endif try { - auto u = std::stoll(val,nullptr,base); + /// TODO fix this to use from_chars + auto u = std::stoll(std::string(t_val),nullptr,base); if (!unsigned_ && !long_ && u >= std::numeric_limits::min() && u <= std::numeric_limits::max()) { @@ -813,7 +814,8 @@ namespace chaiscript } catch (const std::out_of_range &) { // too big to be signed try { - auto u = std::stoull(val,nullptr,base); + /// TODO fix this to use from_chars + auto u = std::stoull(std::string(t_val),nullptr,base); if (!longlong_ && u >= std::numeric_limits::min() && u <= std::numeric_limits::max()) { return const_var(static_cast(u)); @@ -833,9 +835,9 @@ namespace chaiscript } template - std::unique_ptr> make_node(std::string t_match, const int t_prev_line, const int t_prev_col, Param && ...param) + std::unique_ptr> make_node(std::string_view t_match, const int t_prev_line, const int t_prev_col, Param && ...param) { - return chaiscript::make_unique, T>(std::move(t_match), Parse_Location(m_filename, t_prev_line, t_prev_col, m_position.line, m_position.col), std::forward(param)...); + return chaiscript::make_unique, T>(std::string(t_match), Parse_Location(m_filename, t_prev_line, t_prev_col, m_position.line, m_position.col), std::forward(param)...); } /// Reads a number from the input, detecting if it's an integer or floating point @@ -848,20 +850,20 @@ namespace chaiscript if (Hex_()) { auto match = Position::str(start, m_position); auto bv = buildInt(16, match, true); - m_match_stack.emplace_back(make_node>(std::move(match), start.line, start.col, std::move(bv))); + m_match_stack.emplace_back(make_node>(match, start.line, start.col, std::move(bv))); return true; } if (Binary_()) { auto match = Position::str(start, m_position); auto bv = buildInt(2, match, true); - m_match_stack.push_back(make_node>(std::move(match), start.line, start.col, std::move(bv))); + m_match_stack.push_back(make_node>(match, start.line, start.col, std::move(bv))); return true; } if (Float_()) { auto match = Position::str(start, m_position); auto bv = buildFloat(match); - m_match_stack.push_back(make_node>(std::move(match), start.line, start.col, std::move(bv))); + m_match_stack.push_back(make_node>(match, start.line, start.col, std::move(bv))); return true; } else { @@ -869,11 +871,11 @@ namespace chaiscript auto match = Position::str(start, m_position); if (!match.empty() && (match[0] == '0')) { auto bv = buildInt(8, match, false); - m_match_stack.push_back(make_node>(std::move(match), start.line, start.col, std::move(bv))); + m_match_stack.push_back(make_node>(match, start.line, start.col, std::move(bv))); } else if (!match.empty()) { auto bv = buildInt(10, match, false); - m_match_stack.push_back(make_node>(std::move(match), start.line, start.col, std::move(bv))); + m_match_stack.push_back(make_node>(match, start.line, start.col, std::move(bv))); } else { return false; } @@ -932,7 +934,7 @@ namespace chaiscript if (Id_()) { auto text = Position::str(start, m_position); - const auto text_hash = utility::fnv1a_32(text.c_str()); + const auto text_hash = utility::fnv1a_32(text); if (validate) { validate_object_name(text); @@ -945,25 +947,25 @@ namespace chaiscript switch (text_hash) { case utility::fnv1a_32("true"): { - m_match_stack.push_back(make_node>(std::move(text), start.line, start.col, const_var(true))); + m_match_stack.push_back(make_node>(text, start.line, start.col, const_var(true))); } break; case utility::fnv1a_32("false"): { - m_match_stack.push_back(make_node>(std::move(text), start.line, start.col, const_var(false))); + m_match_stack.push_back(make_node>(text, start.line, start.col, const_var(false))); } break; case utility::fnv1a_32("Infinity"): { - m_match_stack.push_back(make_node>(std::move(text), start.line, start.col, + m_match_stack.push_back(make_node>(text, start.line, start.col, const_var(std::numeric_limits::infinity()))); } break; case utility::fnv1a_32("NaN"): { - m_match_stack.push_back(make_node>(std::move(text), start.line, start.col, + m_match_stack.push_back(make_node>(text, start.line, start.col, const_var(std::numeric_limits::quiet_NaN()))); } break; case utility::fnv1a_32("__LINE__"): { - m_match_stack.push_back(make_node>(std::move(text), start.line, start.col, + m_match_stack.push_back(make_node>(text, start.line, start.col, const_var(start.line))); } break; case utility::fnv1a_32("__FILE__"): { - m_match_stack.push_back(make_node>(std::move(text), start.line, start.col, + m_match_stack.push_back(make_node>(text, start.line, start.col, const_var(m_filename))); } break; case utility::fnv1a_32("__FUNC__"): { @@ -976,7 +978,7 @@ namespace chaiscript } } - m_match_stack.push_back(make_node>(std::move(text), start.line, start.col, + m_match_stack.push_back(make_node>(text, start.line, start.col, const_var(fun_name))); } break; case utility::fnv1a_32("__CLASS__"): { @@ -998,10 +1000,11 @@ namespace chaiscript Boxed_Value(std::make_shared()))); } break; default: { - std::string val = std::move(text); + auto val = text; if (*start == '`') { // 'escaped' literal, like an operator name val = Position::str(start+1, m_position-1); + // val.remove_prefix(1); val.remove_suffix(1); } m_match_stack.push_back(make_node>(val, start.line, start.col)); } break; @@ -1444,7 +1447,7 @@ namespace chaiscript return retval; } - bool is_operator(const std::string &t_s) const noexcept { + bool is_operator(const std::string_view &t_s) const noexcept { constexpr Operator_Matches operator_matches; return operator_matches.is_match(t_s); } diff --git a/include/chaiscript/utility/fnv1a.hpp b/include/chaiscript/utility/fnv1a.hpp index 2c55f43d..8442764a 100644 --- a/include/chaiscript/utility/fnv1a.hpp +++ b/include/chaiscript/utility/fnv1a.hpp @@ -18,9 +18,8 @@ namespace chaiscript namespace utility { - - - static constexpr std::uint32_t fnv1a_32(const char *s, std::uint32_t h = 0x811c9dc5) noexcept { + template + static constexpr std::uint32_t fnv1a_32(Itr begin, Itr end) noexcept { #ifdef __GNUC__ #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wsign-conversion" @@ -30,7 +29,14 @@ namespace chaiscript #pragma warning(push) #pragma warning(disable : 4307) #endif - return (*s == 0) ? h : fnv1a_32(s+1, ((h ^ (*s)) * 0x01000193)); + std::uint32_t h = 0x811c9dc5; + + while (begin != end) { + h = (h ^ (*begin)) * 0x01000193; + ++begin; + } + return h; + #ifdef CHAISCRIPT_MSVC #pragma warning(pop) #endif @@ -42,8 +48,16 @@ namespace chaiscript } - } + template + static constexpr std::uint32_t fnv1a_32(const char (&str)[N]) noexcept { + return fnv1a_32(std::begin(str), std::end(str)-1); + } + static constexpr std::uint32_t fnv1a_32(const std::string_view &sv) noexcept { + return fnv1a_32(sv.begin(), sv.end()); + } + + } } diff --git a/include/chaiscript/utility/static_string.hpp b/include/chaiscript/utility/static_string.hpp index fb63f28e..f6e4df1f 100644 --- a/include/chaiscript/utility/static_string.hpp +++ b/include/chaiscript/utility/static_string.hpp @@ -27,6 +27,30 @@ namespace chaiscript constexpr const char *c_str() const noexcept { return data; } + + constexpr auto begin() const noexcept { + return data; + } + + constexpr auto end() const noexcept { + return data + m_size; + } + + constexpr bool operator==(const std::string_view &other) const noexcept { + //return std::string_view(data, m_size) == other; + auto b1 = begin(); + const auto e1 = end(); + auto b2 = other.begin(); + const auto e2 = other.end(); + + if (e1 - b1 != e2 - b2) { return false; } + + while (b1 != e1) { + if (*b1 != *b2) { return false; } + ++b1; ++b2; + } + return true; + } const size_t m_size; const char *data = nullptr;