Initial get_token code

include/etl/private/string_utilities_helper.h

@@ -493,6 +493,39 @@ namespace etl
       }
     }
 
+    //***************************************************************************
+    /// get_token
+    //***************************************************************************
+    template <typename TStringView, typename TIString>
+    TStringView get_token(const TIString& s, typename TIString::const_pointer delimiters, const TStringView& last_view)
+    {
+      typename TIString::const_iterator first = last_view.end();
+      typename TIString::const_iterator last  = last_view.end();
+
+      // The last view was default constructed, so we must be looking for the first token.
+      if (last_view.data() == ETL_NULLPTR)
+      {
+        first = s.begin();
+        last  = s.begin();
+      }
+
+      // Look for the start of the next token.
+      size_t first_position = s.find_first_not_of(delimiters, std::distance(s.begin(), last));
+      size_t last_position  = s.find_first_of(delimiters, first_position);
+
+      if (first_position == TIString::npos)
+      {
+        first_position = s.size();
+      }
+
+      if (last_position == TIString::npos)
+      {
+        last_position = s.size();
+      }
+
+      return TStringView(s.begin() + first_position, s.begin() + last_position);
+    }
+
     ////***************************************************************************
     ///// 
     ////***************************************************************************

include/etl/string_utilities.h

@@ -604,6 +604,14 @@ namespace etl
   {
     etl::private_string_utilities::transform_strings<etl::istring>(s, pairsbegin, pairsend);
   }
+
+  //***************************************************************************
+  /// get_token
+  //***************************************************************************
+  etl::string_view get_token(const etl::istring& s, etl::istring::const_pointer delimiters, const string_view& last_view)
+  {
+    return etl::private_string_utilities::get_token(s, delimiters, last_view);
+  }
 }
 
 #endif

test/test_string_utilities_char.cpp

@@ -31,6 +31,7 @@ SOFTWARE.
 #include "etl/cstring.h"
 #include "etl/string_view.h"
 #include "etl/string_utilities.h"
+#include "etl/vector.h"
 
 #undef STR
 #define STR(x) x
@@ -45,6 +46,7 @@ namespace
     typedef etl::istring             IString;
     typedef etl::string_view         StringView;
     typedef etl::istring::value_type Char;
+    typedef etl::vector<String, 10>  Vector;
 
     //*************************************************************************
     TEST(test_trim_whitespace_left_empty)
@@ -1186,5 +1188,28 @@ namespace
 
       CHECK(expected == text);
     }
+
+    //*************************************************************************
+    TEST(test_get_token_pointer_delimiters)
+    {
+      String text(STR("  The cat.sat,  on;the:mat  "));
+      Vector tokens;
+
+      StringView token;
+      
+      do
+      { 
+        token = etl::get_token(text, STR(" .,;:"), token);
+
+        if (!token.empty())
+        {
+          String s(token.begin(), token.end());
+          tokens.push_back(s);
+        }
+
+      } while (!token.empty());
+
+      CHECK_EQUAL(6U, tokens.size());
+    }
   };
 }