Improved tokenizeString

diff --git a/examples/uai2010-aie-solver.cpp b/examples/uai2010-aie-solver.cpp
index 0fbd17c..7547335 100644 (file)
--- a/examples/uai2010-aie-solver.cpp
+++ b/examples/uai2010-aie-solver.cpp
@@ -276,8 +276,7 @@ int main( int argc, char *argv[] ) {
         }
 
         // build output file name
-        vector<string> pathComponents;
-        tokenizeString( string(argv[1]), pathComponents, "/" );
+        vector<string> pathComponents = tokenizeString( string(argv[1]), true, "/" );
         string outfile = pathComponents.back() + "." + task;
         if( verbose )
             cout << "Output filename:      " << outfile << endl;

diff --git a/include/dai/util.h b/include/dai/util.h
index 783a990..b1378be 100644 (file)
--- a/include/dai/util.h
+++ b/include/dai/util.h
@@ -209,6 +209,17 @@ std::vector<T> concat( const std::vector<T>& u, const std::vector<T>& v ) {
 }
 
 /// Split a string into tokens delimited by one of the characters in \a delim
+/** \param s the string to be split into tokens
+ *  \param singleDelim if \c true, any single delimiter forms a boundary between two tokens; 
+ *         if \c false, a maximal group of consecutive delimiters forms a boundary between two tokens
+ *  \param delim delimiter characters
+ */
+std::vector<std::string> tokenizeString( const std::string& s, bool singleDelim, const std::string& delim="\t\n" );
+
+
+/// Split a string into tokens delimited by one of the characters in \a delim
+/** \deprecated Please use dai::tokenizeString( const std::string&, bool, const std::string& ) instead
+ */
 void tokenizeString( const std::string& s, std::vector<std::string>& outTokens, const std::string& delim="\t\n" );
 
 

diff --git a/src/emalg.cpp b/src/emalg.cpp
index 73ff1d2..9e79ce1 100644 (file)
--- a/src/emalg.cpp
+++ b/src/emalg.cpp
@@ -129,8 +129,7 @@ SharedParameters::SharedParameters( std::istream &is, const FactorGraph &fg )
         while( line.size() == 0 && getline(is, line) )
             ;
 
-        std::vector<std::string> fields;
-        tokenizeString(line, fields, " \t");
+        std::vector<std::string> fields = tokenizeString( line, true, " \t" );
 
         // Lookup the factor in the factorgraph
         if( fields.size() < 1 )

diff --git a/src/evidence.cpp b/src/evidence.cpp
index 0dfe2e9..218993b 100644 (file)
--- a/src/evidence.cpp
+++ b/src/evidence.cpp
@@ -40,7 +40,7 @@ void Evidence::addEvidenceTabFile( std::istream &is, std::map<std::string, Var>
 
     // Parse header
     std::vector<std::string> header_fields;
-    tokenizeString( line, header_fields );
+    header_fields = tokenizeString( line, true );
     std::vector<std::string>::const_iterator p_field = header_fields.begin();
     if( p_field == header_fields.end() )
         DAI_THROWE(INVALID_EVIDENCE_FILE,"Empty header line");
@@ -62,7 +62,7 @@ void Evidence::addEvidenceTabFile( std::istream &is, std::map<std::string, Var>
         line_number++;
 
         std::vector<std::string> fields;
-        tokenizeString( line, fields );
+        fields = tokenizeString( line, true );
         if( fields.size() != vars.size() )
             DAI_THROWE(INVALID_EVIDENCE_FILE,"Invalid number of fields in line " + boost::lexical_cast<std::string>(line_number));
 

diff --git a/src/util.cpp b/src/util.cpp
index 5a79380..f4b7adb 100644 (file)
--- a/src/util.cpp
+++ b/src/util.cpp
@@ -102,6 +102,31 @@ int rnd_int( int min, int max ) {
     return (int)floor(_uni_rnd() * (max + 1 - min) + min);
 }
 
+std::vector<std::string> tokenizeString( const std::string& s, bool singleDelim, const std::string& delim ) {
+    using namespace std;
+    vector<string> tokens;
+
+    string::size_type start = 0;
+    while( start < s.size() ) {
+        string::size_type end = s.find_first_of( delim, start );
+        if( end == string::npos )
+            end = s.size();
+
+        if( end == start && !singleDelim ) {
+            // skip to next non-delimiter
+            start = s.find_first_not_of( delim, start );
+            if( start == string::npos )
+                start = s.size();
+        } else { // we found a token
+            tokens.push_back( s.substr(start, end - start) );
+            start = end + 1;
+        }
+    }
+
+    return tokens;
+}
+
+
 void tokenizeString(const std::string& s, std::vector<std::string>& outTokens, const std::string& delim) {
     size_t start = 0;
     while (start < s.size()) {
@@ -113,4 +138,5 @@ void tokenizeString(const std::string& s, std::vector<std::string>& outTokens, c
     }
 }
 
+
 } // end of namespace dai

diff --git a/tests/unit/util_test.cpp b/tests/unit/util_test.cpp
index 6648b15..eb19529 100644 (file)
--- a/tests/unit/util_test.cpp
+++ b/tests/unit/util_test.cpp
@@ -150,17 +150,110 @@ BOOST_AUTO_TEST_CASE( stringTest ) {
 
 
 BOOST_AUTO_TEST_CASE( tokenizeStringTest ) {
-    std::string s("Hello\tworld.\nThis is it.");
-    std::vector<std::string> words;
-    tokenizeString( s, words );
-    BOOST_CHECK_EQUAL( words.size(), 3 );
-    BOOST_CHECK_EQUAL( words[0], "Hello" );
-    BOOST_CHECK_EQUAL( words[1], "world." );
-    BOOST_CHECK_EQUAL( words[2], "This is it." );
-    words.clear();
-    tokenizeString( s, words, " " );
-    BOOST_CHECK_EQUAL( words.size(), 3 );
-    BOOST_CHECK_EQUAL( words[0], "Hello\tworld.\nThis" );
-    BOOST_CHECK_EQUAL( words[1], "is" );
-    BOOST_CHECK_EQUAL( words[2], "it." );
+    std::string s;
+    std::vector<std::string> tokens;
+
+    s = "";
+    tokens = tokenizeString( s, true, " \t" );
+    BOOST_CHECK_EQUAL( tokens.size(), 0 );
+
+    s = " ";
+    tokens = tokenizeString( s, true, " \t" );
+    BOOST_CHECK_EQUAL( tokens.size(), 1 );
+    BOOST_CHECK_EQUAL( tokens[0], "" );
+    
+    s = " \t";
+    tokens = tokenizeString( s, true, " \t" );
+    BOOST_CHECK_EQUAL( tokens.size(), 2 );
+    BOOST_CHECK_EQUAL( tokens[0], "" );
+    BOOST_CHECK_EQUAL( tokens[1], "" );
+
+    s = " \tHello";
+    tokens = tokenizeString( s, true, " \t" );
+    BOOST_CHECK_EQUAL( tokens.size(), 3 );
+    BOOST_CHECK_EQUAL( tokens[0], "" );
+    BOOST_CHECK_EQUAL( tokens[1], "" );
+    BOOST_CHECK_EQUAL( tokens[2], "Hello" );
+
+    s = " \tHello\r\n there";
+    tokens = tokenizeString( s, true, " \t" );
+    BOOST_CHECK_EQUAL( tokens.size(), 4 );
+    BOOST_CHECK_EQUAL( tokens[0], "" );
+    BOOST_CHECK_EQUAL( tokens[1], "" );
+    BOOST_CHECK_EQUAL( tokens[2], "Hello\r\n" );
+    BOOST_CHECK_EQUAL( tokens[3], "there" );
+
+    s = " \tHello\r\n there !";
+    tokens = tokenizeString( s, true, " \t" );
+    BOOST_CHECK_EQUAL( tokens.size(), 5 );
+    BOOST_CHECK_EQUAL( tokens[0], "" );
+    BOOST_CHECK_EQUAL( tokens[1], "" );
+    BOOST_CHECK_EQUAL( tokens[2], "Hello\r\n" );
+    BOOST_CHECK_EQUAL( tokens[3], "there" );
+    BOOST_CHECK_EQUAL( tokens[4], "!" );
+
+    s = " \tHello\r\n there !\r";
+    tokens = tokenizeString( s, true, " \t" );
+    BOOST_CHECK_EQUAL( tokens.size(), 5 );
+    BOOST_CHECK_EQUAL( tokens[0], "" );
+    BOOST_CHECK_EQUAL( tokens[1], "" );
+    BOOST_CHECK_EQUAL( tokens[2], "Hello\r\n" );
+    BOOST_CHECK_EQUAL( tokens[3], "there" );
+    BOOST_CHECK_EQUAL( tokens[4], "!\r" );
+
+    s = " \tHello\r\n there !\r";
+    tokens = tokenizeString( s, true, " \t\r\n" );
+    BOOST_CHECK_EQUAL( tokens.size(), 7 );
+    BOOST_CHECK_EQUAL( tokens[0], "" );
+    BOOST_CHECK_EQUAL( tokens[1], "" );
+    BOOST_CHECK_EQUAL( tokens[2], "Hello" );
+    BOOST_CHECK_EQUAL( tokens[3], "" );
+    BOOST_CHECK_EQUAL( tokens[4], "" );
+    BOOST_CHECK_EQUAL( tokens[5], "there" );
+    BOOST_CHECK_EQUAL( tokens[6], "!" );
+
+
+    s = "";
+    tokens = tokenizeString( s, false, " \t" );
+    BOOST_CHECK_EQUAL( tokens.size(), 0 );
+
+    s = " ";
+    tokens = tokenizeString( s, false, " \t" );
+    BOOST_CHECK_EQUAL( tokens.size(), 0 );
+    
+    s = " \t";
+    tokens = tokenizeString( s, false, " \t" );
+    BOOST_CHECK_EQUAL( tokens.size(), 0 );
+
+    s = " \tHello";
+    tokens = tokenizeString( s, false, " \t" );
+    BOOST_CHECK_EQUAL( tokens.size(), 1 );
+    BOOST_CHECK_EQUAL( tokens[0], "Hello" );
+
+    s = " \tHello\r\n there";
+    tokens = tokenizeString( s, false, " \t" );
+    BOOST_CHECK_EQUAL( tokens.size(), 2 );
+    BOOST_CHECK_EQUAL( tokens[0], "Hello\r\n" );
+    BOOST_CHECK_EQUAL( tokens[1], "there" );
+
+    s = " \tHello\r\n there !";
+    tokens = tokenizeString( s, false, " \t" );
+    BOOST_CHECK_EQUAL( tokens.size(), 3 );
+    BOOST_CHECK_EQUAL( tokens[0], "Hello\r\n" );
+    BOOST_CHECK_EQUAL( tokens[1], "there" );
+    BOOST_CHECK_EQUAL( tokens[2], "!" );
+
+    s = " \tHello\r\n there !\r";
+    tokens = tokenizeString( s, false, " \t" );
+    BOOST_CHECK_EQUAL( tokens.size(), 3 );
+    BOOST_CHECK_EQUAL( tokens[0], "Hello\r\n" );
+    BOOST_CHECK_EQUAL( tokens[1], "there" );
+    BOOST_CHECK_EQUAL( tokens[2], "!\r" );
+
+    s = " \tHello\r\n there !\r";
+    tokens = tokenizeString( s, false, " \t\r\n" );
+    BOOST_CHECK_EQUAL( tokens.size(), 3 );
+    BOOST_CHECK_EQUAL( tokens[0], "Hello" );
+    BOOST_CHECK_EQUAL( tokens[1], "there" );
+    BOOST_CHECK_EQUAL( tokens[2], "!" );
 }