7 #ifndef _MIMETIC_TOKENIZER_H_
8 #define _MIMETIC_TOKENIZER_H_
18 template<
typename value_type>
19 struct IsDelim:
public std::unary_function<value_type,bool>
21 bool operator()(
const value_type& val)
const
23 return m_delims.count(val) != 0;
25 template<
typename Container>
26 void setDelimList(
const Container& cont)
28 typename Container::const_iterator bit, eit;
29 bit = cont.begin(), eit = cont.end();
30 for(; bit != eit; ++bit)
31 m_delims.insert(*bit);
33 template<
typename Iterator>
34 void setDelimList(Iterator bit, Iterator eit)
36 for(; bit != eit; ++bit)
37 m_delims.insert(*bit);
39 void addDelim(
const value_type& value)
41 m_delims.insert(value);
43 void removeDelim(
const value_type& value)
45 m_delims.erase(value);
48 std::set<value_type> m_delims;
52 struct IsDelim<char>:
public std::unary_function<char, bool>
54 void setDelimList(
const std::string& delims)
56 setDelimList(delims.begin(), delims.end());
58 template<
typename Iterator>
59 void setDelimList(Iterator bit, Iterator eit)
61 memset(&m_lookup, 0,
sizeof(m_lookup));
62 for(; bit != eit; ++bit)
63 m_lookup[(
int)*bit] = 1;
65 bool operator()(
unsigned char val)
const
67 return m_lookup[val] != 0;
75 template<
class Iterator,
typename value_type>
80 : m_bit(bit), m_eit(eit), m_tok_eit(bit)
83 void setSource(Iterator bit, Iterator eit)
89 template<
typename DelimCont>
90 void setDelimList(
const DelimCont& cont)
92 m_delimPred.setDelimList(cont);
95 void setDelimList(It bit, It eit)
97 m_delimPred.setDelimList(bit, eit);
99 template<
typename DestCont>
100 bool next(DestCont& dst)
102 dst.erase(dst.begin(), dst.end());
103 if(m_tok_eit == m_eit)
105 m_tok_eit = std::find_if(m_bit, m_eit, m_delimPred);
107 if(m_tok_eit != m_eit)
108 m_matched = *m_tok_eit;
109 std::copy(m_bit, m_tok_eit, std::back_inserter<DestCont>(dst));
110 m_bit = (m_tok_eit != m_eit && ++m_tok_eit != m_eit ? m_tok_eit :m_eit);
113 const value_type& matched()
const
117 void addDelim(
const value_type& value)
119 m_delimPred.addDelim(value);
121 void removeDelim(
const value_type& value)
123 m_delimPred.removeDelim(value);
126 Iterator m_bit, m_eit, m_tok_eit;
127 IsDelim<value_type> m_delimPred;
128 value_type m_matched;
133 template<
typename Container>
136 typedef typename Container::value_type value_type;
137 typedef typename Container::iterator iterator;
138 typedef typename Container::const_iterator const_iterator;
146 template<
typename DelimCont>
147 ContTokenizer(
const Container* cont,
const DelimCont& delims)
150 this->setDelimList(delims);
152 void setSource(
const Container* cont)
Iterator tokenizer template class.
Definition: tokenizer.h:76
ContTokenizer< std::string > StringTokenizer
std::string tokenizer
Definition: tokenizer.h:162
char container tokenizer template class
Definition: tokenizer.h:134