81 #ifndef __vtkTokenizer_h
82 #define __vtkTokenizer_h
99 typedef vtkstd::pair<vtkUnicodeString::value_type, vtkUnicodeString::value_type>
DelimiterRange;
108 static const DelimiterRanges Punctuation();
111 static const DelimiterRanges Whitespace();
115 static const DelimiterRanges Logosyllabic();
124 void AddDroppedDelimiters(
const DelimiterRanges& ranges);
133 void AddKeptDelimiters(
const DelimiterRanges& ranges);
142 void DropPunctuation();
143 void DropWhitespace();
144 void KeepPunctuation();
145 void KeepWhitespace();
146 void KeepLogosyllabic();
151 void ClearDroppedDelimiters();
154 void ClearKeptDelimiters();
174 Internals*
const Implementation;
178 #endif // __vtkTokenizer_h
void PrintSelf(ostream &os, vtkIndent indent)
static vtkTableAlgorithm * New()
vtkUnicodeStringValueType value_type
a simple class to control print indentation
vtkstd::pair< vtkUnicodeString::value_type, vtkUnicodeString::value_type > DelimiterRange
virtual int FillInputPortInformation(int port, vtkInformation *info)
vtkstd::vector< DelimiterRange > DelimiterRanges
virtual int RequestData(vtkInformation *request, vtkInformationVector **inputVector, vtkInformationVector *outputVector)
Superclass for algorithms that produce only vtkTables as output.
Converts a document collection into a term collection.