public class GermanTokenizerPostProcessor extends CoreLabelProcessor
Modifier and Type | Field and Description |
---|---|
java.util.function.Function<CoreLabel,java.lang.Boolean> |
afterIsEmpty
Check that after() is not null and the empty string
|
java.util.HashSet<java.lang.String> |
germanAbbreviations |
java.util.HashSet<java.lang.String> |
ordinalPredictingWords |
Constructor and Description |
---|
GermanTokenizerPostProcessor() |
Modifier and Type | Method and Description |
---|---|
static void |
mergeTokens(CoreLabel token,
CoreLabel nextToken)
merge the contents of two tokens
|
java.util.List<CoreLabel> |
process(java.util.List<CoreLabel> tokens)
Alter the tokenization of list of tokens (e.g.
|
java.util.List<CoreLabel> |
restore(java.util.List<CoreLabel> originalTokens,
java.util.List<CoreLabel> processedTokens)
Undo the tokenization changes of process, maintaining any tagging
|
public java.util.function.Function<CoreLabel,java.lang.Boolean> afterIsEmpty
public java.util.HashSet<java.lang.String> ordinalPredictingWords
public java.util.HashSet<java.lang.String> germanAbbreviations
public static void mergeTokens(CoreLabel token, CoreLabel nextToken)
public java.util.List<CoreLabel> process(java.util.List<CoreLabel> tokens)
CoreLabelProcessor
process
in class CoreLabelProcessor
public java.util.List<CoreLabel> restore(java.util.List<CoreLabel> originalTokens, java.util.List<CoreLabel> processedTokens)
CoreLabelProcessor
restore
in class CoreLabelProcessor