public class TokenizerPreNgram extends Object implements TokenizerPre
Constructor and Description |
---|
TokenizerPreNgram(List<Integer> idCols,
int tokenizeCol,
org.apache.wink.json4j.JSONObject params) |
Modifier and Type | Method and Description |
---|---|
List<org.apache.sysds.runtime.transform.tokenize.Tokenizer.DocumentToTokens> |
tokenizePre(FrameBlock in) |
List<org.apache.sysds.runtime.transform.tokenize.Tokenizer.Token> |
wordTokenListToNgrams(List<org.apache.sysds.runtime.transform.tokenize.Tokenizer.Token> wordTokens) |
List<org.apache.sysds.runtime.transform.tokenize.Tokenizer.Token> |
wordTokenToNgrams(org.apache.sysds.runtime.transform.tokenize.Tokenizer.Token wordTokens) |
public List<org.apache.sysds.runtime.transform.tokenize.Tokenizer.Token> wordTokenToNgrams(org.apache.sysds.runtime.transform.tokenize.Tokenizer.Token wordTokens)
public List<org.apache.sysds.runtime.transform.tokenize.Tokenizer.Token> wordTokenListToNgrams(List<org.apache.sysds.runtime.transform.tokenize.Tokenizer.Token> wordTokens)
public List<org.apache.sysds.runtime.transform.tokenize.Tokenizer.DocumentToTokens> tokenizePre(FrameBlock in)
tokenizePre
in interface TokenizerPre
Copyright © 2021 The Apache Software Foundation. All rights reserved.