public class TokenizerPostPosition extends Object implements TokenizerPost
Constructor and Description |
---|
TokenizerPostPosition(org.apache.wink.json4j.JSONObject params,
int numIdCols,
int maxTokens,
boolean wideFormat) |
Modifier and Type | Method and Description |
---|---|
void |
appendTokensLong(List<Object> keys,
List<org.apache.sysds.runtime.transform.tokenize.Tokenizer.Token> tokenList,
FrameBlock out) |
void |
appendTokensWide(List<Object> keys,
List<org.apache.sysds.runtime.transform.tokenize.Tokenizer.Token> tokenList,
FrameBlock out) |
long |
getNumCols() |
long |
getNumRows(long inRows) |
Types.ValueType[] |
getOutSchema() |
FrameBlock |
tokenizePost(List<org.apache.sysds.runtime.transform.tokenize.Tokenizer.DocumentToTokens> tl,
FrameBlock out) |
public TokenizerPostPosition(org.apache.wink.json4j.JSONObject params, int numIdCols, int maxTokens, boolean wideFormat)
public FrameBlock tokenizePost(List<org.apache.sysds.runtime.transform.tokenize.Tokenizer.DocumentToTokens> tl, FrameBlock out)
tokenizePost
in interface TokenizerPost
public void appendTokensLong(List<Object> keys, List<org.apache.sysds.runtime.transform.tokenize.Tokenizer.Token> tokenList, FrameBlock out)
public void appendTokensWide(List<Object> keys, List<org.apache.sysds.runtime.transform.tokenize.Tokenizer.Token> tokenList, FrameBlock out)
public Types.ValueType[] getOutSchema()
getOutSchema
in interface TokenizerPost
public long getNumRows(long inRows)
getNumRows
in interface TokenizerPost
public long getNumCols()
getNumCols
in interface TokenizerPost
Copyright © 2021 The Apache Software Foundation. All rights reserved.