Package org.apache.sysds.runtime.transform.tokenize.applier

Class TokenizerApplier

java.lang.Object

org.apache.sysds.runtime.transform.tokenize.applier.TokenizerApplier

All Implemented Interfaces:

Serializable

Direct Known Subclasses:

TokenizerApplierCount, TokenizerApplierHash, TokenizerApplierPosition

public abstract class TokenizerApplier
extends Object
implements Serializable

See Also:

Serialized Form

Field Summary

Fields

Modifier and Type	Field	Description
static String	PADDING_STRING

Constructor Summary

Constructors

Constructor	Description
TokenizerApplier(int numIdCols, int maxTokens, boolean wideFormat, boolean applyPadding)

Method Summary

Modifier and Type	Method	Description
void	allocateInternalMeta(int numDocuments)
int	applyInternalRepresentation(DocumentRepresentation[] internalRepresentation, FrameBlock out)
void	build(DocumentRepresentation[] internalRepresentation, int inputRowStart, int blk)
List<DependencyTask<?>>	getApplyTasks(DocumentRepresentation[] internalRepresentation, FrameBlock out)
List<DependencyTask<?>>	getBuildTasks(DocumentRepresentation[] internalRepresentation)
int	getMaxNumRows(int inRows)
int	getMaxTokens()
long	getNumCols()
abstract int	getNumRows(DocumentRepresentation[] internalRepresentation)
<T,E> int	getOutputRow(int inputRowStart, List<Map<T,E>> internalData)
int	getOutputRow(int inputRowStart, DocumentRepresentation[] internalData)
abstract Types.ValueType[]	getOutSchema()
boolean	hasPadding()
boolean	isWideFormat()

Methods inherited from class java.lang.Object

equals, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

PADDING_STRING

public static final String PADDING_STRING

See Also:

Constant Field Values

Constructor Detail

TokenizerApplier

public TokenizerApplier(int numIdCols,
                        int maxTokens,
                        boolean wideFormat,
                        boolean applyPadding)

Method Detail

applyInternalRepresentation

public int applyInternalRepresentation(DocumentRepresentation[] internalRepresentation,
                                       FrameBlock out)

build

public void build(DocumentRepresentation[] internalRepresentation,
                  int inputRowStart,
                  int blk)

getBuildTasks

public List<DependencyTask<?>> getBuildTasks(DocumentRepresentation[] internalRepresentation)

getApplyTasks

public List<DependencyTask<?>> getApplyTasks(DocumentRepresentation[] internalRepresentation,
                                             FrameBlock out)

getOutSchema

public abstract Types.ValueType[] getOutSchema()

hasPadding

public boolean hasPadding()

getMaxTokens

public int getMaxTokens()

getMaxNumRows

public int getMaxNumRows(int inRows)

getNumRows

public abstract int getNumRows(DocumentRepresentation[] internalRepresentation)

getOutputRow

public <T,E> int getOutputRow(int inputRowStart,
                                    List<Map<T,E>> internalData)

getOutputRow

public int getOutputRow(int inputRowStart,
                        DocumentRepresentation[] internalData)

getNumCols

public long getNumCols()

isWideFormat

public boolean isWideFormat()

allocateInternalMeta

public void allocateInternalMeta(int numDocuments)