Package org.apache.sysds.runtime.controlprogram

Class ParForProgramBlock

java.lang.Object

org.apache.sysds.runtime.controlprogram.ProgramBlock

org.apache.sysds.runtime.controlprogram.ForProgramBlock

org.apache.sysds.runtime.controlprogram.ParForProgramBlock

All Implemented Interfaces:

ParseInfo

public class ParForProgramBlock
extends ForProgramBlock

The ParForProgramBlock has the same execution semantics as a ForProgamBlock but executes the independent iterations in parallel. See ParForStatementBlock for the loop dependency analysis. At runtime level, iterations are guaranteed to be completely independent. NEW FUNCTIONALITIES TODO: reduction variables (operations: +=, -=, /=, *=, min, max) TODO: papply(A,1:2,FUN) language construct (compiled to ParFOR) via DML function repository => modules OK, but second-order functions required

Nested Class Summary

Nested Classes

Modifier and Type	Class	Description
static class	ParForProgramBlock.PartitionFormat	Convenience class to package PDataPartitionFormat and its parameters.
static class	ParForProgramBlock.PDataPartitioner
static class	ParForProgramBlock.PDataPartitionFormat
static class	ParForProgramBlock.PExecMode
static class	ParForProgramBlock.POptMode
static class	ParForProgramBlock.PResultMerge
static class	ParForProgramBlock.PTaskPartitioner

Field Summary

Fields

Modifier and Type	Field	Description
static boolean	ALLOW_BROADCAST_INPUTS
static boolean	ALLOW_NESTED_PARALLELISM
static boolean	ALLOW_REUSE_PARTITION_VARS
static boolean	CONVERT_NESTED_REMOTE_PARFOR
static boolean	COPY_EVAL_FUNCTIONS
static boolean	CREATE_UNSCOPED_RESULTVARS
static boolean	FORCE_CP_ON_REMOTE_SPARK
static boolean	LIVEVAR_AWARE_EXPORT
static int	MAX_RETRYS_ON_ERROR
static boolean	OPTIMIZE
static String	PARFOR_COUNTER_GROUP_NAME
static String	PARFOR_DATAPARTITIONS_FNAME
static String	PARFOR_FNAME_PREFIX
static String	PARFOR_MR_RESULT_TMP_FNAME
static String	PARFOR_MR_RESULTMERGE_FNAME
static String	PARFOR_MR_TASKS_TMP_FNAME
static boolean	RESET_RECOMPILATION_FLAGs
static boolean	USE_PARALLEL_RESULT_MERGE
static boolean	USE_PARALLEL_RESULT_MERGE_REMOTE
static boolean	USE_PB_CACHE
static boolean	USE_RANGE_TASKS_IF_USEFUL
static boolean	USE_STREAMING_TASK_CREATION
static int	WRITE_REPLICATION_FACTOR

Fields inherited from class org.apache.sysds.runtime.controlprogram.ProgramBlock

_beginColumn, _beginLine, _endColumn, _endLine, _filename, _text, CHECK_MATRIX_PROPERTIES, PRED_VAR

Constructor Summary

Constructors

Constructor	Description
ParForProgramBlock(int ID, Program prog, String iterPredVar, HashMap<String,String> params, ArrayList<ParForStatementBlock.ResultVar> resultVars)	ParForProgramBlock constructor.
ParForProgramBlock(Program prog, String iterPredVar, HashMap<String,String> params, ArrayList<ParForStatementBlock.ResultVar> resultVars)

Method Summary

Modifier and Type	Method	Description
static ResultMerge<?>	createResultMerge(ParForProgramBlock.PResultMerge prm, CacheableData<?> out, CacheableData<?>[] in, String fname, boolean accum, int numThreads, ExecutionContext ec)
void	disableJVMReuse()
void	disableOptimization()
void	enableColocatedPartitionedMatrix(String varname)
void	execute(ExecutionContext ec)	Executes this program block (incl recompilation if required).
int	getDegreeOfParallelism()
ParForProgramBlock.PExecMode	getExecMode()
long	getID()
long	getNumIterations()
ParForProgramBlock.POptMode	getOptimizationMode()
String	getParForParam(String key)
HashMap<String,String>	getParForParams()
ArrayList<ParForStatementBlock.ResultVar>	getResultVariables()
Collection<String>	getSparkRepartitionVariables()
boolean	hasFunctions()
boolean	isDegreeOfParallelismFixed()
String	printBlockErrorLocation()
static void	resetWorkerIDs()
void	setCPCaching(boolean flag)
void	setDataPartitioner(ParForProgramBlock.PDataPartitioner partitioner)
void	setDegreeOfParallelism(int k)
void	setDegreeOfParallelismFixed(boolean flag)
void	setExecMode(ParForProgramBlock.PExecMode mode)
void	setExportReplicationFactor(int rep)
void	setOptimizationMode(ParForProgramBlock.POptMode mode)
void	setPartitionReplicationFactor(int rep)
void	setRecompileMemoryBudget(double localMem)
void	setResultMerge(ParForProgramBlock.PResultMerge merge)
void	setRuntimePiggybacking(boolean flag)
void	setSparkEagerCacheVariables(Collection<String> vars)
void	setSparkRepartitionVariables(Collection<String> vars)
void	setTaskPartitioner(ParForProgramBlock.PTaskPartitioner partitioner)
void	setTaskSize(long tasksize)
void	setTransposeSparseColumnVector(boolean flag)

Methods inherited from class org.apache.sysds.runtime.controlprogram.ForProgramBlock

addProgramBlock, getChildBlocks, getFromInstructions, getIncrementInstructions, getIterVar, getToInstructions, isNested, setChildBlocks, setFromInstructions, setIncrementInstructions, setToInstructions

Methods inherited from class org.apache.sysds.runtime.controlprogram.ProgramBlock

executePredicate, getBeginColumn, getBeginLine, getEndColumn, getEndLine, getExitInstruction, getFilename, getProgram, getStatementBlock, getText, getThreadID, hasThreadID, isThreadID, setBeginColumn, setBeginLine, setEndColumn, setEndLine, setExitInstruction, setFilename, setParseInfo, setStatementBlock, setText, setThreadID

Methods inherited from class java.lang.Object

equals, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

OPTIMIZE

public static final boolean OPTIMIZE

See Also:

Constant Field Values

USE_PB_CACHE

public static final boolean USE_PB_CACHE

See Also:

Constant Field Values

USE_RANGE_TASKS_IF_USEFUL

public static final boolean USE_RANGE_TASKS_IF_USEFUL

See Also:

Constant Field Values

USE_STREAMING_TASK_CREATION

public static final boolean USE_STREAMING_TASK_CREATION

See Also:

Constant Field Values

ALLOW_NESTED_PARALLELISM

public static final boolean ALLOW_NESTED_PARALLELISM

See Also:

Constant Field Values

CONVERT_NESTED_REMOTE_PARFOR

public static final boolean CONVERT_NESTED_REMOTE_PARFOR

See Also:

Constant Field Values

USE_PARALLEL_RESULT_MERGE

public static final boolean USE_PARALLEL_RESULT_MERGE

See Also:

Constant Field Values

USE_PARALLEL_RESULT_MERGE_REMOTE

public static final boolean USE_PARALLEL_RESULT_MERGE_REMOTE

See Also:

Constant Field Values

CREATE_UNSCOPED_RESULTVARS

public static final boolean CREATE_UNSCOPED_RESULTVARS

See Also:

Constant Field Values

ALLOW_REUSE_PARTITION_VARS

public static boolean ALLOW_REUSE_PARTITION_VARS

WRITE_REPLICATION_FACTOR

public static final int WRITE_REPLICATION_FACTOR

See Also:

Constant Field Values

MAX_RETRYS_ON_ERROR

public static int MAX_RETRYS_ON_ERROR

FORCE_CP_ON_REMOTE_SPARK

public static final boolean FORCE_CP_ON_REMOTE_SPARK

See Also:

Constant Field Values

LIVEVAR_AWARE_EXPORT

public static final boolean LIVEVAR_AWARE_EXPORT

See Also:

Constant Field Values

RESET_RECOMPILATION_FLAGs

public static final boolean RESET_RECOMPILATION_FLAGs

See Also:

Constant Field Values

ALLOW_BROADCAST_INPUTS

public static boolean ALLOW_BROADCAST_INPUTS

COPY_EVAL_FUNCTIONS

public static final boolean COPY_EVAL_FUNCTIONS

See Also:

Constant Field Values

PARFOR_FNAME_PREFIX

public static final String PARFOR_FNAME_PREFIX

See Also:

Constant Field Values

PARFOR_MR_TASKS_TMP_FNAME

public static final String PARFOR_MR_TASKS_TMP_FNAME

See Also:

Constant Field Values

PARFOR_MR_RESULT_TMP_FNAME

public static final String PARFOR_MR_RESULT_TMP_FNAME

See Also:

Constant Field Values

PARFOR_MR_RESULTMERGE_FNAME

public static final String PARFOR_MR_RESULTMERGE_FNAME

See Also:

Constant Field Values

PARFOR_DATAPARTITIONS_FNAME

public static final String PARFOR_DATAPARTITIONS_FNAME

See Also:

Constant Field Values

PARFOR_COUNTER_GROUP_NAME

public static final String PARFOR_COUNTER_GROUP_NAME

See Also:

Constant Field Values

Constructor Detail

ParForProgramBlock

public ParForProgramBlock(Program prog,
                          String iterPredVar,
                          HashMap<String,String> params,
                          ArrayList<ParForStatementBlock.ResultVar> resultVars)

ParForProgramBlock

public ParForProgramBlock(int ID,
                          Program prog,
                          String iterPredVar,
                          HashMap<String,String> params,
                          ArrayList<ParForStatementBlock.ResultVar> resultVars)

ParForProgramBlock constructor. It reads the specified parameter settings, where defaults for non-specified parameters have been set in ParForStatementBlock.validate(). Furthermore, it generates the IDs for the ParWorkers.

Parameters:

ID - parfor program block id

prog - runtime program

iterPredVar - ?

params - map of parameters

resultVars - list of result variable names

Method Detail

resetWorkerIDs

public static void resetWorkerIDs()

getID

public long getID()

getExecMode

public ParForProgramBlock.PExecMode getExecMode()

getParForParams

public HashMap<String,String> getParForParams()

getParForParam

public String getParForParam(String key)

getResultVariables

public ArrayList<ParForStatementBlock.ResultVar> getResultVariables()

disableOptimization

public void disableOptimization()

getOptimizationMode

public ParForProgramBlock.POptMode getOptimizationMode()

setOptimizationMode

public void setOptimizationMode(ParForProgramBlock.POptMode mode)

getDegreeOfParallelism

public int getDegreeOfParallelism()

setDegreeOfParallelism

public void setDegreeOfParallelism(int k)

isDegreeOfParallelismFixed

public boolean isDegreeOfParallelismFixed()

setDegreeOfParallelismFixed

public void setDegreeOfParallelismFixed(boolean flag)

setCPCaching

public void setCPCaching(boolean flag)

setRuntimePiggybacking

public void setRuntimePiggybacking(boolean flag)

setExecMode

public void setExecMode(ParForProgramBlock.PExecMode mode)

setTaskPartitioner

public void setTaskPartitioner(ParForProgramBlock.PTaskPartitioner partitioner)

setTaskSize

public void setTaskSize(long tasksize)

setDataPartitioner

public void setDataPartitioner(ParForProgramBlock.PDataPartitioner partitioner)

enableColocatedPartitionedMatrix

public void enableColocatedPartitionedMatrix(String varname)

setTransposeSparseColumnVector

public void setTransposeSparseColumnVector(boolean flag)

setPartitionReplicationFactor

public void setPartitionReplicationFactor(int rep)

setExportReplicationFactor

public void setExportReplicationFactor(int rep)

disableJVMReuse

public void disableJVMReuse()

setResultMerge

public void setResultMerge(ParForProgramBlock.PResultMerge merge)

setRecompileMemoryBudget

public void setRecompileMemoryBudget(double localMem)

setSparkRepartitionVariables

public void setSparkRepartitionVariables(Collection<String> vars)

getSparkRepartitionVariables

public Collection<String> getSparkRepartitionVariables()

setSparkEagerCacheVariables

public void setSparkEagerCacheVariables(Collection<String> vars)

getNumIterations

public long getNumIterations()

hasFunctions

public boolean hasFunctions()

execute

public void execute(ExecutionContext ec)

Description copied from class: ProgramBlock

Executes this program block (incl recompilation if required).

Overrides:

execute in class ForProgramBlock

Parameters:

ec - execution context

createResultMerge

public static ResultMerge<?> createResultMerge(ParForProgramBlock.PResultMerge prm,
                                               CacheableData<?> out,
                                               CacheableData<?>[] in,
                                               String fname,
                                               boolean accum,
                                               int numThreads,
                                               ExecutionContext ec)

printBlockErrorLocation

public String printBlockErrorLocation()

Overrides:

printBlockErrorLocation in class ForProgramBlock