org.apache.sysds.runtime.controlprogram.paramserv

Class ParamservUtils

java.lang.Object

org.apache.sysds.runtime.controlprogram.paramserv.ParamservUtils

public class ParamservUtils
extends Object

Field Summary

Fields

Modifier and Type	Field and Description
static String	PS_FUNC_PREFIX
static long	SEED

Constructor Summary

Constructors

Constructor and Description
ParamservUtils()

Method Summary

Modifier and Type	Method and Description
static ListObject	accrueGradients(ListObject accGradients, ListObject gradients, boolean cleanup) Accumulate the given gradients into the accrued gradients
static ListObject	accrueGradients(ListObject accGradients, ListObject gradients, boolean par, boolean cleanup) Accumulate the given gradients into the accrued gradients
static org.apache.spark.api.java.JavaPairRDD<Long,scala.Tuple2<MatrixBlock,MatrixBlock>>	assembleTrainingData(org.apache.spark.api.java.JavaPairRDD<MatrixIndexes,MatrixBlock> featuresRDD, org.apache.spark.api.java.JavaPairRDD<MatrixIndexes,MatrixBlock> labelsRDD) Assemble the matrix of features and labels according to the rowID
static MatrixBlock	cbindMatrix(MatrixBlock left, MatrixBlock right)
static void	cleanupData(ExecutionContext ec, Data data)
static void	cleanupData(ExecutionContext ec, String varName)
static void	cleanupListObject(ExecutionContext ec, ListObject lo)
static void	cleanupListObject(ExecutionContext ec, ListObject lo, boolean[] status)
static void	cleanupListObject(ExecutionContext ec, String lName) Clean up the list object according to its own data status
static void	cleanupListObject(ExecutionContext ec, String lName, boolean[] status) Clean up the list object according to the given array of data status (i.e., false => not be removed)
static void	cleanupListObject(ListObject lo)
static List<ExecutionContext>	copyExecutionContext(ExecutionContext ec, int num)
static ListObject	copyList(ListObject lo, boolean cleanup) Deep copy the list object
static ExecutionContext	createExecutionContext(ExecutionContext ec, LocalVariableMap varsMap, String updFunc, String aggFunc, int k)
static MatrixObject	createShallowCopy(MatrixObject mo)
static org.apache.spark.api.java.JavaPairRDD<Integer,scala.Tuple2<MatrixBlock,MatrixBlock>>	doPartitionOnSpark(SparkExecutionContext sec, MatrixObject features, MatrixObject labels, Statement.PSScheme scheme, int workerNum)
static MatrixBlock	generatePermutation(int numEntries, long seed) Generate the permutation
static MatrixObject	newMatrixObject(MatrixBlock mb)
static MatrixObject	newMatrixObject(MatrixBlock mb, boolean cleanup)
static void	recompileProgramBlocks(int k, List<ProgramBlock> pbs)
static MatrixObject	sliceMatrix(MatrixObject mo, long rl, long rh) Slice the matrix
static MatrixBlock	sliceMatrixBlock(MatrixBlock mb, long rl, long rh) Slice the matrix block and return a matrix block (used in spark)

Methods inherited from class java.lang.Object

equals, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

PS_FUNC_PREFIX

public static final String PS_FUNC_PREFIX

See Also:

Constant Field Values

SEED

public static long SEED

Constructor Detail

ParamservUtils

public ParamservUtils()

Method Detail

copyList

public static ListObject copyList(ListObject lo,
                                  boolean cleanup)

Deep copy the list object

Parameters:

lo - list object

cleanup - clean up the given list object

Returns:

a new copied list object

cleanupListObject

public static void cleanupListObject(ExecutionContext ec,
                                     String lName)

Clean up the list object according to its own data status

Parameters:

ec - execution context

lName - list var name

cleanupListObject

public static void cleanupListObject(ExecutionContext ec,
                                     String lName,
                                     boolean[] status)

Clean up the list object according to the given array of data status (i.e., false => not be removed)

Parameters:

ec - execution context

lName - list var name

status - data status

cleanupListObject

public static void cleanupListObject(ExecutionContext ec,
                                     ListObject lo)

cleanupListObject

public static void cleanupListObject(ExecutionContext ec,
                                     ListObject lo,
                                     boolean[] status)

cleanupData

public static void cleanupData(ExecutionContext ec,
                               Data data)

cleanupData

public static void cleanupData(ExecutionContext ec,
                               String varName)

cleanupListObject

public static void cleanupListObject(ListObject lo)

newMatrixObject

public static MatrixObject newMatrixObject(MatrixBlock mb)

newMatrixObject

public static MatrixObject newMatrixObject(MatrixBlock mb,
                                           boolean cleanup)

createShallowCopy

public static MatrixObject createShallowCopy(MatrixObject mo)

sliceMatrix

public static MatrixObject sliceMatrix(MatrixObject mo,
                                       long rl,
                                       long rh)

Slice the matrix

Parameters:

mo - input matrix

rl - low boundary

rh - high boundary

Returns:

new sliced matrix

sliceMatrixBlock

public static MatrixBlock sliceMatrixBlock(MatrixBlock mb,
                                           long rl,
                                           long rh)

Slice the matrix block and return a matrix block (used in spark)

Parameters:

mb - input matrix

rl - low boundary

rh - high boundary

Returns:

new sliced matrix block

generatePermutation

public static MatrixBlock generatePermutation(int numEntries,
                                              long seed)

Generate the permutation

Parameters:

numEntries - permutation size

seed - seed used to generate random number

Returns:

permutation matrix

createExecutionContext

public static ExecutionContext createExecutionContext(ExecutionContext ec,
                                                      LocalVariableMap varsMap,
                                                      String updFunc,
                                                      String aggFunc,
                                                      int k)

copyExecutionContext

public static List<ExecutionContext> copyExecutionContext(ExecutionContext ec,
                                                          int num)

recompileProgramBlocks

public static void recompileProgramBlocks(int k,
                                          List<ProgramBlock> pbs)

cbindMatrix

public static MatrixBlock cbindMatrix(MatrixBlock left,
                                      MatrixBlock right)

assembleTrainingData

public static org.apache.spark.api.java.JavaPairRDD<Long,scala.Tuple2<MatrixBlock,MatrixBlock>> assembleTrainingData(org.apache.spark.api.java.JavaPairRDD<MatrixIndexes,MatrixBlock> featuresRDD,
                                                                                                                     org.apache.spark.api.java.JavaPairRDD<MatrixIndexes,MatrixBlock> labelsRDD)

Assemble the matrix of features and labels according to the rowID

Parameters:

featuresRDD - indexed features matrix block

labelsRDD - indexed labels matrix block

Returns:

Assembled rdd with rowID as key while matrix of features and labels as value (rowID -> features, labels)

doPartitionOnSpark

public static org.apache.spark.api.java.JavaPairRDD<Integer,scala.Tuple2<MatrixBlock,MatrixBlock>> doPartitionOnSpark(SparkExecutionContext sec,
                                                                                                                      MatrixObject features,
                                                                                                                      MatrixObject labels,
                                                                                                                      Statement.PSScheme scheme,
                                                                                                                      int workerNum)

accrueGradients

public static ListObject accrueGradients(ListObject accGradients,
                                         ListObject gradients,
                                         boolean cleanup)

Accumulate the given gradients into the accrued gradients

Parameters:

accGradients - accrued gradients list object

gradients - given gradients list object

cleanup - clean up the given gradients list object

Returns:

new accrued gradients list object

accrueGradients

public static ListObject accrueGradients(ListObject accGradients,
                                         ListObject gradients,
                                         boolean par,
                                         boolean cleanup)

Accumulate the given gradients into the accrued gradients

Parameters:

accGradients - accrued gradients list object

gradients - given gradients list object

par - parallel execution

cleanup - clean up the given gradients list object

Returns:

new accrued gradients list object