static double |
SparkCostUtils.getAggUnaryInstTime(UnarySPInstruction inst,
VarStats input,
VarStats output,
IOCostUtils.IOMetrics executorMetrics) |
|
static double |
SparkCostUtils.getAppendInstTime(AppendSPInstruction inst,
VarStats input1,
VarStats input2,
VarStats output,
IOCostUtils.IOMetrics driverMetrics,
IOCostUtils.IOMetrics executorMetrics) |
|
static double |
CPCostUtils.getBinaryInstTime(BinaryCPInstruction inst,
VarStats input1,
VarStats input2,
VarStats weights,
VarStats output,
IOCostUtils.IOMetrics metrics) |
|
static double |
SparkCostUtils.getBinaryInstTime(SPInstruction inst,
VarStats input1,
VarStats input2,
VarStats output,
IOCostUtils.IOMetrics driverMetrics,
IOCostUtils.IOMetrics executorMetrics) |
|
static double |
CPCostUtils.getBuiltinNaryInstTime(BuiltinNaryCPInstruction inst,
VarStats[] inputs,
VarStats output,
IOCostUtils.IOMetrics metrics) |
|
static double |
SparkCostUtils.getCastInstTime(CastSPInstruction inst,
VarStats input,
VarStats output,
IOCostUtils.IOMetrics executorMetrics) |
|
static double |
SparkCostUtils.getCentralMomentInstTime(CentralMomentSPInstruction inst,
VarStats input,
VarStats weights,
VarStats output,
IOCostUtils.IOMetrics executorMetrics) |
|
static double |
CPCostUtils.getComputationInstTime(ComputationCPInstruction inst,
VarStats input1,
VarStats input2,
VarStats input3,
VarStats input4,
VarStats output,
IOCostUtils.IOMetrics metrics) |
|
static double |
CPCostUtils.getCPUTime(long nflop,
IOCostUtils.IOMetrics driverMetrics,
VarStats output,
VarStats... inputs) |
|
static double |
SparkCostUtils.getCPUTime(long nflop,
int numPartitions,
IOCostUtils.IOMetrics executorMetrics,
RDDStats output,
RDDStats... inputs) |
Computes an estimate for the time needed by the CPU to execute (including memory access)
an instruction by providing number of floating operations.
|
static double |
SparkCostUtils.getCtableInstTime(CtableSPInstruction tableInst,
VarStats input1,
VarStats input2,
VarStats input3,
VarStats output,
IOCostUtils.IOMetrics executorMetrics) |
|
static double |
CPCostUtils.getDataGenCPInstTime(UnaryCPInstruction inst,
VarStats output,
IOCostUtils.IOMetrics metrics) |
|
static double |
IOCostUtils.getFileSystemReadTime(VarStats stats,
IOCostUtils.IOMetrics metrics) |
Estimates the read time for a file on HDFS or S3 by the Control Program
|
static double |
IOCostUtils.getFileSystemWriteTime(VarStats stats,
IOCostUtils.IOMetrics metrics) |
Estimates the time for writing a file to HDFS or S3.
|
static double |
IOCostUtils.getHadoopReadTime(VarStats stats,
IOCostUtils.IOMetrics metrics) |
Estimates the read time for a file on HDFS or S3 by Spark cluster.
|
static double |
IOCostUtils.getHadoopWriteTime(VarStats stats,
IOCostUtils.IOMetrics metrics) |
Estimates the write time for a file on HDFS or S3 by Spark cluster.
|
static double |
SparkCostUtils.getIndexingInstTime(IndexingSPInstruction inst,
VarStats input1,
VarStats input2,
VarStats output,
IOCostUtils.IOMetrics driverMetrics,
IOCostUtils.IOMetrics executorMetrics) |
|
static double |
SparkCostUtils.getMatMulChainInstTime(MapmmChainSPInstruction inst,
VarStats input1,
VarStats input2,
VarStats input3,
VarStats output,
IOCostUtils.IOMetrics driverMetrics,
IOCostUtils.IOMetrics executorMetrics) |
|
static double |
SparkCostUtils.getMatMulInstTime(BinarySPInstruction inst,
VarStats input1,
VarStats input2,
VarStats output,
IOCostUtils.IOMetrics driverMetrics,
IOCostUtils.IOMetrics executorMetrics) |
|
static double |
IOCostUtils.getMemReadTime(RDDStats stats,
IOCostUtils.IOMetrics metrics) |
Estimate time to scan distributed data sets in memory on Spark.
|
static double |
IOCostUtils.getMemReadTime(VarStats stats,
IOCostUtils.IOMetrics metrics) |
Estimate time to scan object in memory in CP.
|
static double |
IOCostUtils.getMemWriteTime(RDDStats stats,
IOCostUtils.IOMetrics metrics) |
Estimate time to write distributed data set on memory in CP.
|
static double |
IOCostUtils.getMemWriteTime(VarStats stats,
IOCostUtils.IOMetrics metrics) |
Estimate time to write object to memory in CP.
|
static double |
CPCostUtils.getMultiReturnBuiltinInstTime(MultiReturnBuiltinCPInstruction inst,
VarStats input,
VarStats[] outputs,
IOCostUtils.IOMetrics metrics) |
|
static double |
CPCostUtils.getParameterizedBuiltinInstTime(ParameterizedBuiltinCPInstruction inst,
VarStats input,
VarStats output,
IOCostUtils.IOMetrics metrics) |
|
static double |
SparkCostUtils.getParameterizedBuiltinInstTime(ParameterizedBuiltinSPInstruction paramInst,
VarStats input1,
VarStats input2,
VarStats output,
IOCostUtils.IOMetrics driverMetrics,
IOCostUtils.IOMetrics executorMetrics) |
|
static double |
SparkCostUtils.getQSortInstTime(QuantileSortSPInstruction inst,
VarStats input,
VarStats weights,
VarStats output,
IOCostUtils.IOMetrics executorMetrics) |
|
static double |
SparkCostUtils.getQuaternaryInstTime(QuaternarySPInstruction quatInst,
VarStats input1,
VarStats input2,
VarStats input3,
VarStats output,
IOCostUtils.IOMetrics driverMetrics,
IOCostUtils.IOMetrics executorMetrics) |
|
static double |
SparkCostUtils.getRandInstTime(String opcode,
int randType,
VarStats output,
IOCostUtils.IOMetrics executorMetrics) |
|
static double |
SparkCostUtils.getReblockInstTime(String opcode,
VarStats input,
VarStats output,
IOCostUtils.IOMetrics executorMetrics) |
|
static double |
SparkCostUtils.getReorgInstTime(UnarySPInstruction inst,
VarStats input,
VarStats output,
IOCostUtils.IOMetrics executorMetrics) |
|
static double |
IOCostUtils.getSparkCollectTime(RDDStats output,
IOCostUtils.IOMetrics driverMetrics,
IOCostUtils.IOMetrics executorMetrics) |
Estimates the time for collecting Spark Job output;
The output RDD is transferred to the Spark driver at the end of each ResultStage;
time = transfer time (overlaps and dominates the read and deserialization times);
|
static double |
IOCostUtils.getSparkParallelizeTime(RDDStats output,
IOCostUtils.IOMetrics driverMetrics,
IOCostUtils.IOMetrics executorMetrics) |
Estimates the time to parallelize a local object to Spark.
|
static double |
IOCostUtils.getSparkShuffleReadStaticTime(RDDStats input,
IOCostUtils.IOMetrics metrics) |
Estimates the time for reading distributed RDD input at the beginning of a Stage
when a wide-transformation is partition preserving: only local disk reads
|
static double |
IOCostUtils.getSparkShuffleReadTime(RDDStats input,
IOCostUtils.IOMetrics metrics) |
Estimates the time for reading distributed RDD input at the beginning of a Stage;
time = transfer time (overlaps and dominates the read and deserialization times);
For simplification it is assumed that the whole dataset is shuffled;
|
static double |
IOCostUtils.getSparkShuffleTime(RDDStats output,
IOCostUtils.IOMetrics metrics,
boolean withDistribution) |
Combines the shuffle write and read time since these are being typically
added in one place to the general data transmission for instruction estimation.
|
static double |
IOCostUtils.getSparkShuffleWriteTime(RDDStats output,
IOCostUtils.IOMetrics metrics) |
Estimates the time for writing the RDD output to the local system at the end of a ShuffleMapStage;
time = disk write time (overlaps and dominates the serialization time)
The whole data set is being written to shuffle files even if 1 executor is utilized;
|
static double |
SparkCostUtils.getTernaryInstTime(TernarySPInstruction tInst,
VarStats input1,
VarStats input2,
VarStats input3,
VarStats output,
IOCostUtils.IOMetrics executorMetrics) |
|
static double |
SparkCostUtils.getTSMMInstTime(UnarySPInstruction inst,
VarStats input,
VarStats output,
IOCostUtils.IOMetrics driverMetrics,
IOCostUtils.IOMetrics executorMetrics) |
|
static double |
CPCostUtils.getUnaryInstTime(UnaryCPInstruction inst,
VarStats input,
VarStats weights,
VarStats output,
IOCostUtils.IOMetrics metrics) |
|
static double |
SparkCostUtils.getUnaryInstTime(String opcode,
VarStats input,
VarStats output,
IOCostUtils.IOMetrics executorMetrics) |
|
static double |
CPCostUtils.getVariableInstTime(VariableCPInstruction inst,
VarStats input,
VarStats output,
IOCostUtils.IOMetrics metrics) |
|