org.apache.sysds.runtime.util

Class ProgramConverter

java.lang.Object

org.apache.sysds.runtime.util.ProgramConverter

public class ProgramConverter
extends Object

Program converter functionalities for (1) creating deep copies of program blocks, instructions, function program blocks, and (2) serializing and parsing of programs, program blocks, functions program blocks.

Field Summary

Fields

Modifier and Type	Field and Description
static String	CDATA_BEGIN
static String	CDATA_END
static String	COMPONENTS_DELIM
static String	CONF_STATS
static String	DASH
static String	DATA_FIELD_DELIM
static String	EC_BEGIN
static String	EC_END
static String	ELEMENT_DELIM
static String	ELEMENT_DELIM2
static String	EMPTY
static String	INST_BEGIN
static String	INST_END
static String	KEY_VALUE_DELIM
static String	LEVELIN
static String	LEVELOUT
static String	LIST_ELEMENT_DELIM
static String	NEWLINE
static String	NOT_SUPPORTED_EXTERNALFUNCTION_PB
static String	NOT_SUPPORTED_PB
static String	NOT_SUPPORTED_SPARK_INSTRUCTION
static String	NOT_SUPPORTED_SPARK_PARFOR
static String	PARFORBODY_BEGIN
static String	PARFORBODY_END
static String	PB_BEGIN
static String	PB_EFC
static String	PB_END
static String	PB_FC
static String	PB_FOR
static String	PB_IF
static String	PB_PARFOR
static String	PB_WHILE
static String	PBS_BEGIN
static String	PBS_END
static String	PROG_BEGIN
static String	PROG_END
static String	PSBODY_BEGIN
static String	PSBODY_END
static String	REF
static String	VARS_BEGIN
static String	VARS_END

Constructor Summary

Constructors

Constructor and Description
ProgramConverter()

Method Summary

Modifier and Type	Method and Description
static Instruction	cloneInstruction(Instruction oInst, long pid, boolean plain, boolean cpFunctions)
static ExecutionContext	createDeepCopyExecutionContext(ExecutionContext ec) Creates a deep copy of the given execution context.
static ForProgramBlock	createDeepCopyForProgramBlock(ForProgramBlock fpb, long pid, int IDPrefix, Program prog, Set<String> fnStack, Set<String> fnCreated, boolean plain, boolean forceDeepCopy)
static FunctionProgramBlock	createDeepCopyFunctionProgramBlock(FunctionProgramBlock fpb, Set<String> fnStack, Set<String> fnCreated)
static void	createDeepCopyFunctionProgramBlock(String namespace, String oldName, long pid, int IDPrefix, Program prog, Set<String> fnStack, Set<String> fnCreated, boolean plain) This creates a deep copy of a function program block.
static FunctionStatementBlock	createDeepCopyFunctionStatementBlock(FunctionStatementBlock fsb, Set<String> fnStack, Set<String> fnCreated)
static IfProgramBlock	createDeepCopyIfProgramBlock(IfProgramBlock ipb, long pid, int IDPrefix, Program prog, Set<String> fnStack, Set<String> fnCreated, boolean plain, boolean forceDeepCopy)
static ArrayList<Instruction>	createDeepCopyInstructionSet(ArrayList<Instruction> instSet, long pid, int IDPrefix, Program prog, Set<String> fnStack, Set<String> fnCreated, boolean plain, boolean cpFunctions) Creates a deep copy of an array of instructions and replaces the placeholders of parworker IDs with the concrete IDs of this parfor instance.
static ParForProgramBlock	createDeepCopyParForProgramBlock(ParForProgramBlock pfpb, long pid, int IDPrefix, Program prog, Set<String> fnStack, Set<String> fnCreated, boolean plain, boolean forceDeepCopy)
static WhileProgramBlock	createDeepCopyWhileProgramBlock(WhileProgramBlock wpb, long pid, int IDPrefix, Program prog, Set<String> fnStack, Set<String> fnCreated, boolean plain, boolean forceDeepCopy)
static ForStatementBlock	createForStatementBlockCopy(ForStatementBlock sb, boolean forceDeepCopy)
static IfStatementBlock	createIfStatementBlockCopy(IfStatementBlock sb, boolean forceDeepCopy)
static ForProgramBlock	createShallowCopyForProgramBlock(ForProgramBlock fpb, Program prog)
static StatementBlock	createStatementBlockCopy(StatementBlock sb, long pid, boolean plain, boolean forceDeepCopy)
static WhileStatementBlock	createWhileStatementBlockCopy(WhileStatementBlock sb, boolean forceDeepCopy)
static Object[]	parseDataObject(String in) NOTE: MRJobConfiguration cannot be used for the general case because program blocks and related symbol tables can be hierarchically structured.
static ParForBody	parseParForBody(String in, int id)
static ParForBody	parseParForBody(String in, int id, boolean inSpark)
static Program	parseProgram(String in, int id)
static SparkPSBody	parseSparkPSBody(String in, int id)
static ArrayList<ProgramBlock>	rcreateDeepCopyProgramBlocks(ArrayList<ProgramBlock> childBlocks, long pid, int IDPrefix, Set<String> fnStack, Set<String> fnCreated, boolean plain, boolean forceDeepCopy) This recursively creates a deep copy of program blocks and transparently replaces filenames according to the specified parallel worker in order to avoid conflicts between parworkers.
static StatementBlock	rCreateDeepCopyStatementBlock(StatementBlock sb)
static String	saveReplaceFilenameThreadID(String fname, String pattern, String replace)
static String	serializeDataObject(String key, Data dat)
static String	serializeList(List<String> elements)
static String	serializeList(List<String> elements, String delim)
static String	serializeParForBody(ParForBody body)
static String	serializeParForBody(ParForBody body, HashMap<String,byte[]> clsMap)
static String	serializeResultVariables(List<ParForStatementBlock.ResultVar> vars)
static String	serializeSparkPSBody(SparkPSBody body, HashMap<String,byte[]> clsMap)

Methods inherited from class java.lang.Object

equals, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

NEWLINE

public static final String NEWLINE

See Also:

Constant Field Values

COMPONENTS_DELIM

public static final String COMPONENTS_DELIM

See Also:

Constant Field Values

ELEMENT_DELIM

public static final String ELEMENT_DELIM

See Also:

Constant Field Values

ELEMENT_DELIM2

public static final String ELEMENT_DELIM2

See Also:

Constant Field Values

DATA_FIELD_DELIM

public static final String DATA_FIELD_DELIM

See Also:

Constant Field Values

KEY_VALUE_DELIM

public static final String KEY_VALUE_DELIM

See Also:

Constant Field Values

LEVELIN

public static final String LEVELIN

See Also:

Constant Field Values

LEVELOUT

public static final String LEVELOUT

See Also:

Constant Field Values

EMPTY

public static final String EMPTY

See Also:

Constant Field Values

DASH

public static final String DASH

See Also:

Constant Field Values

REF

public static final String REF

See Also:

Constant Field Values

LIST_ELEMENT_DELIM

public static final String LIST_ELEMENT_DELIM

See Also:

Constant Field Values

CDATA_BEGIN

public static final String CDATA_BEGIN

See Also:

Constant Field Values

CDATA_END

public static final String CDATA_END

See Also:

Constant Field Values

PROG_BEGIN

public static final String PROG_BEGIN

See Also:

Constant Field Values

PROG_END

public static final String PROG_END

See Also:

Constant Field Values

VARS_BEGIN

public static final String VARS_BEGIN

See Also:

Constant Field Values

VARS_END

public static final String VARS_END

See Also:

Constant Field Values

PBS_BEGIN

public static final String PBS_BEGIN

See Also:

Constant Field Values

PBS_END

public static final String PBS_END

See Also:

Constant Field Values

INST_BEGIN

public static final String INST_BEGIN

See Also:

Constant Field Values

INST_END

public static final String INST_END

See Also:

Constant Field Values

EC_BEGIN

public static final String EC_BEGIN

See Also:

Constant Field Values

EC_END

public static final String EC_END

See Also:

Constant Field Values

PB_BEGIN

public static final String PB_BEGIN

See Also:

Constant Field Values

PB_END

public static final String PB_END

See Also:

Constant Field Values

PB_WHILE

public static final String PB_WHILE

See Also:

Constant Field Values

PB_FOR

public static final String PB_FOR

See Also:

Constant Field Values

PB_PARFOR

public static final String PB_PARFOR

See Also:

Constant Field Values

PB_IF

public static final String PB_IF

See Also:

Constant Field Values

PB_FC

public static final String PB_FC

See Also:

Constant Field Values

PB_EFC

public static final String PB_EFC

See Also:

Constant Field Values

CONF_STATS

public static final String CONF_STATS

See Also:

Constant Field Values

PARFORBODY_BEGIN

public static final String PARFORBODY_BEGIN

See Also:

Constant Field Values

PARFORBODY_END

public static final String PARFORBODY_END

See Also:

Constant Field Values

PSBODY_BEGIN

public static final String PSBODY_BEGIN

See Also:

Constant Field Values

PSBODY_END

public static final String PSBODY_END

See Also:

Constant Field Values

NOT_SUPPORTED_EXTERNALFUNCTION_PB

public static final String NOT_SUPPORTED_EXTERNALFUNCTION_PB

See Also:

Constant Field Values

NOT_SUPPORTED_SPARK_INSTRUCTION

public static final String NOT_SUPPORTED_SPARK_INSTRUCTION

See Also:

Constant Field Values

NOT_SUPPORTED_SPARK_PARFOR

public static final String NOT_SUPPORTED_SPARK_PARFOR

See Also:

Constant Field Values

NOT_SUPPORTED_PB

public static final String NOT_SUPPORTED_PB

See Also:

Constant Field Values

Constructor Detail

ProgramConverter

public ProgramConverter()

Method Detail

createDeepCopyExecutionContext

public static ExecutionContext createDeepCopyExecutionContext(ExecutionContext ec)
                                                       throws CloneNotSupportedException

Creates a deep copy of the given execution context. For rt_platform=Hadoop, execution context has a symbol table.

Parameters:

ec - execution context

Returns:

execution context

Throws:

CloneNotSupportedException - if CloneNotSupportedException occurs

rcreateDeepCopyProgramBlocks

public static ArrayList<ProgramBlock> rcreateDeepCopyProgramBlocks(ArrayList<ProgramBlock> childBlocks,
                                                                   long pid,
                                                                   int IDPrefix,
                                                                   Set<String> fnStack,
                                                                   Set<String> fnCreated,
                                                                   boolean plain,
                                                                   boolean forceDeepCopy)

This recursively creates a deep copy of program blocks and transparently replaces filenames according to the specified parallel worker in order to avoid conflicts between parworkers. This happens recursively in order to support arbitrary control-flow constructs within a parfor.

Parameters:

childBlocks - child program blocks

pid - ?

IDPrefix - ?

fnStack - ?

fnCreated - ?

plain - if true, full deep copy without id replacement

forceDeepCopy - if true, force deep copy

Returns:

list of program blocks

createDeepCopyWhileProgramBlock

public static WhileProgramBlock createDeepCopyWhileProgramBlock(WhileProgramBlock wpb,
                                                                long pid,
                                                                int IDPrefix,
                                                                Program prog,
                                                                Set<String> fnStack,
                                                                Set<String> fnCreated,
                                                                boolean plain,
                                                                boolean forceDeepCopy)

createDeepCopyIfProgramBlock

public static IfProgramBlock createDeepCopyIfProgramBlock(IfProgramBlock ipb,
                                                          long pid,
                                                          int IDPrefix,
                                                          Program prog,
                                                          Set<String> fnStack,
                                                          Set<String> fnCreated,
                                                          boolean plain,
                                                          boolean forceDeepCopy)

createDeepCopyForProgramBlock

public static ForProgramBlock createDeepCopyForProgramBlock(ForProgramBlock fpb,
                                                            long pid,
                                                            int IDPrefix,
                                                            Program prog,
                                                            Set<String> fnStack,
                                                            Set<String> fnCreated,
                                                            boolean plain,
                                                            boolean forceDeepCopy)

createShallowCopyForProgramBlock

public static ForProgramBlock createShallowCopyForProgramBlock(ForProgramBlock fpb,
                                                               Program prog)

createDeepCopyParForProgramBlock

public static ParForProgramBlock createDeepCopyParForProgramBlock(ParForProgramBlock pfpb,
                                                                  long pid,
                                                                  int IDPrefix,
                                                                  Program prog,
                                                                  Set<String> fnStack,
                                                                  Set<String> fnCreated,
                                                                  boolean plain,
                                                                  boolean forceDeepCopy)

createDeepCopyFunctionProgramBlock

public static void createDeepCopyFunctionProgramBlock(String namespace,
                                                      String oldName,
                                                      long pid,
                                                      int IDPrefix,
                                                      Program prog,
                                                      Set<String> fnStack,
                                                      Set<String> fnCreated,
                                                      boolean plain)

This creates a deep copy of a function program block. The central reference to singletons of function program blocks poses the need for explicit copies in order to prevent conflicting writes of temporary variables (see ExternalFunctionProgramBlock.

Parameters:

namespace - function namespace

oldName - ?

pid - ?

IDPrefix - ?

prog - runtime program

fnStack - ?

fnCreated - ?

plain - ?

createDeepCopyFunctionProgramBlock

public static FunctionProgramBlock createDeepCopyFunctionProgramBlock(FunctionProgramBlock fpb,
                                                                      Set<String> fnStack,
                                                                      Set<String> fnCreated)

createDeepCopyInstructionSet

public static ArrayList<Instruction> createDeepCopyInstructionSet(ArrayList<Instruction> instSet,
                                                                  long pid,
                                                                  int IDPrefix,
                                                                  Program prog,
                                                                  Set<String> fnStack,
                                                                  Set<String> fnCreated,
                                                                  boolean plain,
                                                                  boolean cpFunctions)

Creates a deep copy of an array of instructions and replaces the placeholders of parworker IDs with the concrete IDs of this parfor instance. This is a helper method uses for generating deep copies of program blocks.

Parameters:

instSet - list of instructions

pid - ?

IDPrefix - ?

prog - runtime program

fnStack - ?

fnCreated - ?

plain - ?

cpFunctions - ?

Returns:

list of instructions

cloneInstruction

public static Instruction cloneInstruction(Instruction oInst,
                                           long pid,
                                           boolean plain,
                                           boolean cpFunctions)

createDeepCopyFunctionStatementBlock

public static FunctionStatementBlock createDeepCopyFunctionStatementBlock(FunctionStatementBlock fsb,
                                                                          Set<String> fnStack,
                                                                          Set<String> fnCreated)

rCreateDeepCopyStatementBlock

public static StatementBlock rCreateDeepCopyStatementBlock(StatementBlock sb)

createStatementBlockCopy

public static StatementBlock createStatementBlockCopy(StatementBlock sb,
                                                      long pid,
                                                      boolean plain,
                                                      boolean forceDeepCopy)

createIfStatementBlockCopy

public static IfStatementBlock createIfStatementBlockCopy(IfStatementBlock sb,
                                                          boolean forceDeepCopy)

createWhileStatementBlockCopy

public static WhileStatementBlock createWhileStatementBlockCopy(WhileStatementBlock sb,
                                                                boolean forceDeepCopy)

createForStatementBlockCopy

public static ForStatementBlock createForStatementBlockCopy(ForStatementBlock sb,
                                                            boolean forceDeepCopy)

serializeSparkPSBody

public static String serializeSparkPSBody(SparkPSBody body,
                                          HashMap<String,byte[]> clsMap)

serializeParForBody

public static String serializeParForBody(ParForBody body)

serializeParForBody

public static String serializeParForBody(ParForBody body,
                                         HashMap<String,byte[]> clsMap)

serializeDataObject

public static String serializeDataObject(String key,
                                         Data dat)

serializeResultVariables

public static String serializeResultVariables(List<ParForStatementBlock.ResultVar> vars)

serializeList

public static String serializeList(List<String> elements)

serializeList

public static String serializeList(List<String> elements,
                                   String delim)

parseSparkPSBody

public static SparkPSBody parseSparkPSBody(String in,
                                           int id)

parseParForBody

public static ParForBody parseParForBody(String in,
                                         int id)

parseParForBody

public static ParForBody parseParForBody(String in,
                                         int id,
                                         boolean inSpark)

parseProgram

public static Program parseProgram(String in,
                                   int id)

parseDataObject

public static Object[] parseDataObject(String in)

NOTE: MRJobConfiguration cannot be used for the general case because program blocks and related symbol tables can be hierarchically structured.

Parameters:

in - data object as string

Returns:

array of objects

saveReplaceFilenameThreadID

public static String saveReplaceFilenameThreadID(String fname,
                                                 String pattern,
                                                 String replace)