Package org.apache.sysds.runtime.transform.meta

Class TfMetaUtils

java.lang.Object

org.apache.sysds.runtime.transform.meta.TfMetaUtils

public class TfMetaUtils
extends Object

Constructor Summary

Constructors

Constructor	Description
TfMetaUtils()

Method Summary

Modifier and Type	Method	Description
static void	checkValidEncoders(org.apache.wink.json4j.JSONObject jSpec)
static boolean	containsOmitSpec(String spec, String[] colnames)
static long	getK(org.apache.wink.json4j.JSONObject parsedSpec)	Get K value used for calculation during feature hashing from parsed specifications.
static boolean	isIDSpec(String spec)
static boolean	isIDSpec(org.apache.wink.json4j.JSONObject spec)
static List<Integer>	parseBinningColIDs(String spec, String[] colnames, int minCol, int maxCol)
static List<Integer>	parseBinningColIDs(org.apache.wink.json4j.JSONObject jSpec, String[] colnames, int minCol, int maxCol)
static int[]	parseJsonArrayIDList(org.apache.wink.json4j.JSONArray arr, String[] colnames, int minCol, int maxCol, boolean ids)
static int[]	parseJsonIDList(String spec, String[] colnames, String group)
static int[]	parseJsonIDList(org.apache.wink.json4j.JSONObject spec, String[] colnames, String group)	TODO consolidate external and internal json spec definitions
static int[]	parseJsonIDList(org.apache.wink.json4j.JSONObject spec, String[] colnames, String group, int minCol, int maxCol)
static int	parseJsonObjectID(org.apache.wink.json4j.JSONObject colspec, String[] colnames, int minCol, int maxCol, boolean ids)
static int[]	parseJsonObjectIDList(org.apache.wink.json4j.JSONObject spec, String[] colnames, String group, int minCol, int maxCol)
static int[]	parseJsonPlainArrayIDList(org.apache.wink.json4j.JSONArray arr, String[] colnames, int minCol, int maxCol, boolean ids)
static List<Integer>	parseUDFColIDs(org.apache.wink.json4j.JSONObject jSpec, String[] colnames, int minCol, int maxCol)
static FrameBlock	readTransformMetaDataFromFile(String spec, String metapath, String colDelim)	Reads transform meta data from an HDFS file path and converts it into an in-memory FrameBlock object.
static FrameBlock	readTransformMetaDataFromPath(String spec, String metapath, String colDelim)	Reads transform meta data from the class path and converts it into an in-memory FrameBlock object.

Methods inherited from class java.lang.Object

equals, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Constructor Detail

TfMetaUtils

public TfMetaUtils()

Method Detail

isIDSpec

public static boolean isIDSpec(String spec)

isIDSpec

public static boolean isIDSpec(org.apache.wink.json4j.JSONObject spec)
                        throws org.apache.wink.json4j.JSONException

Throws:

org.apache.wink.json4j.JSONException

containsOmitSpec

public static boolean containsOmitSpec(String spec,
                                       String[] colnames)

parseJsonIDList

public static int[] parseJsonIDList(String spec,
                                    String[] colnames,
                                    String group)

parseJsonIDList

public static int[] parseJsonIDList(org.apache.wink.json4j.JSONObject spec,
                                    String[] colnames,
                                    String group)
                             throws org.apache.wink.json4j.JSONException

TODO consolidate external and internal json spec definitions

Parameters:

spec - transform specification as json string

colnames - column names

group - attribute name in json class

Returns:

list of column ids

Throws:

org.apache.wink.json4j.JSONException - if JSONException occurs

parseJsonIDList

public static int[] parseJsonIDList(org.apache.wink.json4j.JSONObject spec,
                                    String[] colnames,
                                    String group,
                                    int minCol,
                                    int maxCol)
                             throws org.apache.wink.json4j.JSONException

Parameters:

spec - transform specification as json string

colnames - column names

group - attribute name in json class

minCol - start of columns to ignore (1-based, inclusive, if -1 not used)

maxCol - end of columns to ignore (1-based, exclusive, if -1 not used)

Returns:

list of column ids

Throws:

org.apache.wink.json4j.JSONException - if JSONException occurs

parseJsonObjectID

public static int parseJsonObjectID(org.apache.wink.json4j.JSONObject colspec,
                                    String[] colnames,
                                    int minCol,
                                    int maxCol,
                                    boolean ids)
                             throws org.apache.wink.json4j.JSONException

Throws:

org.apache.wink.json4j.JSONException

parseJsonObjectIDList

public static int[] parseJsonObjectIDList(org.apache.wink.json4j.JSONObject spec,
                                          String[] colnames,
                                          String group,
                                          int minCol,
                                          int maxCol)
                                   throws org.apache.wink.json4j.JSONException

Throws:

org.apache.wink.json4j.JSONException

parseJsonArrayIDList

public static int[] parseJsonArrayIDList(org.apache.wink.json4j.JSONArray arr,
                                         String[] colnames,
                                         int minCol,
                                         int maxCol,
                                         boolean ids)
                                  throws org.apache.wink.json4j.JSONException

Throws:

org.apache.wink.json4j.JSONException

parseJsonPlainArrayIDList

public static int[] parseJsonPlainArrayIDList(org.apache.wink.json4j.JSONArray arr,
                                              String[] colnames,
                                              int minCol,
                                              int maxCol,
                                              boolean ids)

getK

public static long getK(org.apache.wink.json4j.JSONObject parsedSpec)
                 throws org.apache.wink.json4j.JSONException

Get K value used for calculation during feature hashing from parsed specifications.

Parameters:

parsedSpec - parsed specifications

Returns:

K value

Throws:

org.apache.wink.json4j.JSONException - if JSONException occurs

readTransformMetaDataFromFile

public static FrameBlock readTransformMetaDataFromFile(String spec,
                                                       String metapath,
                                                       String colDelim)
                                                throws IOException

Reads transform meta data from an HDFS file path and converts it into an in-memory FrameBlock object.

Parameters:

spec - transform specification as json string

metapath - hdfs file path to meta data directory

colDelim - separator for processing column names in the meta data file 'column.names'

Returns:

frame block

Throws:

IOException - if IOException occurs

readTransformMetaDataFromPath

public static FrameBlock readTransformMetaDataFromPath(String spec,
                                                       String metapath,
                                                       String colDelim)
                                                throws IOException

Reads transform meta data from the class path and converts it into an in-memory FrameBlock object.

Parameters:

spec - transform specification as json string

metapath - resource path to meta data directory

colDelim - separator for processing column names in the meta data file 'column.names'

Returns:

frame block

Throws:

IOException - if IOException occurs

parseBinningColIDs

public static List<Integer> parseBinningColIDs(String spec,
                                               String[] colnames,
                                               int minCol,
                                               int maxCol)
                                        throws IOException

Throws:

IOException

parseBinningColIDs

public static List<Integer> parseBinningColIDs(org.apache.wink.json4j.JSONObject jSpec,
                                               String[] colnames,
                                               int minCol,
                                               int maxCol)
                                        throws IOException

Throws:

IOException

parseUDFColIDs

public static List<Integer> parseUDFColIDs(org.apache.wink.json4j.JSONObject jSpec,
                                           String[] colnames,
                                           int minCol,
                                           int maxCol)
                                    throws IOException

Throws:

IOException

checkValidEncoders

public static void checkValidEncoders(org.apache.wink.json4j.JSONObject jSpec)