org.apache.sysml.runtime.instructions.spark.utils

Class SparkUtils

java.lang.Object

org.apache.sysml.runtime.instructions.spark.utils.SparkUtils

public class SparkUtils
extends Object

Field Summary

Fields

Modifier and Type	Field and Description
static org.apache.spark.storage.StorageLevel	DEFAULT_TMP

Constructor Summary

Constructors

Constructor and Description
SparkUtils()

Method Summary

Modifier and Type	Method and Description
static org.apache.spark.api.java.JavaPairRDD<MatrixIndexes,MatrixCell>	cacheBinaryCellRDD(org.apache.spark.api.java.JavaPairRDD<MatrixIndexes,MatrixCell> input)
static MatrixCharacteristics	computeMatrixCharacteristics(org.apache.spark.api.java.JavaPairRDD<MatrixIndexes,MatrixCell> input) Utility to compute dimensions and non-zeros in a given RDD of binary cells.
static org.apache.spark.api.java.JavaPairRDD<MatrixIndexes,MatrixBlock>	copyBinaryBlockMatrix(org.apache.spark.api.java.JavaPairRDD<MatrixIndexes,MatrixBlock> in) Creates a partitioning-preserving deep copy of the input matrix RDD, where the indexes and values are copied.
static org.apache.spark.api.java.JavaPairRDD<MatrixIndexes,MatrixBlock>	copyBinaryBlockMatrix(org.apache.spark.api.java.JavaPairRDD<MatrixIndexes,MatrixBlock> in, boolean deep) Creates a partitioning-preserving copy of the input matrix RDD.
static ArrayList<scala.Tuple2<Long,FrameBlock>>	fromIndexedFrameBlock(ArrayList<Pair<Long,FrameBlock>> in)
static scala.Tuple2<Long,FrameBlock>	fromIndexedFrameBlock(Pair<Long,FrameBlock> in)
static ArrayList<scala.Tuple2<MatrixIndexes,MatrixBlock>>	fromIndexedMatrixBlock(ArrayList<org.apache.sysml.runtime.matrix.mapred.IndexedMatrixValue> in)
static scala.Tuple2<MatrixIndexes,MatrixBlock>	fromIndexedMatrixBlock(org.apache.sysml.runtime.matrix.mapred.IndexedMatrixValue in)
static ArrayList<Pair<MatrixIndexes,MatrixBlock>>	fromIndexedMatrixBlockToPair(ArrayList<org.apache.sysml.runtime.matrix.mapred.IndexedMatrixValue> in)
static Pair<MatrixIndexes,MatrixBlock>	fromIndexedMatrixBlockToPair(org.apache.sysml.runtime.matrix.mapred.IndexedMatrixValue in)
static org.apache.spark.api.java.JavaPairRDD<MatrixIndexes,MatrixBlock>	getEmptyBlockRDD(org.apache.spark.api.java.JavaSparkContext sc, MatrixCharacteristics mc) Creates an RDD of empty blocks according to the given matrix characteristics.
static int	getNumPreferredPartitions(MatrixCharacteristics mc)
static int	getNumPreferredPartitions(MatrixCharacteristics mc, org.apache.spark.api.java.JavaPairRDD<?,?> in)
static String	getPrefixFromSparkDebugInfo(String line)
static String	getStartLineFromSparkDebugInfo(String line)
static boolean	isHashPartitioned(org.apache.spark.api.java.JavaPairRDD<?,?> in) Indicates if the input RDD is hash partitioned, i.e., it has a partitioner of type org.apache.spark.HashPartitioner.
static Pair<Long,FrameBlock>	toIndexedFrameBlock(scala.Tuple2<Long,FrameBlock> in)
static ArrayList<Pair<Long,Long>>	toIndexedLong(List<scala.Tuple2<Long,Long>> in)
static org.apache.sysml.runtime.matrix.mapred.IndexedMatrixValue	toIndexedMatrixBlock(MatrixIndexes ix, MatrixBlock mb)
static org.apache.sysml.runtime.matrix.mapred.IndexedMatrixValue	toIndexedMatrixBlock(scala.Tuple2<MatrixIndexes,MatrixBlock> in)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

DEFAULT_TMP

public static final org.apache.spark.storage.StorageLevel DEFAULT_TMP

Constructor Detail

SparkUtils

public SparkUtils()

Method Detail

toIndexedMatrixBlock

public static org.apache.sysml.runtime.matrix.mapred.IndexedMatrixValue toIndexedMatrixBlock(scala.Tuple2<MatrixIndexes,MatrixBlock> in)

toIndexedMatrixBlock

public static org.apache.sysml.runtime.matrix.mapred.IndexedMatrixValue toIndexedMatrixBlock(MatrixIndexes ix,
                                                                                             MatrixBlock mb)

fromIndexedMatrixBlock

public static scala.Tuple2<MatrixIndexes,MatrixBlock> fromIndexedMatrixBlock(org.apache.sysml.runtime.matrix.mapred.IndexedMatrixValue in)

fromIndexedMatrixBlock

public static ArrayList<scala.Tuple2<MatrixIndexes,MatrixBlock>> fromIndexedMatrixBlock(ArrayList<org.apache.sysml.runtime.matrix.mapred.IndexedMatrixValue> in)

fromIndexedMatrixBlockToPair

public static Pair<MatrixIndexes,MatrixBlock> fromIndexedMatrixBlockToPair(org.apache.sysml.runtime.matrix.mapred.IndexedMatrixValue in)

fromIndexedMatrixBlockToPair

public static ArrayList<Pair<MatrixIndexes,MatrixBlock>> fromIndexedMatrixBlockToPair(ArrayList<org.apache.sysml.runtime.matrix.mapred.IndexedMatrixValue> in)

fromIndexedFrameBlock

public static scala.Tuple2<Long,FrameBlock> fromIndexedFrameBlock(Pair<Long,FrameBlock> in)

fromIndexedFrameBlock

public static ArrayList<scala.Tuple2<Long,FrameBlock>> fromIndexedFrameBlock(ArrayList<Pair<Long,FrameBlock>> in)

toIndexedLong

public static ArrayList<Pair<Long,Long>> toIndexedLong(List<scala.Tuple2<Long,Long>> in)

toIndexedFrameBlock

public static Pair<Long,FrameBlock> toIndexedFrameBlock(scala.Tuple2<Long,FrameBlock> in)

isHashPartitioned

public static boolean isHashPartitioned(org.apache.spark.api.java.JavaPairRDD<?,?> in)

Indicates if the input RDD is hash partitioned, i.e., it has a partitioner of type org.apache.spark.HashPartitioner.

Parameters:

in - input JavaPairRDD

Returns:

true if input is hash partitioned

getNumPreferredPartitions

public static int getNumPreferredPartitions(MatrixCharacteristics mc,
                                            org.apache.spark.api.java.JavaPairRDD<?,?> in)

getNumPreferredPartitions

public static int getNumPreferredPartitions(MatrixCharacteristics mc)

copyBinaryBlockMatrix

public static org.apache.spark.api.java.JavaPairRDD<MatrixIndexes,MatrixBlock> copyBinaryBlockMatrix(org.apache.spark.api.java.JavaPairRDD<MatrixIndexes,MatrixBlock> in)

Creates a partitioning-preserving deep copy of the input matrix RDD, where the indexes and values are copied.

Parameters:

in - matrix as JavaPairRDD<MatrixIndexes,MatrixBlock>

Returns:

matrix as JavaPairRDD<MatrixIndexes,MatrixBlock>

copyBinaryBlockMatrix

public static org.apache.spark.api.java.JavaPairRDD<MatrixIndexes,MatrixBlock> copyBinaryBlockMatrix(org.apache.spark.api.java.JavaPairRDD<MatrixIndexes,MatrixBlock> in,
                                                                                                     boolean deep)

Creates a partitioning-preserving copy of the input matrix RDD. If a deep copy is requested, indexes and values are copied, otherwise they are simply passed through.

Parameters:

in - matrix as JavaPairRDD<MatrixIndexes,MatrixBlock>

deep - if true, perform deep copy

Returns:

matrix as JavaPairRDD<MatrixIndexes,MatrixBlock>

getStartLineFromSparkDebugInfo

public static String getStartLineFromSparkDebugInfo(String line)
                                             throws DMLRuntimeException

Throws:

DMLRuntimeException

getPrefixFromSparkDebugInfo

public static String getPrefixFromSparkDebugInfo(String line)

getEmptyBlockRDD

public static org.apache.spark.api.java.JavaPairRDD<MatrixIndexes,MatrixBlock> getEmptyBlockRDD(org.apache.spark.api.java.JavaSparkContext sc,
                                                                                                MatrixCharacteristics mc)

Creates an RDD of empty blocks according to the given matrix characteristics. This is done in a scalable manner by parallelizing block ranges and generating empty blocks in a distributed manner, under awareness of preferred output partition sizes.

Parameters:

sc - spark context

mc - matrix characteristics

Returns:

pair rdd of empty matrix blocks

cacheBinaryCellRDD

public static org.apache.spark.api.java.JavaPairRDD<MatrixIndexes,MatrixCell> cacheBinaryCellRDD(org.apache.spark.api.java.JavaPairRDD<MatrixIndexes,MatrixCell> input)

computeMatrixCharacteristics

public static MatrixCharacteristics computeMatrixCharacteristics(org.apache.spark.api.java.JavaPairRDD<MatrixIndexes,MatrixCell> input)

Utility to compute dimensions and non-zeros in a given RDD of binary cells.

Parameters:

input - matrix as JavaPairRDD<MatrixIndexes, MatrixCell>

Returns:

matrix characteristics