SnappyContext

final def !=(arg0: Any): Boolean

final def ##(): Int

final def ==(arg0: Any): Boolean

def alterTable(tableName: String, isAddColumn: Boolean, column: StructField, extensions: String = ""): Unit

alter table adds/drops provided column, only supprted for row tables.

def appendToTempTableCache(df: DataFrame, table: String, storageLevel: StorageLevel = StorageLevel.MEMORY_AND_DISK): Unit

Append dataframe to cache table in Spark.

final def asInstanceOf[T0]: T0

def baseRelationToDataFrame(baseRelation: BaseRelation): DataFrame

Convert a BaseRelation created for external data sources into a DataFrame.

def cacheTable(tableName: String): Unit

Caches the specified table in-memory.

def clear(): Unit

def clearCache(): Unit

Removes all cached tables from the in-memory cache.

def clone(): AnyRef

def createApproxTSTopK(topKName: String, baseTable: String, keyColumnName: String, topkOptions: Map[String, String], allowExisting: Boolean): DataFrame

Create approximate structure to query top-K with time series support.

def createApproxTSTopK(topKName: String, baseTable: Option[String], keyColumnName: String, topkOptions: Map[String, String], allowExisting: Boolean): DataFrame

Create approximate structure to query top-K with time series support.

def createApproxTSTopK(topKName: String, baseTable: String, keyColumnName: String, inputDataSchema: StructType, topkOptions: Map[String, String], allowExisting: Boolean): DataFrame

Create approximate structure to query top-K with time series support.

def createApproxTSTopK(topKName: String, baseTable: Option[String], keyColumnName: String, inputDataSchema: StructType, topkOptions: Map[String, String], allowExisting: Boolean = false): DataFrame

Create approximate structure to query top-K with time series support.

def createDataFrame(rows: Seq[Row], schema: StructType): DataFrame

def createDataFrame(data: List[_], beanClass: Class[_]): DataFrame

Applies a schema to a List of Java Beans.

def createDataFrame(rdd: JavaRDD[_], beanClass: Class[_]): DataFrame

Applies a schema to an RDD of Java Beans.

def createDataFrame(rdd: RDD[_], beanClass: Class[_]): DataFrame

Applies a schema to an RDD of Java Beans.

def createDataFrame(rows: List[Row], schema: StructType): DataFrame

:: DeveloperApi :: Creates a DataFrame from a java.util.List containing Rows using the given schema.

def createDataFrame(rowRDD: JavaRDD[Row], schema: StructType): DataFrame

:: DeveloperApi :: Creates a DataFrame from a JavaRDD containing Rows using the given schema.

def createDataFrame(rowRDD: RDD[Row], schema: StructType): DataFrame

:: DeveloperApi :: Creates a DataFrame from an RDD containing Rows using the given schema.

import org.apache.spark.sql._
import org.apache.spark.sql.types._
val sqlContext = new org.apache.spark.sql.SQLContext(sc)

val schema =
  StructType(
    StructField("name", StringType, false) ::
    StructField("age", IntegerType, true) :: Nil)

val people =
  sc.textFile("examples/src/main/resources/people.txt").map(
    _.split(",")).map(p => Row(p(0), p(1).trim.toInt))
val dataFrame = sqlContext.createDataFrame(people, schema)
dataFrame.printSchema
// root
// |-- name: string (nullable = false)
// |-- age: integer (nullable = true)

dataFrame.createOrReplaceTempView("people")
sqlContext.sql("select name from people").collect.foreach(println)

def createDataFrame[A <: Product](data: Seq[A])(implicit arg0: scala.reflect.api.JavaUniverse.TypeTag[A]): DataFrame

:: Experimental :: Creates a DataFrame from a local Seq of Product.

def createDataFrame[A <: Product](rdd: RDD[A])(implicit arg0: scala.reflect.api.JavaUniverse.TypeTag[A]): DataFrame

:: Experimental :: Creates a DataFrame from an RDD of Product (e.g.

def createDataFrameUsingRDD[A <: Product](rdd: RDD[A])(implicit arg0: scala.reflect.api.JavaUniverse.TypeTag[A]): DataFrame

:: Experimental :: Creates a DataFrame from an RDD of Product (e.g.

def createDataset[T](data: List[T])(implicit arg0: Encoder[T]): Dataset[T]

:: Experimental :: Creates a Dataset from a java.util.List of a given type.

List<String> data = Arrays.asList("hello", "world");
Dataset<String> ds = spark.createDataset(data, Encoders.STRING());

def createDataset[T](data: RDD[T])(implicit arg0: Encoder[T]): Dataset[T]

:: Experimental :: Creates a Dataset from an RDD of a given type.

def createDataset[T](data: Seq[T])(implicit arg0: Encoder[T]): Dataset[T]

:: Experimental :: Creates a Dataset from a local Seq of data of a given type.

import spark.implicits._
case class Person(name: String, age: Long)
val data = Seq(Person("Michael", 29), Person("Andy", 30), Person("Justin", 19))
val ds = spark.createDataset(data)

ds.show()
// +-------+---+
// |   name|age|
// +-------+---+
// |Michael| 29|
// |   Andy| 30|
// | Justin| 19|
// +-------+---+

def createExternalTable(tableName: String, source: String, schema: StructType, options: Map[String, String]): DataFrame

(Scala-specific) Create an external table from the given path based on a data source, a schema and a set of options.

def createExternalTable(tableName: String, source: String, schema: StructType, options: Map[String, String]): DataFrame

Create an external table from the given path based on a data source, a schema and a set of options.

def createExternalTable(tableName: String, source: String, options: Map[String, String]): DataFrame

(Scala-specific) Creates an external table from the given path based on a data source and a set of options.

def createExternalTable(tableName: String, source: String, options: Map[String, String]): DataFrame

Creates an external table from the given path based on a data source and a set of options.

def createExternalTable(tableName: String, path: String, source: String): DataFrame

Creates an external table from the given path based on a data source and returns the corresponding DataFrame.

def createExternalTable(tableName: String, path: String): DataFrame

Creates an external table from the given path and returns the corresponding DataFrame.

def createIndex(indexName: String, baseTable: String, indexColumns: Seq[(String, Option[SortDirection])], options: Map[String, String]): Unit

Create an index on a table.

def createIndex(indexName: String, baseTable: String, indexColumns: List[String], sortOrders: List[Boolean], options: Map[String, String]): Unit

Create an index on a table.

def createSampleTable(tableName: String, baseTable: String, schema: StructType, samplingOptions: Map[String, String], allowExisting: Boolean): DataFrame

Create a stratified sample table.

def createSampleTable(tableName: String, baseTable: Option[String], schema: StructType, samplingOptions: Map[String, String], allowExisting: Boolean = false): DataFrame

Create a stratified sample table.

def createSampleTable(tableName: String, baseTable: String, samplingOptions: Map[String, String], allowExisting: Boolean): DataFrame

Create a stratified sample table.

def createSampleTable(tableName: String, baseTable: Option[String], samplingOptions: Map[String, String], allowExisting: Boolean): DataFrame

Create a stratified sample table.

def createTable(tableName: String, provider: String, schemaDDL: String, options: Map[String, String], allowExisting: Boolean): DataFrame

Creates a SnappyData managed JDBC table which takes a free format ddl string.

   val props = Map(
     "url" -> s"jdbc:derby:$path",
     "driver" -> "org.apache.derby.jdbc.EmbeddedDriver",
     "poolImpl" -> "tomcat",
     "user" -> "app",
     "password" -> "app"
   )

val schemaDDL = "(OrderId INT NOT NULL PRIMARY KEY,ItemId INT, ITEMREF INT)"
snappyContext.createTable("jdbcTable", "jdbc", schemaDDL, props)

case class Data(col1: Int, col2: Int, col3: Int)

val data = Seq(Data(1, 2, 3), Data(7, 8, 9), Data(9, 2, 3), Data(4, 2, 3), Data(5, 6, 7))
val dataDF = snc.createDataset(data)(Encoders.product)
dataDF.write.insertInto("jdbcTable")

def createTable(tableName: String, provider: String, schemaDDL: String, options: Map[String, String], allowExisting: Boolean): DataFrame

Creates a SnappyData managed JDBC table which takes a free format ddl string.

   val props = Map(
     "url" -> s"jdbc:derby:$path",
     "driver" -> "org.apache.derby.jdbc.EmbeddedDriver",
     "poolImpl" -> "tomcat",
     "user" -> "app",
     "password" -> "app"
   )

val schemaDDL = "(OrderId INT NOT NULL PRIMARY KEY,ItemId INT, ITEMREF INT)"
snappyContext.createTable("jdbcTable", "jdbc", schemaDDL, props)

case class Data(col1: Int, col2: Int, col3: Int)

val data = Seq(Data(1, 2, 3), Data(7, 8, 9), Data(9, 2, 3), Data(4, 2, 3), Data(5, 6, 7))
val dataDF = snc.createDataset(data)(Encoders.product)
dataDF.write.insertInto("jdbcTable")

def createTable(tableName: String, provider: String, schema: StructType, options: Map[String, String], allowExisting: Boolean): DataFrame

Creates a SnappyData managed table.

case class Data(col1: Int, col2: Int, col3: Int)
val props = Map.empty[String, String]
val data = Seq(Data(1, 2, 3), Data(7, 8, 9), Data(9, 2, 3), Data(4, 2, 3), Data(5, 6, 7))
val dataDF = snc.createDataset(data)(Encoders.product)
snappyContext.createTable(tableName, "column", dataDF.schema, props)

def createTable(tableName: String, provider: String, schema: StructType, options: Map[String, String], allowExisting: Boolean = false): DataFrame

Creates a SnappyData managed table.

case class Data(col1: Int, col2: Int, col3: Int)
val props = Map.empty[String, String]
val data = Seq(Data(1, 2, 3), Data(7, 8, 9), Data(9, 2, 3), Data(4, 2, 3), Data(5, 6, 7))
val dataDF = snc.createDataset(data)(Encoders.product)
snappyContext.createTable(tableName, "column", dataDF.schema, props)

def createTable(tableName: String, provider: String, options: Map[String, String], allowExisting: Boolean): DataFrame

Creates a SnappyData managed table.

val airlineDF = snappyContext.createTable(stagingAirline,
  "column", Map("buckets" -> "29"))

def createTable(tableName: String, provider: String, options: Map[String, String], allowExisting: Boolean): DataFrame

Creates a SnappyData managed table.

val airlineDF = snappyContext.createTable(stagingAirline,
  "column", Map("buckets" -> "29"))

def delete(tableName: String, filterExpr: String): Int

Delete all rows in table that match passed filter expression

def dropIndex(indexName: String, ifExists: Boolean): Unit

Drops an index on a table

def dropTable(tableName: String, ifExists: Boolean = false): Unit

Drop a SnappyData table created by a call to SnappyContext.createTable, createExternalTable or registerTempTable.

def dropTempTable(tableName: String): Unit

Drops the temporary table with the given table name in the catalog.

def emptyDataFrame: DataFrame

Returns a DataFrame with no rows or columns.

final def eq(arg0: AnyRef): Boolean

def equals(arg0: Any): Boolean

def experimental: ExperimentalMethods

:: Experimental :: A collection of methods that are considered experimental, but can be used to hook into the query planner for advanced functionality.

def finalize(): Unit

def getAllConfs: Map[String, String]

Return all the configuration properties that have been set (i.e.

final def getClass(): Class[_]

def getConf(key: String, defaultValue: String): String

Return the value of Spark SQL configuration property for the given key.

def getConf(key: String): String

Return the value of Spark SQL configuration property for the given key.

def hashCode(): Int

object implicits extends SQLImplicits with Serializable

:: Experimental :: (Scala-specific) Implicit methods available in Scala for converting common Scala objects into DataFrames.

val sqlContext = new SQLContext(sc)
import sqlContext.implicits._

def initializeLogIfNecessary(isInterpreter: Boolean, silent: Boolean = false): Boolean

def initializeLogIfNecessary(isInterpreter: Boolean): Unit

def insert(tableName: String, rows: ArrayList[ArrayList[_]]): Int

Insert one or more org.apache.spark.sql.Row into an existing table

java.util.ArrayList[java.util.ArrayList[_] rows = ...    *
snc.insert(tableName, rows)

def insert(tableName: String, rows: Row*): Int

Insert one or more org.apache.spark.sql.Row into an existing table

snc.insert(tableName, dataDF.collect(): _*)

def isCached(tableName: String): Boolean

Returns true if the table is currently cached in-memory.

final def isInstanceOf[T0]: Boolean

def isTraceEnabled(): Boolean

def listenerManager: ExecutionListenerManager

An interface to register custom org.apache.spark.sql.util.QueryExecutionListeners that listen for execution metrics.

def log: Logger

def logDebug(msg: ⇒ String, throwable: Throwable): Unit

def logDebug(msg: ⇒ String): Unit

def logError(msg: ⇒ String, throwable: Throwable): Unit

def logError(msg: ⇒ String): Unit

def logInfo(msg: ⇒ String, throwable: Throwable): Unit

def logInfo(msg: ⇒ String): Unit

def logName: String

def logTrace(msg: ⇒ String, throwable: Throwable): Unit

def logTrace(msg: ⇒ String): Unit

def logWarning(msg: ⇒ String, throwable: Throwable): Unit

def logWarning(msg: ⇒ String): Unit

final def ne(arg0: AnyRef): Boolean

def newSession(): SnappyContext

Returns a SQLContext as new session, with separated SQL configurations, temporary tables, registered functions, but sharing the same SparkContext, cached data and other things.

final def notify(): Unit

final def notifyAll(): Unit

def put(tableName: String, rows: ArrayList[ArrayList[_]]): Int

Upsert one or more org.apache.spark.sql.Row into an existing table

java.util.ArrayList[java.util.ArrayList[_] rows = ...    *
 snSession.put(tableName, rows)

def put(tableName: String, rows: Row*): Int

Upsert one or more org.apache.spark.sql.Row into an existing table

snSession.put(tableName, dataDF.collect(): _*)

def queryApproxTSTopK(topK: String, startTime: Long, endTime: Long, k: Int): DataFrame

def queryApproxTSTopK(topKName: String, startTime: Long, endTime: Long): DataFrame

def queryApproxTSTopK(topKName: String, startTime: String = null, endTime: String = null, k: Int = 1): DataFrame

Fetch the topK entries in the Approx TopK synopsis for the specified time interval.

def range(start: Long, end: Long, step: Long, numPartitions: Int): DataFrame

:: Experimental :: Creates a DataFrame with a single LongType column named id, containing elements in an range from start to end (exclusive) with an step value, with partition number specified.

def range(start: Long, end: Long, step: Long): DataFrame

:: Experimental :: Creates a DataFrame with a single LongType column named id, containing elements in a range from start to end (exclusive) with a step value.

def range(start: Long, end: Long): DataFrame

:: Experimental :: Creates a DataFrame with a single LongType column named id, containing elements in a range from start to end (exclusive) with step value 1.

def range(end: Long): DataFrame

:: Experimental :: Creates a DataFrame with a single LongType column named id, containing elements in a range from 0 to end (exclusive) with step value 1.

def read: DataFrameReader

Returns a DataFrameReader that can be used to read non-streaming data in as a DataFrame.

sqlContext.read.parquet("/path/to/file.parquet")
sqlContext.read.schema(schema).json("/path/to/file.json")

def readStream: DataStreamReader

:: Experimental :: Returns a DataStreamReader that can be used to read streaming data in as a DataFrame.

sparkSession.readStream.parquet("/path/to/directory/of/parquet/files")
sparkSession.readStream.schema(schema).json("/path/to/directory/of/json/files")

def saveStream[T](stream: DStream[T], aqpTables: Seq[String], transformer: Option[(RDD[T]) ⇒ RDD[Row]])(implicit v: scala.reflect.api.JavaUniverse.TypeTag[T]): Unit

:: DeveloperApi ::

def sessionState: SnappySessionState

def setConf(key: String, value: String): Unit

Set the given Spark SQL configuration property.

def setConf(props: Properties): Unit

Set Spark SQL configuration properties.

def setCurrentSchema(schemaName: String): Unit

Set current database/schema.

val snappySession: SnappySession

def sparkContext: SparkContext

val sparkSession: SparkSession

def sql(sqlText: String): DataFrame

Executes a SQL query using Spark, returning the result as a DataFrame.

def sqlUncached(sqlText: String): DataFrame

Run SQL string without any plan caching.

def streams: StreamingQueryManager

Returns a StreamingQueryManager that allows managing all the StreamingQueries active on this context.

final def synchronized[T0](arg0: ⇒ T0): T0

def table(tableName: String): DataFrame

Returns the specified table as a DataFrame.

def tableNames(databaseName: String): Array[String]

Returns the names of tables in the given database as an array.

def tableNames(): Array[String]

Returns the names of tables in the current database as an array.

def tables(databaseName: String): DataFrame

Returns a DataFrame containing names of existing tables in the given database.

def tables(): DataFrame

Returns a DataFrame containing names of existing tables in the current database.

def toString(): String

def truncateTable(tableName: String, ifExists: Boolean = false): Unit

Empties the contents of the table without deleting the catalog entry.

def udf: UDFRegistration

A collection of methods for registering user-defined functions (UDF).

sqlContext.udf.register("myUDF", (arg1: Int, arg2: String) => arg2 + arg1)

sqlContext.udf().register("myUDF",
    new UDF2<Integer, String, String>() {
        @Override
        public String call(Integer arg1, String arg2) {
            return arg2 + arg1;
        }
   }, DataTypes.StringType);

sqlContext.udf().register("myUDF",
    (Integer arg1, String arg2) -> arg2 + arg1,
    DataTypes.StringType);

def uncacheTable(tableName: String): Unit

Removes the specified table from the in-memory cache.

def update(tableName: String, filterExpr: String, newColumnValues: ArrayList[_], updateColumns: ArrayList[String]): Int

Update all rows in table that match passed filter expression

snappyContext.update("jdbcTable", "ITEMREF = 3" , Row(99) , "ITEMREF" )

def update(tableName: String, filterExpr: String, newColumnValues: Row, updateColumns: String*): Int

Update all rows in table that match passed filter expression

snappyContext.update("jdbcTable", "ITEMREF = 3" , Row(99) , "ITEMREF" )

final def wait(): Unit

final def wait(arg0: Long, arg1: Int): Unit

final def wait(arg0: Long): Unit

def applySchema(rdd: JavaRDD[_], beanClass: Class[_]): DataFrame

def applySchema(rdd: RDD[_], beanClass: Class[_]): DataFrame

def applySchema(rowRDD: JavaRDD[Row], schema: StructType): DataFrame

def applySchema(rowRDD: RDD[Row], schema: StructType): DataFrame

def jdbc(url: String, table: String, theParts: Array[String]): DataFrame

Construct a DataFrame representing the database table accessible via JDBC URL url named table.

def jdbc(url: String, table: String, columnName: String, lowerBound: Long, upperBound: Long, numPartitions: Int): DataFrame

Construct a DataFrame representing the database table accessible via JDBC URL url named table.

def jdbc(url: String, table: String): DataFrame

Construct a DataFrame representing the database table accessible via JDBC URL url named table.

def jsonFile(path: String, samplingRatio: Double): DataFrame

def jsonFile(path: String, schema: StructType): DataFrame

Loads a JSON file (one object per line) and applies the given schema, returning the result as a DataFrame.

def jsonFile(path: String): DataFrame

Loads a JSON file (one object per line), returning the result as a DataFrame.

def jsonRDD(json: JavaRDD[String], samplingRatio: Double): DataFrame

Loads a JavaRDD[String] storing JSON objects (one object per record) inferring the schema, returning the result as a DataFrame.

def jsonRDD(json: RDD[String], samplingRatio: Double): DataFrame

Loads an RDD[String] storing JSON objects (one object per record) inferring the schema, returning the result as a DataFrame.

def jsonRDD(json: JavaRDD[String], schema: StructType): DataFrame

Loads an JavaRDD[String] storing JSON objects (one object per record) and applies the given schema, returning the result as a DataFrame.

def jsonRDD(json: RDD[String], schema: StructType): DataFrame

Loads an RDD[String] storing JSON objects (one object per record) and applies the given schema, returning the result as a DataFrame.

def jsonRDD(json: JavaRDD[String]): DataFrame

Loads an RDD[String] storing JSON objects (one object per record), returning the result as a DataFrame.

def jsonRDD(json: RDD[String]): DataFrame

Loads an RDD[String] storing JSON objects (one object per record), returning the result as a DataFrame.

def load(source: String, schema: StructType, options: Map[String, String]): DataFrame

(Scala-specific) Returns the dataset specified by the given data source and a set of options as a DataFrame, using the given schema as the schema of the DataFrame.

def load(source: String, schema: StructType, options: Map[String, String]): DataFrame

(Java-specific) Returns the dataset specified by the given data source and a set of options as a DataFrame, using the given schema as the schema of the DataFrame.

def load(source: String, options: Map[String, String]): DataFrame

(Scala-specific) Returns the dataset specified by the given data source and a set of options as a DataFrame.

def load(source: String, options: Map[String, String]): DataFrame

(Java-specific) Returns the dataset specified by the given data source and a set of options as a DataFrame.

def load(path: String, source: String): DataFrame

Returns the dataset stored at path as a DataFrame, using the given data source.

def load(path: String): DataFrame

Returns the dataset stored at path as a DataFrame, using the default data source configured by spark.sql.sources.default.

def parquetFile(paths: String*): DataFrame

Loads a Parquet file, returning the result as a DataFrame.