TestHiveSparkSession

abstract def existingSharedState: Option[SharedState]

abstract def getCachedDataSourceTable(table: TableIdentifier): LogicalPlan

abstract def hiveDefaultTableFilePath(name: TableIdentifier): String

abstract def loadTestTables: Boolean

abstract def metadataHive: HiveClient

abstract def reset(): Unit

abstract def sc: SparkContext

final def !=(arg0: Any): Boolean

final def ##(): Int

final def ==(arg0: Any): Boolean

final def asInstanceOf[T0]: T0

def baseRelationToDataFrame(baseRelation: BaseRelation): DataFrame

Convert a BaseRelation created for external data sources into a DataFrame.

lazy val catalog: Catalog

Interface through which the user may create, drop, alter or query underlying databases, tables, functions etc.

def clone(): AnyRef

def close(): Unit

Synonym for stop().

lazy val conf: RuntimeConfig

Runtime configuration interface for Spark.

def createDataFrame(data: List[_], beanClass: Class[_]): DataFrame

Applies a schema to a List of Java Beans.

def createDataFrame(rdd: JavaRDD[_], beanClass: Class[_]): DataFrame

Applies a schema to an RDD of Java Beans.

def createDataFrame(rdd: RDD[_], beanClass: Class[_]): DataFrame

Applies a schema to an RDD of Java Beans.

def createDataFrame(rows: List[Row], schema: StructType): DataFrame

:: DeveloperApi :: Creates a DataFrame from a java.util.List containing Rows using the given schema.

def createDataFrame(rowRDD: JavaRDD[Row], schema: StructType): DataFrame

:: DeveloperApi :: Creates a DataFrame from a JavaRDD containing Rows using the given schema.

def createDataFrame(rowRDD: RDD[Row], schema: StructType): DataFrame

:: DeveloperApi :: Creates a DataFrame from an RDD containing Rows using the given schema.

import org.apache.spark.sql._
import org.apache.spark.sql.types._
val sparkSession = new org.apache.spark.sql.SparkSession(sc)

val schema =
  StructType(
    StructField("name", StringType, false) ::
    StructField("age", IntegerType, true) :: Nil)

val people =
  sc.textFile("examples/src/main/resources/people.txt").map(
    _.split(",")).map(p => Row(p(0), p(1).trim.toInt))
val dataFrame = sparkSession.createDataFrame(people, schema)
dataFrame.printSchema
// root
// |-- name: string (nullable = false)
// |-- age: integer (nullable = true)

dataFrame.createOrReplaceTempView("people")
sparkSession.sql("select name from people").collect.foreach(println)

def createDataFrame[A <: Product](data: Seq[A])(implicit arg0: scala.reflect.api.JavaUniverse.TypeTag[A]): DataFrame

:: Experimental :: Creates a DataFrame from a local Seq of Product.

def createDataFrame[A <: Product](rdd: RDD[A])(implicit arg0: scala.reflect.api.JavaUniverse.TypeTag[A]): DataFrame

:: Experimental :: Creates a DataFrame from an RDD of Product (e.g.

def createDataset[T](data: List[T])(implicit arg0: Encoder[T]): Dataset[T]

:: Experimental :: Creates a Dataset from a java.util.List of a given type.

List<String> data = Arrays.asList("hello", "world");
Dataset<String> ds = spark.createDataset(data, Encoders.STRING());

def createDataset[T](data: RDD[T])(implicit arg0: Encoder[T]): Dataset[T]

:: Experimental :: Creates a Dataset from an RDD of a given type.

def createDataset[T](data: Seq[T])(implicit arg0: Encoder[T]): Dataset[T]

:: Experimental :: Creates a Dataset from a local Seq of data of a given type.

import spark.implicits._
case class Person(name: String, age: Long)
val data = Seq(Person("Michael", 29), Person("Andy", 30), Person("Justin", 19))
val ds = spark.createDataset(data)

ds.show()
// +-------+---+
// |   name|age|
// +-------+---+
// |Michael| 29|
// |   Andy| 30|
// | Justin| 19|
// +-------+---+

val describedTable: Regex

lazy val emptyDataFrame: DataFrame

Returns a DataFrame with no rows or columns.

def emptyDataset[T](implicit arg0: Encoder[T]): Dataset[T]

:: Experimental :: Creates a new Dataset of type T containing zero elements.

final def eq(arg0: AnyRef): Boolean

def equals(arg0: Any): Boolean

def experimental: ExperimentalMethods

:: Experimental :: A collection of methods that are considered experimental, but can be used to hook into the query planner for advanced functionality.

def finalize(): Unit

final def getClass(): Class[_]

def getHiveFile(path: String): File

def getWarehousePath(): String

def hashCode(): Int

lazy val hiveDevHome: Option[File]

The location of the hive source code.

val hiveFilesTemp: File

lazy val hiveHome: Option[File]

The location of the compiled hive distribution

object implicits extends SQLImplicits with Serializable

:: Experimental :: (Scala-specific) Implicit methods available in Scala for converting common Scala objects into DataFrames.

val sparkSession = SparkSession.builder.getOrCreate()
import sparkSession.implicits._

def initializeLogIfNecessary(isInterpreter: Boolean, silent: Boolean = false): Boolean

def initializeLogIfNecessary(isInterpreter: Boolean): Unit

final def isInstanceOf[T0]: Boolean

def isTraceEnabled(): Boolean

def listenerManager: ExecutionListenerManager

:: Experimental :: An interface to register custom org.apache.spark.sql.util.QueryExecutionListeners that listen for execution metrics.

def loadTestTable(name: String): Unit

val loadedTables: HashSet[String]

def log: Logger

def logDebug(msg: ⇒ String, throwable: Throwable): Unit

def logDebug(msg: ⇒ String): Unit

def logError(msg: ⇒ String, throwable: Throwable): Unit

def logError(msg: ⇒ String): Unit

def logInfo(msg: ⇒ String, throwable: Throwable): Unit

def logInfo(msg: ⇒ String): Unit

def logName: String

def logTrace(msg: ⇒ String, throwable: Throwable): Unit

def logTrace(msg: ⇒ String): Unit

def logWarning(msg: ⇒ String, throwable: Throwable): Unit

def logWarning(msg: ⇒ String): Unit

final def ne(arg0: AnyRef): Boolean

def newSession(): SparkSession

Start a new session with isolated SQL configurations, temporary tables, registered functions are isolated, but sharing the underlying SparkContext and cached data.

final def notify(): Unit

final def notifyAll(): Unit

val originalUDFs: Set[String]

Records the UDFs present when the server starts, so we can delete ones that are created by tests.

def parseDataType(dataTypeString: String): DataType

Parses the data type in our internal string representation.

def range(start: Long, end: Long, step: Long, numPartitions: Int): Dataset[Long]

:: Experimental :: Creates a Dataset with a single LongType column named id, containing elements in a range from start to end (exclusive) with a step value, with partition number specified.

def range(start: Long, end: Long, step: Long): Dataset[Long]

:: Experimental :: Creates a Dataset with a single LongType column named id, containing elements in a range from start to end (exclusive) with a step value.

def range(start: Long, end: Long): Dataset[Long]

:: Experimental :: Creates a Dataset with a single LongType column named id, containing elements in a range from start to end (exclusive) with step value 1.

def range(end: Long): Dataset[Long]

:: Experimental :: Creates a Dataset with a single LongType column named id, containing elements in a range from 0 to end (exclusive) with step value 1.

def read: DataFrameReader

Returns a DataFrameReader that can be used to read non-streaming data in as a DataFrame.

sparkSession.read.parquet("/path/to/file.parquet")
sparkSession.read.schema(schema).json("/path/to/file.json")

def readStream: DataStreamReader

:: Experimental :: Returns a DataStreamReader that can be used to read streaming data in as a DataFrame.

sparkSession.readStream.parquet("/path/to/directory/of/parquet/files")
sparkSession.readStream.schema(schema).json("/path/to/directory/of/json/files")

def registerTestTable(testTable: TestTable): Unit

def setCacheTables(c: Boolean): Unit

lazy val sharedState: SharedState

State shared across sessions, including the SparkContext, cached data, listener, and a catalog that interacts with external systems.

val sparkContext: SparkContext

def sql(sqlText: String): DataFrame

Executes a SQL query using Spark, returning the result as a DataFrame.

val sqlContext: SQLContext

A wrapped version of this session in the form of a SQLContext, for backward compatibility.

def stop(): Unit

Stop the underlying SparkContext.

def streams: StreamingQueryManager

:: Experimental :: Returns a StreamingQueryManager that allows managing all the StreamingQuerys active on this.

final def synchronized[T0](arg0: ⇒ T0): T0

def table(tableName: String): DataFrame

Returns the specified table as a DataFrame.

lazy val testTables: HashMap[String, TestTable]

A list of test tables and the DDL required to initialize them.

def time[T](f: ⇒ T): T

Executes some code block and prints to stdout the time taken to execute the block.

def toString(): String

def udf: UDFRegistration

A collection of methods for registering user-defined functions (UDF).

sparkSession.udf.register("myUDF", (arg1: Int, arg2: String) => arg2 + arg1)

sparkSession.udf().register("myUDF",
    new UDF2<Integer, String, String>() {
        @Override
        public String call(Integer arg1, String arg2) {
            return arg2 + arg1;
        }
   }, DataTypes.StringType);

sparkSession.udf().register("myUDF",
    (Integer arg1, String arg2) -> arg2 + arg1,
    DataTypes.StringType);

def updateUIWithStructuredStreamingTab(): Unit

Adds or updates structured streaming UI tab.

def version: String

The version of Spark on which this application is running.

final def wait(): Unit

final def wait(arg0: Long, arg1: Int): Unit

final def wait(arg0: Long): Unit