SQLConf

Instance Constructors

new SQLConf()

Value Members

final def !=(arg0: Any): Boolean

Definition Classes
AnyRef → Any
final def ##(): Int

Definition Classes
AnyRef → Any
final def ==(arg0: Any): Boolean

Definition Classes
AnyRef → Any
def adaptiveExecutionEnabled: Boolean
final def asInstanceOf[T0]: T0

Definition Classes
Any
def autoBroadcastJoinThreshold: Long
def broadcastTimeout: Int
def bucketingEnabled: Boolean
def caseSensitiveAnalysis: Boolean
def caseSensitiveInferenceMode: SQLConf.HiveCaseSensitiveInferenceMode.Value
def checkpointLocation: Option[String]
def clear(): Unit
def clone(): AnyRef

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( ... )
def columnBatchSize: Int
def columnNameOfCorruptRecord: String
def constraintPropagationEnabled: Boolean
def contains(key: String): Boolean

Return whether a given key is set in this SQLConf.
def convertCTAS: Boolean
def crossJoinEnabled: Boolean
def dataFramePivotMaxValues: Int
def dataFrameRetainGroupColumns: Boolean
def dataFrameSelfJoinAutoResolveAmbiguity: Boolean
def defaultDataSourceName: String
def defaultSizeInBytes: Long
def enableRadixSort: Boolean
def enableTwoLevelAggMap: Boolean
final def eq(arg0: AnyRef): Boolean

Definition Classes
AnyRef
def equals(arg0: Any): Boolean

Definition Classes
AnyRef → Any
def exchangeReuseEnabled: Boolean
def fallBackToHdfsForStatsEnabled: Boolean
def fileCommitProtocolClass: String
def fileSinkLogCleanupDelay: Long
def fileSinkLogCompactInterval: Int
def fileSinkLogDeletion: Boolean
def fileSourceLogCleanupDelay: Long
def fileSourceLogCompactInterval: Int
def fileSourceLogDeletion: Boolean
def filesMaxPartitionBytes: Long
def filesOpenCostInBytes: Long
def filesourcePartitionFileCacheSize: Long
def finalize(): Unit

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( classOf[java.lang.Throwable] )
def gatherFastStats: Boolean
def getAllConfs: Map[String, String]

Return all the configuration properties that have been set (i.e.
Return all the configuration properties that have been set (i.e. not the default). This creates a new copy of the config properties in the form of a Map.
def getAllDefinedConfs: Seq[(String, String, String)]

Return all the configuration definitions that have been defined in SQLConf.
Return all the configuration definitions that have been defined in SQLConf. Each definition contains key, defaultValue and doc.
final def getClass(): Class[_]

Definition Classes
AnyRef → Any
def getConf[T](entry: OptionalConfigEntry[T]): Option[T]

Return the value of an optional Spark SQL configuration property for the given key.
Return the value of an optional Spark SQL configuration property for the given key. If the key is not set yet, returns None.
def getConf[T](entry: ConfigEntry[T]): T

Return the value of Spark SQL configuration property for the given key.
Return the value of Spark SQL configuration property for the given key. If the key is not set yet, return defaultValue in ConfigEntry.
def getConf[T](entry: ConfigEntry[T], defaultValue: T): T

Return the value of Spark SQL configuration property for the given key.
Return the value of Spark SQL configuration property for the given key. If the key is not set yet, return defaultValue. This is useful when defaultValue in ConfigEntry is not the desired one.
def getConfString(key: String, defaultValue: String): String

Return the string value of Spark SQL configuration property for the given key.
Return the string value of Spark SQL configuration property for the given key. If the key is not set yet, return defaultValue.
def getConfString(key: String): String

Return the value of Spark SQL configuration property for the given key.
Return the value of Spark SQL configuration property for the given key.

Annotations
@throws( "if key is not set" )
def groupByOrdinal: Boolean
def hashCode(): Int

Definition Classes
AnyRef → Any
def ignoreCorruptFiles: Boolean
def inMemoryPartitionPruning: Boolean
def initializeLogIfNecessary(isInterpreter: Boolean, silent: Boolean = false): Boolean

Attributes
protected
Definition Classes
Logging
def initializeLogIfNecessary(isInterpreter: Boolean): Unit

Attributes
protected
Definition Classes
Logging
final def isInstanceOf[T0]: Boolean

Definition Classes
Any
def isParquetBinaryAsString: Boolean
def isParquetINT96AsTimestamp: Boolean
def isParquetSchemaMergingEnabled: Boolean
def isParquetSchemaRespectSummaries: Boolean
def isTraceEnabled(): Boolean

Attributes
protected
Definition Classes
Logging
def isUnsupportedOperationCheckEnabled: Boolean
def limitScaleUpFactor: Int
def log: Logger

Attributes
protected
Definition Classes
Logging
def logDebug(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logDebug(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
def logError(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logError(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
def logInfo(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logInfo(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
def logName: String

Attributes
protected
Definition Classes
Logging
def logTrace(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logTrace(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
def logWarning(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logWarning(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
def manageFilesourcePartitions: Boolean
def maxBatchesToRetainInMemory: Int
def maxCaseBranchesForCodegen: Int
def metastorePartitionPruning: Boolean
def minBatchesToRetain: Int
def minNumPostShufflePartitions: Int
def ndvMaxError: Double
final def ne(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def notify(): Unit

Definition Classes
AnyRef
final def notifyAll(): Unit

Definition Classes
AnyRef
def numShufflePartitions: Int
def optimizerInSetConversionThreshold: Int
def optimizerMaxIterations: Int

************************ Spark SQL Params/Hints *******************
def optimizerMetadataOnly: Boolean
def orcFilterPushDown: Boolean
def orderByOrdinal: Boolean
def parallelPartitionDiscoveryParallelism: Int
def parallelPartitionDiscoveryThreshold: Int
def parquetCacheMetadata: Boolean
def parquetCompressionCodec: String
def parquetFilterPushDown: Boolean
def parquetOutputCommitterClass: String
def parquetVectorizedReaderEnabled: Boolean
def partitionColumnTypeInferenceEnabled: Boolean
def preferSortMergeJoin: Boolean
def resolver: Resolver

Returns the Resolver for the current configuration, which can be used to determine if two identifiers are equal.
def runSQLonFile: Boolean
def setConf[T](entry: ConfigEntry[T], value: T): Unit

Set the given Spark SQL configuration property.
def setConf(props: Properties): Unit

Set Spark SQL configuration properties.
def setConfString(key: String, value: String): Unit

Set the given Spark SQL configuration property using a string value.
val settings: Map[String, String]

Only low degree of contention is expected for conf, thus NOT using ConcurrentHashMap.
Only low degree of contention is expected for conf, thus NOT using ConcurrentHashMap.

Attributes
protected[org.apache.spark]
def sortBeforeRepartition: Boolean
def stateStoreMinDeltasForSnapshot: Int
def streamingFileCommitProtocolClass: String
def streamingMetricsEnabled: Boolean
def streamingNoDataProgressEventInterval: Long
def streamingPollingDelay: Long
def streamingProgressRetention: Int
def streamingSchemaInference: Boolean
def streamingUIRunningQueriesDisplayLimit: Int
def streamingUITrendsMaxSampleSize: Int
def subexpressionEliminationEnabled: Boolean
final def synchronized[T0](arg0: ⇒ T0): T0

Definition Classes
AnyRef
def targetPostShuffleInputSize: Long
def toString(): String

Definition Classes
AnyRef → Any
def unsetConf(entry: ConfigEntry[_]): Unit
def unsetConf(key: String): Unit
def useCompression: Boolean
def variableSubstituteDepth: Int
def variableSubstituteEnabled: Boolean
def verifyPartitionPath: Boolean
final def wait(): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long, arg1: Int): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
def warehousePath: String
def wholeStageEnabled: Boolean
def wholeStageFallback: Boolean
def wholeStageMaxNumFields: Int
def writeLegacyParquetFormat: Boolean

Related Docs: object SQLConf | package internal

class SQLConf extends Serializable with internal.Logging

Instance Constructors

new SQLConf()

Value Members

final def !=(arg0: Any): Boolean

final def ##(): Int

final def ==(arg0: Any): Boolean

def adaptiveExecutionEnabled: Boolean

final def asInstanceOf[T0]: T0

def autoBroadcastJoinThreshold: Long

def broadcastTimeout: Int

def bucketingEnabled: Boolean

def caseSensitiveAnalysis: Boolean

def caseSensitiveInferenceMode: SQLConf.HiveCaseSensitiveInferenceMode.Value

def checkpointLocation: Option[String]

def clear(): Unit

def clone(): AnyRef

def columnBatchSize: Int

def columnNameOfCorruptRecord: String

def constraintPropagationEnabled: Boolean

def contains(key: String): Boolean

def convertCTAS: Boolean

def crossJoinEnabled: Boolean

def dataFramePivotMaxValues: Int

def dataFrameRetainGroupColumns: Boolean

def dataFrameSelfJoinAutoResolveAmbiguity: Boolean

def defaultDataSourceName: String

def defaultSizeInBytes: Long

def enableRadixSort: Boolean

def enableTwoLevelAggMap: Boolean

final def eq(arg0: AnyRef): Boolean

def equals(arg0: Any): Boolean

def exchangeReuseEnabled: Boolean

def fallBackToHdfsForStatsEnabled: Boolean

def fileCommitProtocolClass: String

def fileSinkLogCleanupDelay: Long

def fileSinkLogCompactInterval: Int

def fileSinkLogDeletion: Boolean

def fileSourceLogCleanupDelay: Long

def fileSourceLogCompactInterval: Int

def fileSourceLogDeletion: Boolean

def filesMaxPartitionBytes: Long

def filesOpenCostInBytes: Long

def filesourcePartitionFileCacheSize: Long

def finalize(): Unit

def gatherFastStats: Boolean

def getAllConfs: Map[String, String]

def getAllDefinedConfs: Seq[(String, String, String)]

final def getClass(): Class[_]

def getConf[T](entry: OptionalConfigEntry[T]): Option[T]

def getConf[T](entry: ConfigEntry[T]): T

def getConf[T](entry: ConfigEntry[T], defaultValue: T): T

def getConfString(key: String, defaultValue: String): String

def getConfString(key: String): String

def groupByOrdinal: Boolean

def hashCode(): Int

def ignoreCorruptFiles: Boolean

def inMemoryPartitionPruning: Boolean

def initializeLogIfNecessary(isInterpreter: Boolean, silent: Boolean = false): Boolean

def initializeLogIfNecessary(isInterpreter: Boolean): Unit

final def isInstanceOf[T0]: Boolean

def isParquetBinaryAsString: Boolean

def isParquetINT96AsTimestamp: Boolean

def isParquetSchemaMergingEnabled: Boolean

def isParquetSchemaRespectSummaries: Boolean

def isTraceEnabled(): Boolean

def isUnsupportedOperationCheckEnabled: Boolean

def limitScaleUpFactor: Int

def log: Logger

def logDebug(msg: ⇒ String, throwable: Throwable): Unit

def logDebug(msg: ⇒ String): Unit

def logError(msg: ⇒ String, throwable: Throwable): Unit

def logError(msg: ⇒ String): Unit

def logInfo(msg: ⇒ String, throwable: Throwable): Unit

def logInfo(msg: ⇒ String): Unit

def logName: String

def logTrace(msg: ⇒ String, throwable: Throwable): Unit

def logTrace(msg: ⇒ String): Unit