FileStreamSource

Value Members

1. final def !=(arg0: Any): Boolean

   

   Definition Classes

   AnyRef → Any

2. final def ##(): Int

   

   Definition Classes

   AnyRef → Any

3. final def ==(arg0: Any): Boolean

   

   Definition Classes

   AnyRef → Any

4. final def asInstanceOf[T0]: T0

   

   Definition Classes

   Any

5. def clone(): AnyRef

   

   Attributes

   protected[java.lang]

   Definition Classes

   AnyRef

   Annotations

   @throws( ... )

6. def commit(end: Offset): Unit

   

   Informs the source that Spark has completed processing all data for offsets less than or equal to end and will only request offsets greater than end in the future.

   Informs the source that Spark has completed processing all data for offsets less than or equal to end and will only request offsets greater than end in the future.

   Definition Classes

   FileStreamSource → Source

7. def currentLogOffset: Long

   

   Return the latest offset in the FileStreamSourceLog

8. final def eq(arg0: AnyRef): Boolean

   

   Definition Classes

   AnyRef

9. def equals(arg0: Any): Boolean

   

   Definition Classes

   AnyRef → Any

10. def finalize(): Unit

    

    Attributes

    protected[java.lang]

    Definition Classes

    AnyRef

    Annotations

    @throws( classOf[java.lang.Throwable] )

11. def getBatch(start: Option[Offset], end: Offset): DataFrame

    

    Returns the data that is between the offsets (start, end].

    Returns the data that is between the offsets (start, end].

    Definition Classes

    FileStreamSource → Source

12. final def getClass(): Class[_]

    

    Definition Classes

    AnyRef → Any

13. def getOffset: Option[Offset]

    

    Returns the maximum available offset for this source.

    Returns the maximum available offset for this source. Returns None if this source has never received any data.

    Definition Classes

    FileStreamSource → Source

14. def hashCode(): Int

    

    Definition Classes

    AnyRef → Any

15. def initializeLogIfNecessary(isInterpreter: Boolean, silent: Boolean = false): Boolean

    

    Attributes

    protected

    Definition Classes

    Logging

16. def initializeLogIfNecessary(isInterpreter: Boolean): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

17. final def isInstanceOf[T0]: Boolean

    

    Definition Classes

    Any

18. def isTraceEnabled(): Boolean

    

    Attributes

    protected

    Definition Classes

    Logging

19. def log: Logger

    

    Attributes

    protected

    Definition Classes

    Logging

20. def logDebug(msg: ⇒ String, throwable: Throwable): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

21. def logDebug(msg: ⇒ String): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

22. def logError(msg: ⇒ String, throwable: Throwable): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

23. def logError(msg: ⇒ String): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

24. def logInfo(msg: ⇒ String, throwable: Throwable): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

25. def logInfo(msg: ⇒ String): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

26. def logName: String

    

    Attributes

    protected

    Definition Classes

    Logging

27. def logTrace(msg: ⇒ String, throwable: Throwable): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

28. def logTrace(msg: ⇒ String): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

29. def logWarning(msg: ⇒ String, throwable: Throwable): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

30. def logWarning(msg: ⇒ String): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

31. final def ne(arg0: AnyRef): Boolean

    

    Definition Classes

    AnyRef

32. final def notify(): Unit

    

    Definition Classes

    AnyRef

33. final def notifyAll(): Unit

    

    Definition Classes

    AnyRef

34. val schema: StructType

    

    Returns the schema of the data from this source

    Returns the schema of the data from this source

    Definition Classes

    FileStreamSource → Source

35. val seenFiles: SeenFilesMap

    

    A mapping from a file that we have processed to some timestamp it was last modified.

36. def stop(): Unit

    

    Stop this source and free any resources it has allocated.

    Stop this source and free any resources it has allocated.

    Definition Classes

    FileStreamSource → Source

37. final def synchronized[T0](arg0: ⇒ T0): T0

    

    Definition Classes

    AnyRef

38. def toString(): String

    

    Definition Classes

    FileStreamSource → AnyRef → Any

39. final def wait(): Unit

    

    Definition Classes

    AnyRef

    Annotations

    @throws( ... )

40. final def wait(arg0: Long, arg1: Int): Unit

    

    Definition Classes

    AnyRef

    Annotations

    @throws( ... )

41. final def wait(arg0: Long): Unit

    

    Definition Classes

    AnyRef

    Annotations

    @throws( ... )

42. def withBatchingLocked[T](func: ⇒ T): T

    

    For test only.

    For test only. Run func with the internal lock to make sure when func is running, the current offset won't be changed and no new batch will be emitted.