object NetworkManager extends Serializable
- Alphabetic
- By Inheritance
- NetworkManager
- Serializable
- Serializable
- AnyRef
- Any
- Hide All
- Show All
- Public
- All
Value Members
-
final
def
!=(arg0: Any): Boolean
- Definition Classes
- AnyRef → Any
-
final
def
##(): Int
- Definition Classes
- AnyRef → Any
-
final
def
==(arg0: Any): Boolean
- Definition Classes
- AnyRef → Any
-
final
def
asInstanceOf[T0]: T0
- Definition Classes
- Any
-
def
clone(): AnyRef
- Attributes
- protected[lang]
- Definition Classes
- AnyRef
- Annotations
- @throws( ... ) @native()
-
def
create(numTasks: Int, spark: SparkSession, driverListenPort: Int, timeout: Double, useBarrierExecutionMode: Boolean): NetworkManager
Create a NetworkManager, which will encapsulate all network operations.
Create a NetworkManager, which will encapsulate all network operations. This method will opens a socket communications channel on the driver, and then initialize the network manager itself. The NetworkManager object will start a thread that waits for the host:port from the executors, and then sends back the information to the executors.
- numTasks
The total number of training tasks to wait for.
- spark
The Spark session.
- driverListenPort
The port to listen for the driver on.
- timeout
The timeout (in seconds).
- useBarrierExecutionMode
Whether to use barrier mode.
- returns
The NetworkTopology.
-
final
def
eq(arg0: AnyRef): Boolean
- Definition Classes
- AnyRef
-
def
equals(arg0: Any): Boolean
- Definition Classes
- AnyRef → Any
-
def
finalize(): Unit
- Attributes
- protected[lang]
- Definition Classes
- AnyRef
- Annotations
- @throws( classOf[java.lang.Throwable] )
-
final
def
getClass(): Class[_]
- Definition Classes
- AnyRef → Any
- Annotations
- @native()
-
def
getGlobalNetworkInfo(ctx: TrainingContext, log: Logger, taskId: Long, partitionId: Int, shouldExecuteTraining: Boolean, measures: TaskInstrumentationMeasures): NetworkTopologyInfo
Retrieve the network nodes and current port information from the driver.
Retrieve the network nodes and current port information from the driver.
Establish local socket connection.
Note: Ideally we would start the socket connections in the C layer, this opens us up for race conditions in case other applications open sockets on cluster, but usually this should not be a problem
- ctx
Information about the current training session.
- log
The Logger.
- taskId
The task id.
- partitionId
The partition id.
- shouldExecuteTraining
Whether this task should be a part of the training network.
- measures
Instrumentation for perf measurements.
- returns
Information about the network topology.
-
def
getMainWorkerPort(nodes: String, log: Logger): Int
Gets the main node's port that will return the LightGBM Booster.
Gets the main node's port that will return the LightGBM Booster. Used to minimize network communication overhead in reduce step.
- returns
The main node's port number.
-
def
hashCode(): Int
- Definition Classes
- AnyRef → Any
- Annotations
- @native()
- def initLightGBMNetwork(ctx: PartitionTaskContext, log: Logger, retry: Int = LightGBMConstants.NetworkRetries, delay: Long = LightGBMConstants.InitialDelay): Unit
-
final
def
isInstanceOf[T0]: Boolean
- Definition Classes
- Any
-
final
def
ne(arg0: AnyRef): Boolean
- Definition Classes
- AnyRef
-
final
def
notify(): Unit
- Definition Classes
- AnyRef
- Annotations
- @native()
-
final
def
notifyAll(): Unit
- Definition Classes
- AnyRef
- Annotations
- @native()
- def parseWorkerMessage(message: String): TaskMessageInfo
-
final
def
synchronized[T0](arg0: ⇒ T0): T0
- Definition Classes
- AnyRef
-
def
toString(): String
- Definition Classes
- AnyRef → Any
-
final
def
wait(): Unit
- Definition Classes
- AnyRef
- Annotations
- @throws( ... )
-
final
def
wait(arg0: Long, arg1: Int): Unit
- Definition Classes
- AnyRef
- Annotations
- @throws( ... )
-
final
def
wait(arg0: Long): Unit
- Definition Classes
- AnyRef
- Annotations
- @throws( ... ) @native()