Spark源码解读(二)

二、通信环境的建立

private[spark] def createSparkEnv(...): SparkEnv = {
  SparkEnv.createDriverEnv(conf, isLocal, listenerBus, SparkContext.numDriverCores(master, conf))
}
----------------------------------------------------------
private[spark] def createDriverEnv(...): SparkEnv = {
    ...
  create(
    conf,
    SparkContext.DRIVER_IDENTIFIER,
    bindAddress,
    advertiseAddress,
    Option(port),
    isLocal,
    numCores,
    ioEncryptionKey,
    listenerBus = listenerBus,
    mockOutputCommitCoordinator = mockOutputCommitCoordinator
  )
}
----------------------------------------------------------
private def create(...): SparkEnv = {
  ...
  // 创建env
  val rpcEnv = RpcEnv.create(systemName, bindAddress, advertiseAddress, port.getOrElse(-1), conf,securityManager, numUsableCores, !isDriver)
}
----------------------------------------------------------
def create(...): RpcEnv = {
  val config = RpcEnvConfig(conf, name, bindAddress, advertiseAddress, port, securityManager,numUsableCores, clientMode)
  new NettyRpcEnvFactory().create(config)  // 通过netty创建
}
----------------------------------------------------------
private[rpc] class NettyRpcEnvFactory extends RpcEnvFactory with Logging {
  def create(config: RpcEnvConfig): RpcEnv = {
    ...
    val startNettyRpcEnv: Int => (NettyRpcEnv, Int) = { actualPort =>
      nettyEnv.startServer(config.bindAddress, actualPort)  // 实际上在这里启动
      (nettyEnv, nettyEnv.address.port)
    }
      try {
        //  在端口号上启动服务，将startNettyRpcEnv函数传进去
        Utils.startServiceOnPort(config.port, startNettyRpcEnv, sparkConf, config.name)._1
      } 
  }
}
----------------------------------------------------------
def startServiceOnPort[T](...): (T, Int) = {
  ...
  try {
    val (service, port) = startService(tryPort) // 启动服务,startService是外面传进来的参数
    return (service, port)
  }
  ...
}
----------------------------------------------------------
def startServer(bindAddress: String, port: Int): Unit = {
  ...
  server = transportContext.createServer(bindAddress, port, bootstraps) // 1. 启动服务
  dispatcher.registerRpcEndpoint(
    RpcEndpointVerifier.NAME, new RpcEndpointVerifier(this, dispatcher))  // 注册终端
}
----------------------------------------------------------
1. 启动服务

public TransportServer createServer(String host, int port, List<TransportServerBootstrap> bootstraps) {
    // 实际上是创建了TransportServer
    return new TransportServer(this, host, port, this.rpcHandler, bootstraps);
}
----------------------------------------------------------
public TransportServer(...) {
  ...
  try {
    init(hostToBind, portToBind);  // 实际上在这里进行初始化
  } 
}
----------------------------------------------------------
private void init(String hostToBind, int portToBind) {
   IOMode ioMode = IOMode.valueOf(conf.ioMode());  // 获取IO模式
   bootstrap = new ServerBootstrap()
     .group(bossGroup, workerGroup)
     .channel(NettyUtils.getServerChannelClass(ioMode))  // 1.1 模式匹配类型
     .option(ChannelOption.ALLOCATOR, pooledAllocator)
     .option(ChannelOption.SO_REUSEADDR, !SystemUtils.IS_OS_WINDOWS)
     .childOption(ChannelOption.ALLOCATOR, pooledAllocator);
}
----------------------------------------------------------
1.1 模式匹配类型

public static Class<? extends ServerChannel> getServerChannelClass(IOMode mode) {
  switch (mode) {
    case NIO: return NioServerSocketChannel.class;
    case EPOLL: return EpollServerSocketChannel.class; // 采用的是epoll模式
    default: throw new IllegalArgumentException("Unknown io mode: " + mode);
  }
}

RpcEndpoint 用于接收数据，里面包含一个收件箱(InBox)。

RpcEndpointRef 用于发送数据，里面包含一个发件箱(OutBox)，OutBox有多个，和地址绑定。

这里主要是driver端的通信环境建立过程，和executor端的流程一样。

三、RDD依赖

new MapPartitionsRDD[U, T](this, (_, _, iter) => iter.map(cleanF)) // this就是当前调用的对象
----------------------------------------------------
private[spark] class MapPartitionsRDD[U: ClassTag, T: ClassTag](...)
  extends RDD[U](prev) {}
----------------------------------------------------
def this(@transient oneParent: RDD[_]) =
    this(oneParent.context, List(new OneToOneDependency(oneParent)))  // OneToOneDependency建立了学院关系
----------------------------------------------------
class OneToOneDependency[T](rdd: RDD[T]) extends NarrowDependency[T](rdd) {
  override def getParents(partitionId: Int): List[Int] = List(partitionId)
}
----------------------------------------------------
abstract class NarrowDependency[T](_rdd: RDD[T]) extends Dependency[T] {
  // 将RDD进行赋值
  override def rdd: RDD[T] = _rdd  
}

四、任务的调度

在划分完任务以后就会执行任务的调度，首先会将所有的任务转换成任务集的形式。

if (tasks.nonEmpty) {
    taskScheduler.submitTasks(new TaskSet(tasks.toArray, stage.id, stage.latestInfo.attemptNumber, jobId, properties,stage.resourceProfileId))  // 提交任务的调度
}
-----------------------------------------------
private[spark] class TaskSchedulerImpl(...)extends TaskScheduler with Logging {
  override def submitTasks(taskSet: TaskSet): Unit = {
    val tasks = taskSet.tasks
    this.synchronized {
      val manager = createTaskSetManager(taskSet, maxTaskFailures)  // 1. 创建任务
      ...
      // 2.1 任务的调度模式
      // 2.2 任务的调度
      schedulableBuilder.addTaskSetManager(manager, manager.taskSet.properties) 
      ...
    }
    backend.reviveOffers()  // 3. 获取任务
  }
}
------------------------------------------------------------
1. 创建任务

private[scheduler] def createTaskSetManager(...): TaskSetManager = {
  // 又在taskSet外面包装了一层
  new TaskSetManager(this, taskSet, maxTaskFailures, healthTrackerOpt, clock)
}
------------------------------------------------------------
2.1 任务的调度模式

def initialize(backend: SchedulerBackend): Unit = {
  this.backend = backend
   schedulableBuilder = {  // schedulingMode 来自配置文件默认是FIFO
    schedulingMode match {
      case SchedulingMode.FIFO => new FIFOSchedulableBuilder(rootPool)
      case SchedulingMode.FAIR => new FairSchedulableBuilder(rootPool, conf)
      ...
    }
  }
  schedulableBuilder.buildPools()
}
------------------------------------------------------------
2.2 任务的调度，由于默认的是FIFO，所以先看FIFO

private[spark] class FIFOSchedulableBuilder(val rootPool: Pool)extends SchedulableBuilder with Logging {
  override def addTaskSetManager(manager: Schedulable, properties: Properties): Unit = {
    rootPool.addSchedulable(manager)  // 添加调度，将任务添加到任务池中
  }
}
------------------------------------------------------------
3. 获取任务，有两种不同的实现，主要看集群的

override def reviveOffers(): Unit = Utils.tryLogNonFatalError {
  driverEndpoint.send(ReviveOffers)  // 发送消息
}
------------------------------------------------------------
override def receive: PartialFunction[Any, Unit] = {
  ...
  case ReviveOffers => makeOffers()
  ...
}
------------------------------------------------------------
private def makeOffers(): Unit = {
  val taskDescs = withLock {
    val activeExecutors = executorDataMap.filterKeys(isExecutorActive)
       val workOffers = activeExecutors.map {
        case (id, executorData) =>
          new WorkerOffer(id, executorData.executorHost, executorData.freeCores,
            Some(executorData.executorAddress.hostPort),
            executorData.resourcesInfo.map { case (rName, rInfo) =>
              (rName, rInfo.availableAddrs.toBuffer)
            }, executorData.resourceProfileId)
      }.toIndexedSeq
      scheduler.resourceOffers(workOffers, true)  // 3.1 获取任务
    }
  if (taskDescs.nonEmpty) {
    launchTasks(taskDescs) // 3.2 如果任务不为空就启动任务
  }
}
------------------------------------------------------------
3.1 获取任务

def resourceOffers(...): Seq[Seq[TaskDescription]] = synchronized {
  ...
  val sortedTaskSets = rootPool.getSortedTaskSetQueue // 3.1.1 获取任务队列

  for (taskSet <- sortedTaskSets) {
    // 本地化级别，发送任务到哪里执行
    for (currentMaxLocality <- taskSet.myLocalityLevels) {
        var launchedTaskAtCurrentMaxLocality = false
        do {
          val (noDelayScheduleReject, minLocality) = resourceOfferSingleTaskSet(
            taskSet, currentMaxLocality, shuffledOffers, availableCpus,
            availableResources, tasks, addressesWithDescs)
          launchedTaskAtCurrentMaxLocality = minLocality.isDefined
          launchedAnyTask |= launchedTaskAtCurrentMaxLocality
          noDelaySchedulingRejects &= noDelayScheduleReject
          globalMinLocality = minTaskLocality(globalMinLocality, minLocality)
        } while (launchedTaskAtCurrentMaxLocality)
      }
  }
  ...
}
------------------------------------------------------------
3.1.1 获取任务队列

override def getSortedTaskSetQueue: ArrayBuffer[TaskSetManager] = {
  val sortedTaskSetQueue = new ArrayBuffer[TaskSetManager]
  // 3.1.1.1 调度任务的算法
  val sortedSchedulableQueue =
    schedulableQueue.asScala.toSeq.sortWith(taskSetSchedulingAlgorithm.comparator)
  for (schedulable <- sortedSchedulableQueue) {
    sortedTaskSetQueue ++= schedulable.getSortedTaskSetQueue.filter(_.isSchedulable)
  }
  sortedTaskSetQueue
}
------------------------------------------------------------
3.1.1.1 调度任务的算法

private val taskSetSchedulingAlgorithm: SchedulingAlgorithm = {
  schedulingMode match {
    case SchedulingMode.FAIR =>new FairSchedulingAlgorithm()
    case SchedulingMode.FIFO =>new FIFOSchedulingAlgorithm() // 匹配到FIFO模式进行调度计算
  }
}

private[spark] class FIFOSchedulingAlgorithm extends SchedulingAlgorithm {
  override def comparator(s1: Schedulable, s2: Schedulable): Boolean = {
    val priority1 = s1.priority
    val priority2 = s2.priority
    var res = math.signum(priority1 - priority2)
    if (res == 0) {
      val stageId1 = s1.stageId
      val stageId2 = s2.stageId
      res = math.signum(stageId1 - stageId2)
    }
    res < 0
  }
}
--------------------------------------------------------------
3.2 启动任务

private def launchTasks(tasks: Seq[Seq[TaskDescription]]): Unit = {
  ...
  executorData.executorEndpoint.send(LaunchTask(new SerializableBuffer(serializedTask)))
}

本地化级别就是数据和计算在不同位置上的级别：

• 进程本地化：数据和计算都在同一个进程中，效率是最高的。
• 节点本地化：数据和计算都在同一个节点中。
• 机架本地化：数据和计算都在同一个机架上。
• 任意

五、任务的执行

通过上面已经将任务处理完毕，之后该执行任务。

Executor会接受任务

override def receive: PartialFunction[Any, Unit] = {
  case LaunchTask(data) =>
     ...
     val taskDesc = TaskDescription.decode(data.value)
     taskResources(taskDesc.taskId) = taskDesc.resources
     executor.launchTask(this, taskDesc)  // executor去执行任务
     ... 
}
---------------------------------------------------------
def launchTask(context: ExecutorBackend, taskDescription: TaskDescription): Unit = {
  val tr = new TaskRunner(context, taskDescription, plugins) // 每个TaskRunner会实现Runnable
  runningTasks.put(taskDescription.taskId, tr)
  threadPool.execute(tr)   // 通过线程池的方式执行
}
---------------------------------------------------------
会执行每个Runner的run方法

override def run(): Unit = {
  ...
  val value = Utils.tryWithSafeFinally {
    val res = task.run(     // task开始运行
    taskAttemptId = taskId,
    attemptNumber = taskDescription.attemptNumber,
    metricsSystem = env.metricsSystem,
    resources = taskDescription.resources,
    plugins = plugins)
    threwException = false
    res
   }       
  ...
}
----------------------------------------------------------
final def run(): T = {
  runTask(context)  // 实际上执行每个executor的逻辑
}