From 53e0600b1fce9b13d8747a6037d5f0f0e2f8024c Mon Sep 17 00:00:00 2001 From: Akash Mishra Date: Wed, 16 Dec 2015 11:06:17 +0100 Subject: [PATCH] [SPARK-10618] [Mesos] removing meetconstraints check as it is already done --- .../mesos/CoarseMesosSchedulerBackend.scala | 8 +++----- .../CoarseMesosSchedulerBackendSuite.scala | 17 ++++++----------- 2 files changed, 9 insertions(+), 16 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/CoarseMesosSchedulerBackend.scala b/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/CoarseMesosSchedulerBackend.scala index 3cf296b7cb021..287830e007b02 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/CoarseMesosSchedulerBackend.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/CoarseMesosSchedulerBackend.scala @@ -254,7 +254,7 @@ private[spark] class CoarseMesosSchedulerBackend( val cpus = getResource(offer.getResourcesList, "cpus").toInt val id = offer.getId.getValue if (meetsConstraints) { - if (isOfferSatisfiesRequirements(meetsConstraints, slaveId, mem, cpus, sc)) { + if (isOfferSatisfiesRequirements(slaveId, mem, cpus, sc)) { // Launch an executor on the slave val cpusToUse = math.min(cpus, maxCores - totalCoresAcquired) totalCoresAcquired += cpusToUse @@ -304,9 +304,8 @@ private[spark] class CoarseMesosSchedulerBackend( } // ToDo: Abstract out each condition and log them. - def isOfferSatisfiesRequirements(meetsConstraints: Boolean, - slaveId: String, mem: Double, - cpusOffered: Int, sc: SparkContext): Boolean = { + def isOfferSatisfiesRequirements(slaveId: String, mem: Double, cpusOffered: Int, + sc: SparkContext): Boolean = { val meetsMemoryRequirements = mem >= calculateTotalMemory(sc) val meetsCPURequirements = cpusOffered >= 1 val needMoreCores = totalCoresAcquired < maxCores @@ -317,7 +316,6 @@ private[spark] class CoarseMesosSchedulerBackend( executorNotRunningOnSlave && taskOnEachSlaveLessThanExecutorLimit && needMoreCores && - meetsConstraints && meetsMemoryRequirements && meetsCPURequirements && healthySlave diff --git a/core/src/test/scala/org/apache/spark/scheduler/cluster/mesos/CoarseMesosSchedulerBackendSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/cluster/mesos/CoarseMesosSchedulerBackendSuite.scala index c9bcfce8d846c..f6517c9090415 100644 --- a/core/src/test/scala/org/apache/spark/scheduler/cluster/mesos/CoarseMesosSchedulerBackendSuite.scala +++ b/core/src/test/scala/org/apache/spark/scheduler/cluster/mesos/CoarseMesosSchedulerBackendSuite.scala @@ -197,25 +197,20 @@ class CoarseMesosSchedulerBackendSuite extends SparkFunSuite test("isOfferSatisfiesRequirements return true when there is a valid offer") { val schedulerBackend = createSchedulerBackendForGivenSparkConf(sc) - assert(schedulerBackend.isOfferSatisfiesRequirements(true, "Slave1", 10000, 5, sc)) + assert(schedulerBackend.isOfferSatisfiesRequirements("Slave1", 10000, 5, sc)) } - test("isOfferSatisfiesRequirements return false when offer do not meet constraints") { - val schedulerBackend = createSchedulerBackendForGivenSparkConf(sc) - - assert(schedulerBackend.isOfferSatisfiesRequirements(false, "Slave1", 10000, 5, sc) === false) - } test("isOfferSatisfiesRequirements return false when memory in offer is less than required memory") { val schedulerBackend = createSchedulerBackendForGivenSparkConf(sc) - assert(schedulerBackend.isOfferSatisfiesRequirements(true, "Slave1", 1, 5, sc) === false) + assert(schedulerBackend.isOfferSatisfiesRequirements("Slave1", 1, 5, sc) === false) } test("isOfferSatisfiesRequirements return false when cpu in offer is less than required cpu") { val schedulerBackend = createSchedulerBackendForGivenSparkConf(sc) - assert(schedulerBackend.isOfferSatisfiesRequirements(true, "Slave1", 10000, 0, sc) === false) + assert(schedulerBackend.isOfferSatisfiesRequirements("Slave1", 10000, 0, sc) === false) } test("isOfferSatisfiesRequirements return false when offer is from slave already running" + @@ -223,7 +218,7 @@ class CoarseMesosSchedulerBackendSuite extends SparkFunSuite val schedulerBackend = createSchedulerBackendForGivenSparkConf(sc) schedulerBackend.slaveIdsWithExecutors += "Slave2" - assert(schedulerBackend.isOfferSatisfiesRequirements(true, "Slave2", 10000, 5, sc) === false) + assert(schedulerBackend.isOfferSatisfiesRequirements("Slave2", 10000, 5, sc) === false) } test("isOfferSatisfiesRequirements return false when task is failed more than " + @@ -231,13 +226,13 @@ class CoarseMesosSchedulerBackendSuite extends SparkFunSuite val schedulerBackend = createSchedulerBackendForGivenSparkConf(sc) schedulerBackend.failuresBySlaveId("Slave3") = 2 - assert(schedulerBackend.isOfferSatisfiesRequirements(true, "Slave3", 10000, 5, sc) === false) + assert(schedulerBackend.isOfferSatisfiesRequirements("Slave3", 10000, 5, sc) === false) } test("isOfferSatisfiesRequirements return false when max core is already acquired") { val schedulerBackend = createSchedulerBackendForGivenSparkConf(sc) schedulerBackend.totalCoresAcquired = 10 - assert(schedulerBackend.isOfferSatisfiesRequirements(true, "Slave1", 10000, 5, sc) === false) + assert(schedulerBackend.isOfferSatisfiesRequirements("Slave1", 10000, 5, sc) === false) } }