-
Notifications
You must be signed in to change notification settings - Fork 8
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* collect task execution results and store them in the memory optimizer * restructured TaskScaler * cleanup debug logs * cleanup Scheduler * cleanup Task * cleanup Task * add hook after workflow is completed * change MemoryOptimizer to be an interface, add two different Optimizers * round suggestions to ceiling * introduced LinearPredictor * changed indentation to 4 spaces, like rest of the project uses * fix mis-formatting * fix typos * initial implementation linearPredictor * builder for observations * added test for constant predictor * remove wasted calculation from observation * remove wasted calculation from observation * add NonePredictorTest * sanity checks for observations * add negative case for ConstantPredictor * assert rise and fall of suggestions * added LinearPredictorTest * avoid negative preditions * use SimpleRegression for LinearPredictor * fix naming to always be prediction, instead of suggestion * fix naming * fix some minor issues * collect statistics * removed Limits, rely solely on Requests instead * added new CombiPredictor * remove solved fixme * csv export * save statistics summary and csv into file in workflow baseDir * added Tasks realtime to statistic, moved NfTrace reader to own utility class * add test if trace file is missing and handling for that * fix no resize when request was 0 * apply config only in dev profile * statistics log execution and predictor * log makespan * add peak_vmem for sanity checks * add unit tests for statistics * added todos for missing testcases * fix for-loop should continue, not break * only invoke TaskScaler when config was given * get memory predictor from config, not from environment * removed double code * prepare application.yml for merge * prepare application.yml for merge * fixed decimal seperator * fix decimal seperator * changed logging in dev profile * improved predictor selection order * added template for square predictor * collect wasted in summary * add wasted to statistics * avoid updating tasks when no new model is available * added new testcases * change return value for missing file to -1 * changed sanity check * fix constant predictor * faster overprovisioning * add wary predictor * fix imports * wary predictor * filter realtime 0 * use vmem instead of rss * correct tests * require 4 successful observations * ignore list feature * never provide predictions lower than the lowest successful value was * prevent cws from get stuck * blacklist failed tasks * removed flawed wasted from csv, added assigned node * lower limit for request size 256MiB * fix bad naming * junit test for TaskScaler * add remark for TaskScalerTest * fix used predictor * removed unimplemended square predictor * removed unused generation feature from constant predictor * removed unused generation feature * cleanup classname * removed testcase that is no longer in line with desired behaviour * fixed comments * add description to README * add description to README * catch exception that is thrown when InPlacePodVerticalScaling is not enabled * add note on profiles in README * always write log to file * check reason for exception and improve error message, then disable task scaling * fix comment * fix formatting * moved patchTaskMemory method * add tracing note in README * reduce loglevel * change predictor interface to return BigDecimal * extracted constant for lowest memory request value * add o.taskName to log, when available
- Loading branch information
1 parent
c1a30da
commit fb7ebac
Showing
26 changed files
with
3,107 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
85 changes: 85 additions & 0 deletions
85
src/main/java/cws/k8s/scheduler/memory/CombiPredictor.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,85 @@ | ||
| /* | ||
| * Copyright (c) 2023, Florian Friederici. All rights reserved. | ||
| * | ||
| * This code is free software: you can redistribute it and/or modify it under | ||
| * the terms of the GNU General Public License as published by the Free | ||
| * Software Foundation, either version 3 of the License, or (at your option) | ||
| * any later version. | ||
| * | ||
| * This code is distributed in the hope that it will be useful, but WITHOUT ANY | ||
| * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS | ||
| * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more | ||
| * details. | ||
| * | ||
| * You should have received a copy of the GNU General Public License along with | ||
| * this work. If not, see <https://www.gnu.org/licenses/>. | ||
| */ | ||
|
|
||
| package cws.k8s.scheduler.memory; | ||
|
|
||
| import java.math.BigDecimal; | ||
|
|
||
| import cws.k8s.scheduler.model.Task; | ||
| import lombok.extern.slf4j.Slf4j; | ||
|
|
||
| //@formatter:off | ||
| /** | ||
| * CombiPredictor will combine predictions made by ConstantPredictor and | ||
| * LineraPredictor. | ||
| * | ||
| * LinearPredictor fails if there are no inputSize differences to tasks, | ||
| * ConstantPredictor can handle this case. So CombiPredictor will run both and | ||
| * decide dynamically which predictions to apply. | ||
| * | ||
| * @author Florian Friederici | ||
| * | ||
| */ | ||
| //@formatter:on | ||
| @Slf4j | ||
| public class CombiPredictor implements MemoryPredictor { | ||
|
|
||
| ConstantPredictor constantPredictor; | ||
| LinearPredictor linearPredictor; | ||
|
|
||
| public CombiPredictor() { | ||
| this.constantPredictor = new ConstantPredictor(); | ||
| this.linearPredictor = new LinearPredictor(); | ||
| } | ||
|
|
||
| @Override | ||
| public void addObservation(Observation o) { | ||
| log.debug("CombiPredictor.addObservation({})", o); | ||
| constantPredictor.addObservation(o); | ||
| linearPredictor.addObservation(o); | ||
| } | ||
|
|
||
| @Override | ||
| public BigDecimal queryPrediction(Task task) { | ||
| String taskName = task.getConfig().getTask(); | ||
| log.debug("CombiPredictor.queryPrediction({},{})", taskName, task.getInputSize()); | ||
|
|
||
| BigDecimal constantPrediction = constantPredictor.queryPrediction(task); | ||
| BigDecimal linearPrediction = linearPredictor.queryPrediction(task); | ||
|
|
||
| if (constantPrediction==null && linearPrediction==null) { | ||
| // no prediction available at all | ||
| return null; | ||
| } | ||
|
|
||
| if (constantPrediction!=null && linearPrediction==null) { | ||
| // only the constantPrediction is available | ||
| return constantPrediction; | ||
| } | ||
|
|
||
| if (constantPrediction==null && linearPrediction!=null) { | ||
| // only the linearPrediction is available (unusual case) | ||
| return linearPrediction; | ||
| } | ||
|
|
||
| log.debug("constantPrediction={}, linearPrediction={}, difference={}", constantPrediction, linearPrediction, constantPrediction.subtract(linearPrediction)); | ||
|
|
||
| // prefer linearPrediction if both would be available | ||
| return linearPrediction; | ||
| } | ||
|
|
||
| } |
98 changes: 98 additions & 0 deletions
98
src/main/java/cws/k8s/scheduler/memory/ConstantPredictor.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,98 @@ | ||
| /* | ||
| * Copyright (c) 2023, Florian Friederici. All rights reserved. | ||
| * | ||
| * This code is free software: you can redistribute it and/or modify it under | ||
| * the terms of the GNU General Public License as published by the Free | ||
| * Software Foundation, either version 3 of the License, or (at your option) | ||
| * any later version. | ||
| * | ||
| * This code is distributed in the hope that it will be useful, but WITHOUT ANY | ||
| * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS | ||
| * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more | ||
| * details. | ||
| * | ||
| * You should have received a copy of the GNU General Public License along with | ||
| * this work. If not, see <https://www.gnu.org/licenses/>. | ||
| */ | ||
|
|
||
| package cws.k8s.scheduler.memory; | ||
|
|
||
| import java.math.BigDecimal; | ||
| import java.math.RoundingMode; | ||
| import java.util.HashMap; | ||
| import java.util.Map; | ||
|
|
||
| import cws.k8s.scheduler.model.Task; | ||
| import lombok.extern.slf4j.Slf4j; | ||
|
|
||
| // @formatter:off | ||
| /** | ||
| * ConstantPredictor will use the following strategy: | ||
| * | ||
| * - In case task was successful: | ||
| * - let the next prediction be 10% higher, then the peakRss was | ||
| * | ||
| * - In case task has failed: | ||
| * - reset to initial value | ||
| * | ||
| * I.e. the suggestions from ConstantPredictor are not dependent on the input | ||
| * size of the tasks. | ||
| * | ||
| * @author Florian Friederici | ||
| * | ||
| */ | ||
| // @formatter:on | ||
| @Slf4j | ||
| class ConstantPredictor implements MemoryPredictor { | ||
|
|
||
| Map<String, BigDecimal> model; | ||
| Map<String, BigDecimal> initialValue; | ||
|
|
||
| public ConstantPredictor() { | ||
| model = new HashMap<>(); | ||
| initialValue = new HashMap<>(); | ||
| } | ||
|
|
||
| @Override | ||
| public void addObservation(Observation o) { | ||
| log.debug("ConstantPredictor.addObservation({})", o); | ||
| if (!TaskScaler.checkObservationSanity(o)) { | ||
| log.warn("dismiss observation {}", o); | ||
| return; | ||
| } | ||
|
|
||
| // store initial ramRequest value per task | ||
| if (!initialValue.containsKey(o.task)) { | ||
| initialValue.put(o.task, o.getRamRequest()); | ||
| } | ||
|
|
||
| if (Boolean.TRUE.equals(o.success)) { | ||
| // set model to peakRss + 10% | ||
| if (model.containsKey(o.task)) { | ||
| model.replace(o.task, o.peakRss.multiply(new BigDecimal("1.1")).setScale(0, RoundingMode.CEILING)); | ||
| } else { | ||
| model.put(o.task, o.peakRss.multiply(new BigDecimal("1.1")).setScale(0, RoundingMode.CEILING)); | ||
| } | ||
| } else { | ||
| // reset to initialValue | ||
| if (model.containsKey(o.task)) { | ||
| model.replace(o.task, this.initialValue.get(o.task)); | ||
| } else { | ||
| model.put(o.task, o.ramRequest.multiply(new BigDecimal(2)).setScale(0, RoundingMode.CEILING)); | ||
| } | ||
| } | ||
|
|
||
| } | ||
|
|
||
| @Override | ||
| public BigDecimal queryPrediction(Task task) { | ||
| String taskName = task.getConfig().getTask(); | ||
| log.debug("ConstantPredictor.queryPrediction({})", taskName); | ||
|
|
||
| if (model.containsKey(taskName)) { | ||
| return model.get(taskName); | ||
| } else { | ||
| return null; | ||
| } | ||
| } | ||
| } |
Oops, something went wrong.