Skip to content

Commit

Permalink
[GH-82] Default disk size to 6 GB.
Browse files Browse the repository at this point in the history 
Default the disk size to 6 GB to align with old behavior.

Add tests for retrieving the disk size.
  • Loading branch information
@jealous
jealous committed Sep 9, 2024
1 parent eefc99e commit 4a7eca8
Show file tree
Hide file tree
Showing 4 changed files with 60 additions and 19 deletions.
1 change: 1 addition & 0 deletions plugins/nf-float/src/main/com/memverge/nextflow/FloatConf.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ class FloatConf {
static final String NF_TASK_NAME = 'nextflow-io-task-name'
static final String FLOAT_INPUT_SIZE = 'input-size'
static final String FLOAT_JOB_KIND = 'job-kind'
static final long MIN_VOL_SIZE = 6
static final int DFT_MAX_CPU_FACTOR = 4
static final int DFT_MAX_MEM_FACTOR = 4

Expand Down
13 changes: 7 additions & 6 deletions plugins/nf-float/src/main/com/memverge/nextflow/FloatGridExecutor.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,6 @@ import java.nio.file.StandardCopyOption
class FloatGridExecutor extends AbstractGridExecutor {
private static final int DFT_MEM_GB = 1
private static final long FUSION_MIN_VOL_SIZE = 80
private static final long MIN_VOL_SIZE = 40
private static final long DISK_INPUT_FACTOR = 5

private FloatJobs _floatJobs
Expand Down Expand Up @@ -403,8 +402,8 @@ class FloatGridExecutor extends AbstractGridExecutor {
return cmd
}

private void addVolSize(List<String> cmd, TaskRun task) {
long size = MIN_VOL_SIZE
void addVolSize(List<String> cmd, TaskRun task) {
long size = FloatConf.MIN_VOL_SIZE

def disk = task.config.getDisk()
if (disk) {
Expand All @@ -413,9 +412,11 @@ class FloatGridExecutor extends AbstractGridExecutor {
if (isFusionEnabled()) {
size = Math.max(size, FUSION_MIN_VOL_SIZE)
}
long inputSizeGB = (long)(getInputFileSize(task) / 1024 / 1024 / 1024)
long minDiskSizeBasedOnInput = inputSizeGB * DISK_INPUT_FACTOR
size = Math.max(size, minDiskSizeBasedOnInput)
if (task.scratch) {
double inputSizeGB = (double)(getInputFileSize(task)) / 1024 / 1024 / 1024
long minDiskSizeBasedOnInput = Math.round(inputSizeGB * DISK_INPUT_FACTOR)
size = Math.max(size, minDiskSizeBasedOnInput)
}
cmd << '--imageVolSize' << size.toString()
}

Expand Down
2 changes: 1 addition & 1 deletion plugins/nf-float/src/test/com/memverge/nextflow/FloatBaseTest.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -122,6 +122,6 @@ class FloatBaseTest extends BaseTest {
'--customTag', "${FloatConf.NF_TASK_NAME}:foo--$taskIDStr-",
'--customTag', "${FloatConf.FLOAT_INPUT_SIZE}:0",
'--customTag', "${FloatConf.NF_RUN_NAME}:test-run",
'--imageVolSize', '40']
'--imageVolSize', FloatConf.MIN_VOL_SIZE]
}
}
63 changes: 51 additions & 12 deletions plugins/nf-float/src/test/com/memverge/nextflow/FloatGridExecutorTest.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@ class FloatGridExecutorTest extends FloatBaseTest {
given:
final exec = newTestExecutor()
exec.floatJobs.updateJob(FloatJob.parse(
"""
"""
id: a
customTags:
nf-job-id: tJob-1
Expand Down Expand Up @@ -211,7 +211,7 @@ class FloatGridExecutorTest extends FloatBaseTest {
'10']

then:
cmd == expected
cmd.join(' ') == expected.join(' ')
}

def "add specific extras"() {
Expand Down Expand Up @@ -421,10 +421,10 @@ class FloatGridExecutorTest extends FloatBaseTest {
def "use time directive"() {
given:
final exec = newTestExecutor([
float: [address: addr,
username: user,
password: pass,
nfs: nfs,
float: [address : addr,
username : user,
password : pass,
nfs : nfs,
ignoreTimeFactor: false]
])
final task = newTask(exec, 0, new TaskConfig(
Expand All @@ -442,12 +442,12 @@ class FloatGridExecutorTest extends FloatBaseTest {
def "use timeout factor"() {
given:
final exec = newTestExecutor(
[float: [address : addr,
username : user,
password : pass,
nfs : nfs,
[float: [address : addr,
username : user,
password : pass,
nfs : nfs,
ignoreTimeFactor: false,
timeFactor: 1.1]])
timeFactor : 1.1]])
final task = newTask(exec, 0, new TaskConfig(
container: image,
time: '1h',
Expand Down Expand Up @@ -629,7 +629,7 @@ class FloatGridExecutorTest extends FloatBaseTest {
when:
final count = taskMap.size()
// update a list of the task status
for ( def entry : taskMap.entrySet()) {
for (def entry : taskMap.entrySet()) {
exec.floatJobs.updateJob(FloatJob.parse(taskStatus(entry.key, entry.value)))
}
def res = exec.parseQueueStatus("")
Expand Down Expand Up @@ -692,4 +692,43 @@ class FloatGridExecutorTest extends FloatBaseTest {
setEnv('MMC_USERNAME')
setEnv('MMC_PASSWORD')
}

def "get disk size when scratch is not enabled"() {
given:
final exec = newTestExecutor()
final task = newTask(exec, 0)

when:
List<String> cmd = []
exec.addVolSize(cmd, task)

then:
cmd.join(' ') == "--imageVolSize ${FloatConf.MIN_VOL_SIZE}"
}

def "get disk size when specified in task"() {
given:
final exec = newTestExecutor()
final task = newTask(exec, 0, new TaskConfig(disk: '18.GB'))

when:
List<String> cmd = []
exec.addVolSize(cmd, task)

then:
cmd.join(' ') == "--imageVolSize 18"
}

def "get disk size when specified size is smaller than min"() {
given:
final exec = newTestExecutor()
final task = newTask(exec, 0, new TaskConfig(disk: '2.GB'))

when:
List<String> cmd = []
exec.addVolSize(cmd, task)

then:
cmd.join(' ') == "--imageVolSize ${FloatConf.MIN_VOL_SIZE}"
}
}

0 comments on commit 4a7eca8

Please sign in to comment.