From 7ba5ece298355183840bc736d711d53105ee2cfe Mon Sep 17 00:00:00 2001 From: Cedric Zhuang Date: Tue, 11 Jul 2023 17:04:49 +0800 Subject: [PATCH] [GH-29] Read credentials from ENV or secrets. Enable the plugin to read credentials from the environment variable: * MMC_ADDRESS: address of the op center * MMC_USERNAME: login username * MMC_PASSWORD: login password Hide the password in the log. Add samples in the README to illustrate how to input the credentials with NextFlow secrets. --- README.md | 35 +++++++++++++ .../com/memverge/nextflow/FloatConf.groovy | 19 ++++--- .../nextflow/FloatGridExecutor.groovy | 15 ++++-- .../nextflow/FloatGridExecutorTest.groovy | 50 +++++++++++++++++++ 4 files changed, 107 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index dace569..18cf8d6 100644 --- a/README.md +++ b/README.md @@ -122,6 +122,41 @@ float { * `commonExtra` allows the user to specify other submit parameters. This parameter will be appended to every float submit command. +### Configure with environment variables + +The plugin allows the user to set credentials with environment variables. +If the credentials are not available in the configuration file, it will try +reading these environment variables. + +* `MMC_ADDRESS` for operation center address. Separate multiple addresses with `,`. +* `MMC_USERNAME` for login username +* `MMC_PASSWORD` for login password + + +### Configure with NextFlow secrets + +User can use NextFlow secrets to input the credentials. Here is an example: + +```bash +nextflow secrets set MMC_USERNAME "..." +nextflow secrets set MMC_PASSWORD "..." +``` + +In the configuration file, you can reference the secrets like this: + +```groovy +float { + username = secrets.MMC_USERNAME + password = secrets.MMC_PASSWORD +} +``` + +If the secret is not available, NextFlow reports error like this: + +``` +Unknown config secret 'MMC_USERNAME' +``` + ## Task Sample For each process, users could supply their requirements for the CPU, memory and image. diff --git a/plugins/nf-float/src/main/com/memverge/nextflow/FloatConf.groovy b/plugins/nf-float/src/main/com/memverge/nextflow/FloatConf.groovy index 5635750..086f370 100644 --- a/plugins/nf-float/src/main/com/memverge/nextflow/FloatConf.groovy +++ b/plugins/nf-float/src/main/com/memverge/nextflow/FloatConf.groovy @@ -25,6 +25,11 @@ import org.apache.commons.lang.StringUtils */ @CompileStatic class FloatConf { + static String MMC_ADDRESS = "MMC_ADDRESS" + static String MMC_USERNAME = "MMC_USERNAME" + static String MMC_PASSWORD = "MMC_PASSWORD" + static String ADDR_SEP = "," + /** credentials for op center */ String username String password @@ -66,15 +71,14 @@ class FloatConf { if (node == null) { return } - username = node.username - password = node.password + username = node.username ?: System.getenv(MMC_USERNAME) + password = node.password ?: System.getenv(MMC_PASSWORD) if (node.address instanceof Collection) { - addresses = (node.address as Collection).collect { - it.toString() - } + addresses = node.address.collect {it.toString()} } else { - addresses = node.address.toString() - .split(",") + String address = node.address ?: System.getenv(MMC_ADDRESS) ?: "" + addresses = address.toString() + .split(ADDR_SEP) .toList() .stream() .filter { it.size() > 0 } @@ -117,6 +121,7 @@ class FloatConf { } List getCliPrefix(String address = "") { + validate() if (StringUtils.length(address) == 0) { address = addresses[0] } diff --git a/plugins/nf-float/src/main/com/memverge/nextflow/FloatGridExecutor.groovy b/plugins/nf-float/src/main/com/memverge/nextflow/FloatGridExecutor.groovy index 069c43c..ac4e3e8 100644 --- a/plugins/nf-float/src/main/com/memverge/nextflow/FloatGridExecutor.groovy +++ b/plugins/nf-float/src/main/com/memverge/nextflow/FloatGridExecutor.groovy @@ -162,6 +162,11 @@ class FloatGridExecutor extends AbstractGridExecutor { return floatConf.getCliPrefix(address) } + private String toCmdString(List floatCmd) { + def ret = floatCmd.join(" ") + return ret.replace("-p ${floatConf.password}", "-p ***") + } + @Override List getSubmitCommandLine(TaskRun task, Path scriptFile) { validateTaskConf(task.config) @@ -180,7 +185,7 @@ class FloatGridExecutor extends AbstractGridExecutor { cmd << '--name' cmd << floatJobs.getJobName(task.id) cmd.addAll(getExtra(task)) - log.info "[float] submit job: ${cmd.join(' ')}" + log.info "[float] submit job: ${toCmdString(cmd)}" return cmd } @@ -212,7 +217,7 @@ class FloatGridExecutor extends AbstractGridExecutor { if (ret != 0) { def m = """\ Unable to kill pending jobs - - cmd executed: ${cmd.join(' ')} + - cmd executed: ${toCmdString(cmd)}} - exit status : $ret - output : """.stripIndent() @@ -243,7 +248,7 @@ class FloatGridExecutor extends AbstractGridExecutor { cmd << 'scancel' cmd << '-j' cmd << id - log.info "[float] cancel job: ${cmd.join(' ')}" + log.info "[float] cancel job: ${toCmdString(cmd)}" ret.add(cmd) } return ret @@ -260,7 +265,7 @@ class FloatGridExecutor extends AbstractGridExecutor { def cmd = getCmdPrefix0() cmd << 'scancel' cmd << '-j' - log.info "[float] cancel job: ${cmd.join(' ')}" + log.info "[float] cancel job: ${toCmdString(cmd)}" return cmd } @@ -298,7 +303,7 @@ class FloatGridExecutor extends AbstractGridExecutor { cmd << 'squeue' cmd << '--format' cmd << 'json' - log.info "[float] query job status: ${cmd.join(' ')}" + log.info "[float] query job status: ${toCmdString(cmd)}" return cmd } diff --git a/plugins/nf-float/src/test/com/memverge/nextflow/FloatGridExecutorTest.groovy b/plugins/nf-float/src/test/com/memverge/nextflow/FloatGridExecutorTest.groovy index 15c11fa..b80e73a 100644 --- a/plugins/nf-float/src/test/com/memverge/nextflow/FloatGridExecutorTest.groovy +++ b/plugins/nf-float/src/test/com/memverge/nextflow/FloatGridExecutorTest.groovy @@ -25,6 +25,19 @@ import spock.lang.Specification import java.nio.file.Paths class FloatGridExecutorTest extends Specification { + def setEnv(String key, String value) { + try { + def env = System.getenv(); + def cl = env.getClass(); + def field = cl.getDeclaredField("m"); + field.setAccessible(true); + def writableEnv = (Map) field.get(env); + writableEnv.put(key, value); + } catch (Exception e) { + throw new IllegalStateException("Failed to set environment variable", e); + } + } + def newTestExecutor(Map config = null) { if (config == null) { config = [float: [address : addr, @@ -512,4 +525,41 @@ class FloatGridExecutorTest extends Specification { '-p', pass, 'squeue', '--format', 'json'] } + + def "retrieve the credentials from env"() { + given: + setEnv('MMC_ADDRESS', addr) + setEnv('MMC_USERNAME', user) + setEnv('MMC_PASSWORD', pass) + def exec = newTestExecutor( + [float: [cpu: cpu, + mem: mem, + nfs: nfs]]) + + + exec.session.workDir = Paths.get(workDir) + def task = [:] as TaskRun + + when: + task.config = [image: image] as TaskConfig + task.id = taskID + def cmd = exec.getSubmitCommandLine(task, Paths.get(script)) + + then: + cmd.join(' ') == ['float', '-a', addr, + '-u', user, + '-p', pass, + 'sbatch', + '--dataVolume', nfs + ':' + workDir, + '--image', image, + '--cpu', cpu.toString(), + '--mem', mem.toString(), + '--job', script, + '--name', jobID(taskID)].join(' ') + + cleanup: + setEnv('MMC_ADDRESS', '') + setEnv('MMC_USERNAME', '') + setEnv('MMC_PASSWORD', '') + } }