Skip to content

Commit

Permalink
[GH-29] Read credentials from ENV or secrets.
Browse files Browse the repository at this point in the history
Enable the plugin to read credentials from the environment variable:
* MMC_ADDRESS: address of the op center
* MMC_USERNAME: login username
* MMC_PASSWORD: login password

Hide the password in the log.

Add samples in the README to illustrate how to input the credentials
with NextFlow secrets.
  • Loading branch information
jealous committed Jul 11, 2023
1 parent a35634d commit 3637d78
Show file tree
Hide file tree
Showing 3 changed files with 66 additions and 7 deletions.
35 changes: 35 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -122,6 +122,41 @@ float {
* `commonExtra` allows the user to specify other submit parameters. This parameter
will be appended to every float submit command.

### Configure with environment variables

The plugin allows the user to set credentials with environment variables.
If the credentials are not available in the configuration file, it will try
reading these environment variables.

* `MMC_ADDRESS` for operation center address. Separate multiple addresses with `,`.
* `MMC_USERNAME` for login username
* `MMC_PASSWORD` for login password


### Configure with NextFlow secrets

User can use NextFlow secrets to input the credentials. Here is an example:

```bash
nextflow secrets set MMC_USERNAME "..."
nextflow secrets set MMC_PASSWORD "..."
```

In the configuration file, you can reference the secrets like this:

```groovy
float {
username = secrets.MMC_USERNAME
password = secrets.MMC_PASSWORD
}
```

If the secret is not available, NextFlow reports error like this:

```
Unknown config secret 'MMC_USERNAME'
```

## Task Sample

For each process, users could supply their requirements for the CPU, memory and image.
Expand Down
23 changes: 21 additions & 2 deletions plugins/nf-float/src/main/com/memverge/nextflow/FloatConf.groovy
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,11 @@ import org.apache.commons.lang.StringUtils
*/
@CompileStatic
class FloatConf {
static String MMC_ADDRESS = "MMC_ADDRESS"
static String MMC_USERNAME = "MMC_USERNAME"
static String MMC_PASSWORD = "MMC_PASSWORD"
static String ADDR_SEP = ","

/** credentials for op center */
String username
String password
Expand Down Expand Up @@ -67,14 +72,27 @@ class FloatConf {
return
}
username = node.username
if (StringUtils.length(username) == 0) {
username = System.getenv(MMC_USERNAME)
}
password = node.password
if (StringUtils.length(password) == 0) {
password = System.getenv(MMC_PASSWORD)
}
if (node.address instanceof Collection) {
addresses = (node.address as Collection).collect {
it.toString()
}
} else {
addresses = node.address.toString()
.split(",")
String address = node.address
if (StringUtils.length(address) == 0) {
address = System.getenv(MMC_ADDRESS)
if (address == null) {
address = ""
}
}
addresses = address.toString()
.split(ADDR_SEP)
.toList()
.stream()
.filter { it.size() > 0 }
Expand Down Expand Up @@ -117,6 +135,7 @@ class FloatConf {
}

List<String> getCliPrefix(String address = "") {
validate()
if (StringUtils.length(address) == 0) {
address = addresses[0]
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -162,6 +162,11 @@ class FloatGridExecutor extends AbstractGridExecutor {
return floatConf.getCliPrefix(address)
}

private String toCmdString(List<String> floatCmd) {
def ret = floatCmd.join(" ")
return ret.replace("-p ${floatConf.password}", "-p ***")
}

@Override
List<String> getSubmitCommandLine(TaskRun task, Path scriptFile) {
validateTaskConf(task.config)
Expand All @@ -180,7 +185,7 @@ class FloatGridExecutor extends AbstractGridExecutor {
cmd << '--name'
cmd << floatJobs.getJobName(task.id)
cmd.addAll(getExtra(task))
log.info "[float] submit job: ${cmd.join(' ')}"
log.info "[float] submit job: ${toCmdString(cmd)}"
return cmd
}

Expand Down Expand Up @@ -212,7 +217,7 @@ class FloatGridExecutor extends AbstractGridExecutor {
if (ret != 0) {
def m = """\
Unable to kill pending jobs
- cmd executed: ${cmd.join(' ')}
- cmd executed: ${toCmdString(cmd)}}
- exit status : $ret
- output :
""".stripIndent()
Expand Down Expand Up @@ -243,7 +248,7 @@ class FloatGridExecutor extends AbstractGridExecutor {
cmd << 'scancel'
cmd << '-j'
cmd << id
log.info "[float] cancel job: ${cmd.join(' ')}"
log.info "[float] cancel job: ${toCmdString(cmd)}"
ret.add(cmd)
}
return ret
Expand All @@ -260,7 +265,7 @@ class FloatGridExecutor extends AbstractGridExecutor {
def cmd = getCmdPrefix0()
cmd << 'scancel'
cmd << '-j'
log.info "[float] cancel job: ${cmd.join(' ')}"
log.info "[float] cancel job: ${toCmdString(cmd)}"
return cmd
}

Expand Down Expand Up @@ -298,7 +303,7 @@ class FloatGridExecutor extends AbstractGridExecutor {
cmd << 'squeue'
cmd << '--format'
cmd << 'json'
log.info "[float] query job status: ${cmd.join(' ')}"
log.info "[float] query job status: ${toCmdString(cmd)}"
return cmd
}

Expand Down

0 comments on commit 3637d78

Please sign in to comment.