diff --git a/nextflow/build.gradle b/nextflow/build.gradle new file mode 100644 index 00000000..01f789da --- /dev/null +++ b/nextflow/build.gradle @@ -0,0 +1,10 @@ +import org.labkey.gradle.util.BuildUtils + +plugins { + id 'org.labkey.build.module' +} + +dependencies { + BuildUtils.addLabKeyDependency(project: project, config: "modules", depProjectPath: BuildUtils.getPlatformModuleProjectPath(project.gradle, "pipeline"), depProjectConfig: "published", depExtension: "module") +} + diff --git a/nextflow/module.properties b/nextflow/module.properties new file mode 100644 index 00000000..be867ce9 --- /dev/null +++ b/nextflow/module.properties @@ -0,0 +1,8 @@ +ModuleClass: org.labkey.nextflow.NextFlowModule +Label: NextFlow module +Description: This module provides the functionality \ + for running the NextFlow pipeline jobs on PanoramaWeb. +License: Apache 2.0 +LicenseURL: http://www.apache.org/licenses/LICENSE-2.0 +SupportedDatabases: pgsql +ManageVersion: false diff --git a/nextflow/resources/views/begin.html b/nextflow/resources/views/begin.html new file mode 100644 index 00000000..9ba623a2 --- /dev/null +++ b/nextflow/resources/views/begin.html @@ -0,0 +1,17 @@ + + + + + + \ No newline at end of file diff --git a/nextflow/resources/views/nextFlowConfiguration.html b/nextflow/resources/views/nextFlowConfiguration.html new file mode 100644 index 00000000..fd2a895c --- /dev/null +++ b/nextflow/resources/views/nextFlowConfiguration.html @@ -0,0 +1,88 @@ + + + + + + + + + + + + + + + + + + + + + +
NextFlow Config File Path
AWS Account Name
AWS Identity
AWS S3 Bucket Path
AWS Credential
+ + + + + diff --git a/nextflow/resources/views/nextFlowConfiguration.view.xml b/nextflow/resources/views/nextFlowConfiguration.view.xml new file mode 100644 index 00000000..8a589d3c --- /dev/null +++ b/nextflow/resources/views/nextFlowConfiguration.view.xml @@ -0,0 +1,5 @@ + + + + + \ No newline at end of file diff --git a/nextflow/src/org/labkey/nextflow/NextFlowController.java b/nextflow/src/org/labkey/nextflow/NextFlowController.java new file mode 100644 index 00000000..f22af232 --- /dev/null +++ b/nextflow/src/org/labkey/nextflow/NextFlowController.java @@ -0,0 +1,308 @@ +package org.labkey.nextflow; + +import org.apache.logging.log4j.Logger; +import org.labkey.api.action.ApiResponse; +import org.labkey.api.action.ApiSimpleResponse; +import org.labkey.api.action.FormViewAction; +import org.labkey.api.action.MutatingApiAction; +import org.labkey.api.action.ReadOnlyApiAction; +import org.labkey.api.action.SpringActionController; +import org.labkey.api.data.PropertyManager; +import org.labkey.api.data.PropertyStore; +import org.labkey.api.module.Module; +import org.labkey.api.module.ModuleHtmlView; +import org.labkey.api.module.ModuleLoader; +import org.labkey.api.pipeline.PipeRoot; +import org.labkey.api.pipeline.PipelineJob; +import org.labkey.api.pipeline.PipelineService; +import org.labkey.api.pipeline.PipelineStatusUrls; +import org.labkey.api.security.AdminConsoleAction; +import org.labkey.api.security.RequiresPermission; +import org.labkey.api.security.SecurityManager; +import org.labkey.api.security.permissions.AdminOperationsPermission; +import org.labkey.api.security.permissions.AdminPermission; +import org.labkey.api.security.permissions.SiteAdminPermission; +import org.labkey.api.util.Button; +import org.labkey.api.util.PageFlowUtil; +import org.labkey.api.util.URLHelper; +import org.labkey.api.util.logging.LogHelper; +import org.labkey.api.view.ActionURL; +import org.labkey.api.view.HtmlView; +import org.labkey.api.view.NavTree; +import org.labkey.api.view.ViewBackgroundInfo; +import org.labkey.nextflow.pipeline.NextFlowPipelineJob; +import org.springframework.validation.BindException; +import org.springframework.validation.Errors; +import org.springframework.web.servlet.ModelAndView; + +import java.util.HashSet; +import java.util.Set; + +import static org.labkey.api.util.DOM.Attribute.method; +import static org.labkey.api.util.DOM.DIV; +import static org.labkey.api.util.DOM.LK.FORM; +import static org.labkey.api.util.DOM.P; +import static org.labkey.api.util.DOM.at; +import static org.labkey.nextflow.NextFlowManager.NEXTFLOW_CONFIG; + +public class NextFlowController extends SpringActionController +{ + private static final DefaultActionResolver _actionResolver = new DefaultActionResolver(NextFlowController.class); + public static final String NAME = "nextflow"; + private static final String IS_NEXTFLOW_ENABLED = "enabled"; + + private static final Logger LOG = LogHelper.getLogger(NextFlowController.class, NAME); + + public NextFlowController() + { + setActionResolver(_actionResolver); + } + + @RequiresPermission(AdminPermission.class) + public class GetNextFlowConfigurationAction extends ReadOnlyApiAction + { + @Override + public ApiResponse execute(Object form, BindException errors) throws Exception + { + return new ApiSimpleResponse("config", PropertyManager.getEncryptedStore().getProperties(NEXTFLOW_CONFIG)); + } + } + + @RequiresPermission(AdminPermission.class) + public class DeleteNextFlowConfigurationAction extends MutatingApiAction + { + @Override + public ApiResponse execute(Object form, BindException errors) throws Exception + { + PropertyStore store = PropertyManager.getEncryptedStore(); + store.deletePropertySet(NEXTFLOW_CONFIG); + return new ApiSimpleResponse("success", true); + } + } + + + @AdminConsoleAction + @RequiresPermission(AdminOperationsPermission.class) + public static class NextFlowConfigurationAction extends FormViewAction + { + + @Override + public void validateCommand(NextFlowConfiguration target, Errors errors) + { + + } + + @Override + public ModelAndView getView(NextFlowConfiguration nextFlowConfiguration, boolean reshow, BindException errors) throws Exception + { + return ModuleHtmlView.get(ModuleLoader.getInstance().getModule(NextFlowModule.class), "nextFlowConfiguration"); + } + + @Override + public boolean handlePost(NextFlowConfiguration nextFlowConfiguration, BindException errors) throws Exception + { + NextFlowManager.get().addConfiguration(nextFlowConfiguration, errors); + return !errors.hasErrors(); + } + + @Override + public URLHelper getSuccessURL(NextFlowConfiguration nextFlowConfiguration) + { + return getContainer().getStartURL(getUser()); + } + + @Override + public void addNavTrail(NavTree root) + { + + } + } + + public static class NextFlowConfiguration + { + private String nextFlowConfigFilePath; + private String accountName; + private String identity; + private String s3BucketPath; + private String credential; + + public String getNextFlowConfigFilePath() + { + return nextFlowConfigFilePath; + } + + public void setNextFlowConfigFilePath(String nextFlowConfigFilePath) + { + this.nextFlowConfigFilePath = nextFlowConfigFilePath; + } + + public String getAccountName() + { + return accountName; + } + + public void setAccountName(String accountName) + { + this.accountName = accountName; + } + + public String getIdentity() + { + return identity; + } + + public void setIdentity(String identity) + { + this.identity = identity; + } + + public String getS3BucketPath() + { + return s3BucketPath; + } + + public void setS3BucketPath(String s3BucketPath) + { + this.s3BucketPath = s3BucketPath; + } + + public String getCredential() + { + return credential; + } + + public void setCredential(String credential) + { + this.credential = credential; + } + } + + @RequiresPermission(SiteAdminPermission.class) + public static class NextFlowEnableAction extends FormViewAction + { + + @Override + public void validateCommand(Object target, Errors errors) + { + + } + + @Override + public ModelAndView getView(Object form, boolean reshow, BindException errors) throws Exception + { + PropertyStore store = PropertyManager.getNormalStore(); + PropertyManager.PropertyMap map = store.getProperties(NextFlowManager.NEXTFLOW_ENABLE); + String btnTxt = "Enable NextFlow"; + // check if nextflow is enabled + if (Boolean.parseBoolean(map.get(IS_NEXTFLOW_ENABLED))) + { + btnTxt = "Disable NextFlow"; + } + else + { + btnTxt = "Enable NextFlow"; + } + + return new HtmlView("Enable/Disable Nextflow", DIV( P("NextFlow is currently " + (Boolean.parseBoolean(map.get(IS_NEXTFLOW_ENABLED)) ? "enabled" : "disabled")), + FORM(at(method, "POST"), + new Button.ButtonBuilder(btnTxt).submit(true).build()))); + } + + @Override + public boolean handlePost(Object form, BindException errors) throws Exception + { + PropertyStore store = PropertyManager.getNormalStore(); + PropertyManager.WritablePropertyMap map = store.getWritableProperties(NextFlowManager.NEXTFLOW_ENABLE, true); + if (map.isEmpty()) + { + map.put(IS_NEXTFLOW_ENABLED, Boolean.TRUE.toString()); + } + else + { + if (Boolean.parseBoolean(map.get(IS_NEXTFLOW_ENABLED))) + { + map.put(IS_NEXTFLOW_ENABLED, Boolean.FALSE.toString()); + } + else + { + map.put(IS_NEXTFLOW_ENABLED, Boolean.TRUE.toString()); + } + } + map.save(); + return true; + } + + @Override + public void addNavTrail(NavTree root) + { + + } + + @Override + public URLHelper getSuccessURL(Object o) + { + return getContainer().getStartURL(getUser()); + } + } + + @RequiresPermission(AdminOperationsPermission.class) + public class NextFlowRunAction extends FormViewAction + { + private ActionURL _successURL; + @Override + public void validateCommand(Object o, Errors errors) + { + PropertyStore store = PropertyManager.getNormalStore(); + PropertyManager.PropertyMap map = store.getProperties(NextFlowManager.NEXTFLOW_ENABLE); + if (!Boolean.parseBoolean(map.get(IS_NEXTFLOW_ENABLED))) + { + errors.reject(ERROR_MSG, "NextFlow is not enabled"); + } + } + + @Override + public ModelAndView getView(Object o, boolean b, BindException errors) throws Exception + { + return new HtmlView("NextFlow Runner", DIV("Run NextFlow Pipeline", + FORM(at(method, "POST"), + new Button.ButtonBuilder("Start NextFlow").submit(true).build()))); + } + + @Override + public boolean handlePost(Object o, BindException errors) throws Exception + { + // check if nextflow is enabled + PropertyStore store = PropertyManager.getNormalStore(); + PropertyManager.PropertyMap map = store.getProperties(NextFlowManager.NEXTFLOW_ENABLE); + if (map == null || !Boolean.parseBoolean(map.get(IS_NEXTFLOW_ENABLED))) + { + errors.reject(ERROR_MSG, "NextFlow is not enabled"); + return false; + } + + try (SecurityManager.TransformSession session = SecurityManager.createTransformSession(getViewContext())) + { + // TODO: pass the apiKey to Nextflow job + String apiKey = session.getApiKey(); + ViewBackgroundInfo info = getViewBackgroundInfo(); + PipeRoot root = PipelineService.get().findPipelineRoot(info.getContainer()); + PipelineJob job = new NextFlowPipelineJob(info, root, apiKey); + PipelineService.get().queueJob(job); + } + + return !errors.hasErrors(); + } + + @Override + public URLHelper getSuccessURL(Object o) + { + return PageFlowUtil.urlProvider(PipelineStatusUrls.class).urlBegin(getContainer()); + } + + @Override + public void addNavTrail(NavTree navTree) + { + navTree.addChild("NextFlow Runner"); + } + } +} diff --git a/nextflow/src/org/labkey/nextflow/NextFlowManager.java b/nextflow/src/org/labkey/nextflow/NextFlowManager.java new file mode 100644 index 00000000..23684b32 --- /dev/null +++ b/nextflow/src/org/labkey/nextflow/NextFlowManager.java @@ -0,0 +1,106 @@ +package org.labkey.nextflow; + +import org.apache.commons.lang3.StringUtils; +import org.labkey.api.data.CoreSchema; +import org.labkey.api.data.DbScope; +import org.labkey.api.data.PropertyManager; +import org.labkey.api.data.PropertyStore; +import org.springframework.validation.BindException; + +import java.util.HashMap; +import java.util.Map; + +import static org.labkey.api.action.SpringActionController.ERROR_MSG; + +public class NextFlowManager +{ + public static final String NEXTFLOW_CONFIG = "nextflow-config"; + public static final String NEXTFLOW_ENABLE = "nextflow-enable"; + + private static final String NEXTFLOW_ACCOUNT_NAME = "accountName"; + private static final String NEXTFLOW_CONFIG_FILE_PATH = "nextFlowConfigFilePath"; + private static final String NEXTFLOW_IDENTITY = "identity"; + private static final String NEXTFLOW_CREDENTIAL = "credential"; + private static final String NEXTFLOW_S3_BUCKET_PATH = "s3BucketPath"; + + private static final NextFlowManager _instance = new NextFlowManager(); + + // Normal store is used for enabled/disabled module + private static final PropertyStore _normalStore = PropertyManager.getNormalStore(); + + // Encrypted store is used for aws settings & nextflow file configuration + private static final PropertyStore _encryptedStore = PropertyManager.getEncryptedStore(); + + private NextFlowManager() + { + // prevent external construction with a private default constructor + } + + public static NextFlowManager get() + { + return _instance; + } + + + private void checkArgs(String nextFlowConfigFilePath, String name, String identity, String credential,String s3BucketPath, BindException errors) + { + if (StringUtils.isEmpty(nextFlowConfigFilePath)) + errors.rejectValue("nextFlowConfigFilePath", ERROR_MSG, "NextFlow config file path is required"); + + if (StringUtils.isEmpty(name)) + errors.rejectValue("name", ERROR_MSG, "AWS account name is required"); + + if (StringUtils.isEmpty(identity)) + errors.rejectValue("identity", ERROR_MSG, "AWS identity is required"); + + if (StringUtils.isEmpty(credential)) + errors.rejectValue("credential", ERROR_MSG, "AWS credential is required"); + + } + + public NextFlowController.NextFlowConfiguration getConfiguration() + { + PropertyManager.PropertyMap props = _encryptedStore.getWritableProperties(NEXTFLOW_CONFIG, false); + if (props != null) + { + NextFlowController.NextFlowConfiguration configuration = new NextFlowController.NextFlowConfiguration(); + configuration.setAccountName(props.get(NEXTFLOW_ACCOUNT_NAME)); + configuration.setNextFlowConfigFilePath(props.get(NEXTFLOW_CONFIG_FILE_PATH)); + configuration.setIdentity(props.get(NEXTFLOW_IDENTITY)); + configuration.setCredential(props.get(NEXTFLOW_CREDENTIAL)); + configuration.setS3BucketPath(props.get(NEXTFLOW_S3_BUCKET_PATH)); + return configuration; + } + + return null; + } + + public void addConfiguration(NextFlowController.NextFlowConfiguration configuration, BindException errors) + { + checkArgs(configuration.getNextFlowConfigFilePath(), configuration.getAccountName(), configuration.getIdentity(), configuration.getCredential(), configuration.getS3BucketPath(), errors); + + if (!errors.hasErrors()) + saveConfiguration(configuration); + } + + private void saveConfiguration( NextFlowController.NextFlowConfiguration configuration) + { + try (DbScope.Transaction tx = CoreSchema.getInstance().getSchema().getScope().ensureTransaction()) + { + Map properties = new HashMap<>(); + properties.put(NEXTFLOW_CONFIG_FILE_PATH, configuration.getNextFlowConfigFilePath()); + properties.put(NEXTFLOW_IDENTITY, configuration.getIdentity()); + properties.put(NEXTFLOW_CREDENTIAL, configuration.getCredential()); + properties.put(NEXTFLOW_S3_BUCKET_PATH, configuration.getS3BucketPath()); + properties.put(NEXTFLOW_ACCOUNT_NAME, configuration.getAccountName()); + + PropertyManager.WritablePropertyMap props = _encryptedStore.getWritableProperties(NEXTFLOW_CONFIG, true); + props.clear(); + props.putAll(properties); + props.save(); + + tx.commit(); + } + } + +} diff --git a/nextflow/src/org/labkey/nextflow/NextFlowModule.java b/nextflow/src/org/labkey/nextflow/NextFlowModule.java new file mode 100644 index 00000000..a962764d --- /dev/null +++ b/nextflow/src/org/labkey/nextflow/NextFlowModule.java @@ -0,0 +1,49 @@ +package org.labkey.nextflow; + +import org.jetbrains.annotations.NotNull; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.module.ModuleContext; +import org.labkey.api.module.SpringModule; +import org.labkey.api.security.permissions.AdminPermission; +import org.labkey.api.settings.AdminConsole; +import org.labkey.api.view.ActionURL; +import org.labkey.api.view.WebPartFactory; + +import java.util.Collection; +import java.util.List; + +public class NextFlowModule extends SpringModule +{ + @Override + protected void startupAfterSpringConfig(ModuleContext moduleContext) + { + ActionURL adminUrl = new ActionURL(NextFlowController.NextFlowConfigurationAction.class, ContainerManager.getRoot()); + AdminConsole.addLink(AdminConsole.SettingsLinkType.Configuration, "NextFlow Configuration", adminUrl, AdminPermission.class); + } + + @Override + protected void init() + { + addController(NextFlowController.NAME, NextFlowController.class); + } + + @Override + protected @NotNull Collection createWebPartFactories() + { + return List.of(); + } + + @Override + public boolean hasScripts() + { + return false; + } + + @Override + public @NotNull Collection getSchemaNames() + { + return List.of(); + } + +} diff --git a/nextflow/src/org/labkey/nextflow/pipeline/NextFlowPipelineJob.java b/nextflow/src/org/labkey/nextflow/pipeline/NextFlowPipelineJob.java new file mode 100644 index 00000000..42c2882f --- /dev/null +++ b/nextflow/src/org/labkey/nextflow/pipeline/NextFlowPipelineJob.java @@ -0,0 +1,54 @@ +package org.labkey.nextflow.pipeline; + +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.labkey.api.pipeline.PipeRoot; +import org.labkey.api.pipeline.PipelineJob; +import org.labkey.api.pipeline.PipelineJobService; +import org.labkey.api.pipeline.TaskId; +import org.labkey.api.pipeline.TaskPipeline; +import org.labkey.api.util.FileUtil; +import org.labkey.api.util.URLHelper; +import org.labkey.api.util.UnexpectedException; +import org.labkey.api.view.ViewBackgroundInfo; + +import java.io.File; +import java.io.IOException; + +public class NextFlowPipelineJob extends PipelineJob +{ + private String _apiKey; + // For serialization + protected NextFlowPipelineJob() + {} + + public NextFlowPipelineJob(ViewBackgroundInfo info, @NotNull PipeRoot root, String apiKey) + { + super(null, info, root); + this._apiKey = apiKey; + setLogFile(new File(String.valueOf(root.getLogDirectory()), FileUtil.makeFileNameWithTimestamp("NextFlowPipelineJob", "log")).toPath()); + } + + public String getApiKey() + { + return _apiKey; + } + + @Override + public URLHelper getStatusHref() + { + return null; + } + + @Override + public String getDescription() + { + return "NextFlow Job"; + } + + @Override + public TaskPipeline getTaskPipeline() + { + return PipelineJobService.get().getTaskPipeline(new TaskId(NextFlowPipelineJob.class)); + } +} diff --git a/nextflow/src/org/labkey/nextflow/pipeline/NextFlowRunTask.java b/nextflow/src/org/labkey/nextflow/pipeline/NextFlowRunTask.java new file mode 100644 index 00000000..a75312a3 --- /dev/null +++ b/nextflow/src/org/labkey/nextflow/pipeline/NextFlowRunTask.java @@ -0,0 +1,90 @@ +package org.labkey.nextflow.pipeline; + +import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.NotNull; +import org.labkey.api.files.FileContentService; +import org.labkey.api.pipeline.AbstractTaskFactory; +import org.labkey.api.pipeline.AbstractTaskFactorySettings; +import org.labkey.api.pipeline.PipelineJob; +import org.labkey.api.pipeline.PipelineJobException; +import org.labkey.api.pipeline.PipelineJobService; +import org.labkey.api.pipeline.RecordedActionSet; +import org.labkey.api.util.FileType; +import org.labkey.api.util.FileUtil; +import org.labkey.nextflow.NextFlowController; +import org.labkey.nextflow.NextFlowManager; + +import java.io.File; +import java.util.Collections; +import java.util.List; + +public class NextFlowRunTask extends PipelineJob.Task +{ + public NextFlowRunTask(Factory factory, PipelineJob job) + { + super(factory, job); + } + + @Override + public @NotNull RecordedActionSet run() throws PipelineJobException + { + Logger log = getJob().getLogger(); + NextFlowController.NextFlowConfiguration config = NextFlowManager.get().getConfiguration(); + String nextFlowConfigFilePath = config.getNextFlowConfigFilePath(); + String s3BucketPath = config.getS3BucketPath(); + String s3Path = "s3://" + s3BucketPath; + String apiKey = getJob().getApiKey(); + ProcessBuilder pb = new ProcessBuilder( "nextflow" , "secrets", "set", "PANORAMA_API_KEY", apiKey); + log.info("Job Started"); + File dir = FileContentService.get().getDefaultRootInfo(getJob().getContainer()).getPath().toFile(); + getJob().runSubProcess(pb, dir); + pb.command("nextflow" , "run", "-resume", "-r", "main", "-profile", "aws", "mriffle/nf-skyline-dia-ms", "-bucket-dir", s3Path, "-c", nextFlowConfigFilePath); + getJob().runSubProcess(pb, dir); + log.info("Job Finished"); + return new RecordedActionSet(); + } + + @Override + public NextFlowPipelineJob getJob() + { + return (NextFlowPipelineJob) super.getJob(); + } + + public static class Factory extends AbstractTaskFactory + { + public Factory() + { + super(NextFlowRunTask.class); + } + + @Override + public PipelineJob.Task createTask(PipelineJob job) + { + return new NextFlowRunTask(this, job); + } + + @Override + public List getInputTypes() + { + return Collections.emptyList(); + } + + @Override + public List getProtocolActionNames() + { + return Collections.emptyList(); + } + + @Override + public String getStatusName() + { + return "NextFlow Run"; + } + + @Override + public boolean isJobComplete(PipelineJob job) + { + return false; + } + } +} diff --git a/nextflow/webapp/WEB-INF/nextflow/nextflowContext.xml b/nextflow/webapp/WEB-INF/nextflow/nextflowContext.xml new file mode 100644 index 00000000..3cb859bd --- /dev/null +++ b/nextflow/webapp/WEB-INF/nextflow/nextflowContext.xml @@ -0,0 +1,28 @@ + + + + + + + + + + + + + + + + + + org.labkey.nextflow.pipeline.NextFlowRunTask + + + + + + + +