From 8b13c2a6cf6a0674530b7ca465132fdd1885f7e0 Mon Sep 17 00:00:00 2001 From: Josh Eckels Date: Wed, 11 Dec 2024 10:34:06 -0800 Subject: [PATCH] Let users launch NextFlow from file browser (#470) --- nextflow/build.gradle | 7 +- .../labkey/nextflow/NextFlowController.java | 93 +++++++++++++++--- .../org/labkey/nextflow/NextFlowManager.java | 9 ++ .../org/labkey/nextflow/NextFlowModule.java | 2 + .../pipeline/NextFlowPipelineJob.java | 98 ++++++++++++++++--- .../pipeline/NextFlowPipelineProvider.java | 19 +++- .../nextflow/pipeline/NextFlowProtocol.java | 69 +++++++++++++ .../nextflow/pipeline/NextFlowRunTask.java | 72 ++++++++++---- .../WEB-INF/nextflow/nextflowContext.xml | 11 +++ 9 files changed, 331 insertions(+), 49 deletions(-) create mode 100644 nextflow/src/org/labkey/nextflow/pipeline/NextFlowProtocol.java diff --git a/nextflow/build.gradle b/nextflow/build.gradle index 01f789da..bb072afd 100644 --- a/nextflow/build.gradle +++ b/nextflow/build.gradle @@ -6,5 +6,10 @@ plugins { dependencies { BuildUtils.addLabKeyDependency(project: project, config: "modules", depProjectPath: BuildUtils.getPlatformModuleProjectPath(project.gradle, "pipeline"), depProjectConfig: "published", depExtension: "module") -} + compileOnly "org.projectlombok:lombok:${lombokVersion}" + annotationProcessor "org.projectlombok:lombok:${lombokVersion}" + + BuildUtils.addLabKeyDependency(project: project, config: "modules", depProjectPath: BuildUtils.getPlatformModuleProjectPath(project.gradle, "experiment"), depProjectConfig: "published", depExtension: "module") + BuildUtils.addLabKeyDependency(project: project, config: "modules", depProjectPath: BuildUtils.getPlatformModuleProjectPath(project.gradle, "pipeline"), depProjectConfig: "published", depExtension: "module") +} diff --git a/nextflow/src/org/labkey/nextflow/NextFlowController.java b/nextflow/src/org/labkey/nextflow/NextFlowController.java index b6b285d1..0fe2de2f 100644 --- a/nextflow/src/org/labkey/nextflow/NextFlowController.java +++ b/nextflow/src/org/labkey/nextflow/NextFlowController.java @@ -1,5 +1,8 @@ package org.labkey.nextflow; +import lombok.Data; +import lombok.Getter; +import lombok.Setter; import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.Logger; import org.labkey.api.action.ApiResponse; @@ -15,6 +18,7 @@ import org.labkey.api.pipeline.PipelineJob; import org.labkey.api.pipeline.PipelineService; import org.labkey.api.pipeline.PipelineStatusUrls; +import org.labkey.api.pipeline.browse.PipelinePathForm; import org.labkey.api.security.AdminConsoleAction; import org.labkey.api.security.RequiresPermission; import org.labkey.api.security.permissions.AdminOperationsPermission; @@ -22,8 +26,13 @@ import org.labkey.api.security.permissions.ReadPermission; import org.labkey.api.security.permissions.SiteAdminPermission; import org.labkey.api.util.Button; +import org.labkey.api.util.DOM; +import org.labkey.api.util.FileUtil; +import org.labkey.api.util.HtmlString; import org.labkey.api.util.PageFlowUtil; +import org.labkey.api.util.Path; import org.labkey.api.util.URLHelper; +import org.labkey.api.util.element.Select; import org.labkey.api.util.logging.LogHelper; import org.labkey.api.view.ActionURL; import org.labkey.api.view.HtmlView; @@ -36,14 +45,24 @@ import org.springframework.validation.Errors; import org.springframework.web.servlet.ModelAndView; +import javax.swing.text.html.FormView; + +import java.io.File; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.List; + import static org.labkey.api.util.DOM.Attribute.checked; +import static org.labkey.api.util.DOM.Attribute.hidden; import static org.labkey.api.util.DOM.Attribute.method; import static org.labkey.api.util.DOM.Attribute.name; import static org.labkey.api.util.DOM.Attribute.type; import static org.labkey.api.util.DOM.Attribute.value; import static org.labkey.api.util.DOM.DIV; import static org.labkey.api.util.DOM.INPUT; +import static org.labkey.api.util.DOM.LI; import static org.labkey.api.util.DOM.LK.FORM; +import static org.labkey.api.util.DOM.UL; import static org.labkey.api.util.DOM.at; import static org.labkey.nextflow.NextFlowManager.NEXTFLOW_CONFIG; @@ -245,11 +264,18 @@ public URLHelper getSuccessURL(EnabledForm o) } } + @Getter @Setter + public static class AnalyzeForm extends PipelinePathForm + { + private boolean launch = false; + private String configFile; + } + @RequiresPermission(AdminOperationsPermission.class) - public class NextFlowRunAction extends FormViewAction + public class NextFlowRunAction extends FormViewAction { @Override - public void validateCommand(Object o, Errors errors) + public void validateCommand(AnalyzeForm o, Errors errors) { if (!NextFlowManager.get().isEnabled(getContainer())) { @@ -258,26 +284,69 @@ public void validateCommand(Object o, Errors errors) } @Override - public ModelAndView getView(Object o, boolean b, BindException errors) + public ModelAndView getView(AnalyzeForm o, boolean b, BindException errors) { - return new HtmlView("NextFlow Runner", DIV("Run NextFlow Pipeline", - FORM(at(method, "POST"), - new Button.ButtonBuilder("Start NextFlow").submit(true).build()))); + NextFlowConfiguration config = NextFlowManager.get().getConfiguration(); + if (config.getNextFlowConfigFilePath() != null) + { + File configDir = new File(config.getNextFlowConfigFilePath()); + if (configDir.isDirectory()) + { + File[] files = configDir.listFiles(); + if (files != null && files.length > 0) + { + List configFiles = Arrays.asList(files); + return new HtmlView("NextFlow Runner", DIV( + FORM(at(method, "POST"), + INPUT(at(hidden, true, name, "launch", value, true)), + Arrays.stream(o.getFile()).map(f -> INPUT(at(hidden, true, name, "file", value, f))).toList(), + "Files: ", + UL(Arrays.stream(o.getFile()).map(DOM::LI)), + "Config: ", + new Select.SelectBuilder().name("configFile").addOptions(configFiles.stream().filter(f -> f.isFile() && f.getName().toLowerCase().endsWith(".config")).map(File::getName).sorted(String.CASE_INSENSITIVE_ORDER).toList()).build(), + new Button.ButtonBuilder("Start NextFlow").submit(true).build()))); + } + } + } + return new HtmlView(HtmlString.of("Couldn't find NextFlow config file(s)")); } @Override - public boolean handlePost(Object o, BindException errors) throws Exception + public boolean handlePost(AnalyzeForm form, BindException errors) throws Exception { - ViewBackgroundInfo info = getViewBackgroundInfo(); - PipeRoot root = PipelineService.get().findPipelineRoot(info.getContainer()); - PipelineJob job = new NextFlowPipelineJob(info, root); - PipelineService.get().queueJob(job); + if (!form.isLaunch()) + { + return false; + } + + NextFlowConfiguration config = NextFlowManager.get().getConfiguration(); + File configDir = new File(config.getNextFlowConfigFilePath()); + File configFile = FileUtil.appendPath(configDir, Path.parse(form.getConfigFile())); + if (!configFile.exists()) + { + errors.reject(ERROR_MSG, "Config file does not exist"); + } + else + { + List inputFiles = form.getValidatedFiles(getContainer()); + if (inputFiles.isEmpty()) + { + errors.reject(ERROR_MSG, "No input files"); + } + else + { + ViewBackgroundInfo info = getViewBackgroundInfo(); + PipeRoot root = PipelineService.get().findPipelineRoot(info.getContainer()); + PipelineJob job = NextFlowPipelineJob.create(info, root, configFile.toPath(), inputFiles.stream().map(File::toPath).toList()); + PipelineService.get().queueJob(job); + } + } return !errors.hasErrors(); } @Override - public URLHelper getSuccessURL(Object o) + public URLHelper getSuccessURL(AnalyzeForm o) { return PageFlowUtil.urlProvider(PipelineStatusUrls.class).urlBegin(getContainer()); } diff --git a/nextflow/src/org/labkey/nextflow/NextFlowManager.java b/nextflow/src/org/labkey/nextflow/NextFlowManager.java index 485380d7..e0325be0 100644 --- a/nextflow/src/org/labkey/nextflow/NextFlowManager.java +++ b/nextflow/src/org/labkey/nextflow/NextFlowManager.java @@ -7,6 +7,9 @@ import org.labkey.api.data.PropertyManager; import org.springframework.validation.BindException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.HashMap; import java.util.Map; @@ -44,6 +47,12 @@ private void checkArgs(NextFlowConfiguration config, BindException errors) if (StringUtils.isEmpty(config.getNextFlowConfigFilePath())) errors.rejectValue("nextFlowConfigFilePath", ERROR_MSG, "NextFlow config file path is required"); + Path configPath = Paths.get(config.getNextFlowConfigFilePath()); + if (Files.isDirectory(configPath)) + { + errors.rejectValue("nextFlowConfigFilePath", ERROR_MSG, "NextFlow config file path must be a directory"); + } + // Not yet used // if (StringUtils.isEmpty(config.getAccountName())) // errors.rejectValue("accountName", ERROR_MSG, "AWS account name is required"); diff --git a/nextflow/src/org/labkey/nextflow/NextFlowModule.java b/nextflow/src/org/labkey/nextflow/NextFlowModule.java index 68b35cd0..e9d769db 100644 --- a/nextflow/src/org/labkey/nextflow/NextFlowModule.java +++ b/nextflow/src/org/labkey/nextflow/NextFlowModule.java @@ -21,6 +21,8 @@ protected void startupAfterSpringConfig(ModuleContext moduleContext) { ActionURL adminUrl = new ActionURL(NextFlowController.NextFlowConfigurationAction.class, ContainerManager.getRoot()); AdminConsole.addLink(AdminConsole.SettingsLinkType.Configuration, "NextFlow Configuration", adminUrl, AdminPermission.class); + + PipelineService.get().registerPipelineProvider(new NextFlowPipelineProvider(this)); } @Override diff --git a/nextflow/src/org/labkey/nextflow/pipeline/NextFlowPipelineJob.java b/nextflow/src/org/labkey/nextflow/pipeline/NextFlowPipelineJob.java index 92d952b5..b0250318 100644 --- a/nextflow/src/org/labkey/nextflow/pipeline/NextFlowPipelineJob.java +++ b/nextflow/src/org/labkey/nextflow/pipeline/NextFlowPipelineJob.java @@ -1,44 +1,120 @@ package org.labkey.nextflow.pipeline; +import lombok.Getter; +import org.apache.commons.lang3.StringUtils; import org.jetbrains.annotations.NotNull; +import org.labkey.api.data.Container; +import org.labkey.api.files.FileContentService; +import org.labkey.api.pipeline.ParamParser; import org.labkey.api.pipeline.PipeRoot; -import org.labkey.api.pipeline.PipelineJob; import org.labkey.api.pipeline.PipelineJobService; import org.labkey.api.pipeline.TaskId; import org.labkey.api.pipeline.TaskPipeline; +import org.labkey.api.pipeline.file.AbstractFileAnalysisJob; import org.labkey.api.util.FileUtil; -import org.labkey.api.util.URLHelper; +import org.labkey.api.util.PageFlowUtil; import org.labkey.api.view.ViewBackgroundInfo; +import java.io.BufferedWriter; import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; -public class NextFlowPipelineJob extends PipelineJob +@Getter +public class NextFlowPipelineJob extends AbstractFileAnalysisJob { + private Path config; + @SuppressWarnings("unused") // For serialization protected NextFlowPipelineJob() {} - public NextFlowPipelineJob(ViewBackgroundInfo info, @NotNull PipeRoot root) + public static NextFlowPipelineJob create(ViewBackgroundInfo info, @NotNull PipeRoot root, Path templateConfig, List inputFiles) throws IOException { - super(null, info, root); - setLogFile(new File(String.valueOf(root.getLogDirectory()), FileUtil.makeFileNameWithTimestamp("NextFlowPipelineJob", "log")).toPath()); + Path parentDir = inputFiles.get(0).getParent(); + + String jobName = FileUtil.makeFileNameWithTimestamp("NextFlow"); + Path jobDir = parentDir.resolve(jobName); + Path log = jobDir.resolve(jobName + ".log"); + FileUtil.createDirectory(jobDir); + + Path config = createConfig(templateConfig, log.getParent(), jobDir, info.getContainer()); + + return new NextFlowPipelineJob(info, root, config, inputFiles, log); } - @Override - public URLHelper getStatusHref() + public NextFlowPipelineJob(ViewBackgroundInfo info, @NotNull PipeRoot root, Path config, List inputFiles, Path log) throws IOException { - return null; + super(new NextFlowProtocol(), NextFlowPipelineProvider.NAME, info, root, config.getFileName().toString(), config, inputFiles, false, false); + this.config = config; + setLogFile(log); + } + + @Override + public ParamParser getInputParameters() + { + return PipelineJobService.get().createParamParser(); + } + + /** Take the template config file and substitute in the values for this job */ + private static Path createConfig(Path configTemplate, Path parentDir, Path jobDir, Container container) throws IOException + { + String template; + try (InputStream in = Files.newInputStream(configTemplate)) + { + template = PageFlowUtil.getStreamContentsAsString(in); + } + + String webdavUrl = FileContentService.get().getWebDavUrl(parentDir, container, FileContentService.PathType.full); + webdavUrl = StringUtils.stripEnd(webdavUrl, "/"); + + String substitutedContent = template.replace("${quant_spectra_dir}", "quant_spectra_dir = '" + webdavUrl + "'"); + + Path substitutedFile = jobDir.resolve(configTemplate.getFileName()); + try (BufferedWriter writer = Files.newBufferedWriter(substitutedFile)) + { + writer.write(substitutedContent); + } + return substitutedFile; } @Override public String getDescription() { - return "NextFlow Job"; + return "NextFlow analysis using " + config.getFileName() + " of " + getInputFilePaths().size() + " files"; } @Override public TaskPipeline getTaskPipeline() { - return PipelineJobService.get().getTaskPipeline(new TaskId(NextFlowPipelineJob.class)); + return PipelineJobService.get().getTaskPipeline(getTaskPipelineId()); } + + @Override + public TaskId getTaskPipelineId() + { + return new TaskId(NextFlowPipelineJob.class); + } + + @Override + public AbstractFileAnalysisJob createSingleFileJob(File file) + { + throw new UnsupportedOperationException(); + } + + @Override + public File findInputFile(String name) + { + throw new UnsupportedOperationException(); + } + + @Override + public File findOutputFile(String name) + { + return null; + } + } diff --git a/nextflow/src/org/labkey/nextflow/pipeline/NextFlowPipelineProvider.java b/nextflow/src/org/labkey/nextflow/pipeline/NextFlowPipelineProvider.java index 449a3154..328990e3 100644 --- a/nextflow/src/org/labkey/nextflow/pipeline/NextFlowPipelineProvider.java +++ b/nextflow/src/org/labkey/nextflow/pipeline/NextFlowPipelineProvider.java @@ -1,19 +1,22 @@ package org.labkey.nextflow.pipeline; -import org.labkey.api.module.Module; import org.labkey.api.pipeline.PipeRoot; import org.labkey.api.pipeline.PipelineDirectory; import org.labkey.api.pipeline.PipelineProvider; import org.labkey.api.security.permissions.InsertPermission; import org.labkey.api.view.ViewContext; +import org.labkey.nextflow.NextFlowController; import org.labkey.nextflow.NextFlowManager; import org.labkey.nextflow.NextFlowModule; public class NextFlowPipelineProvider extends PipelineProvider { + + public static final String NAME = "NextFlow"; + public NextFlowPipelineProvider(NextFlowModule owningModule) { - super("NextFlow", owningModule); + super(NAME, owningModule); } @Override @@ -23,7 +26,15 @@ public void updateFileProperties(ViewContext context, PipeRoot pr, PipelineDirec return; if (!NextFlowManager.get().isEnabled(context.getContainer())) return; - } - + String actionId = createActionId(NextFlowController.NextFlowRunAction.class, "Analyze with NextFlow"); + addAction(actionId, + NextFlowController.NextFlowRunAction.class, + "Analyze with NextFlow", + directory, + directory.listPaths(new FileTypesEntryFilter(NextFlowProtocol.INPUT_TYPES)), + true, + true, + includeAll); + } } diff --git a/nextflow/src/org/labkey/nextflow/pipeline/NextFlowProtocol.java b/nextflow/src/org/labkey/nextflow/pipeline/NextFlowProtocol.java new file mode 100644 index 00000000..a2d7270c --- /dev/null +++ b/nextflow/src/org/labkey/nextflow/pipeline/NextFlowProtocol.java @@ -0,0 +1,69 @@ +package org.labkey.nextflow.pipeline; + +import org.jetbrains.annotations.Nullable; +import org.labkey.api.pipeline.PipeRoot; +import org.labkey.api.pipeline.file.AbstractFileAnalysisProtocol; +import org.labkey.api.pipeline.file.AbstractFileAnalysisProtocolFactory; +import org.labkey.api.util.FileType; +import org.labkey.api.view.ViewBackgroundInfo; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Path; +import java.util.List; +import java.util.Map; + +public class NextFlowProtocol extends AbstractFileAnalysisProtocol +{ + public static final List INPUT_TYPES = List.of( + new FileType(".RAW"), + new FileType(".mzML")); + + public NextFlowProtocol() + { + super("NextFlow", null, null); + } + + @Override + public List getInputTypes() + { + return INPUT_TYPES; + } + + @Override + public void setXml(String xml) + { + // No-op since NextFlow doesn't use XML + } + + @Override + public AbstractFileAnalysisProtocolFactory getFactory() + { + return new AbstractFileAnalysisProtocolFactory<>() + { + @Override + public NextFlowProtocol createProtocolInstance(String name, String description, String xml) + { + return new NextFlowProtocol(); + } + + @Override + public Path getDefaultParametersFile(PipeRoot root) + { + return null; + } + + @Override + public String getName() + { + return "NextFlow"; + } + }; + } + + @Override + public NextFlowPipelineJob createPipelineJob(ViewBackgroundInfo info, PipeRoot root, List filesInput, File fileParameters, @Nullable Map variableMap) throws IOException + { + throw new UnsupportedOperationException(); + } +} diff --git a/nextflow/src/org/labkey/nextflow/pipeline/NextFlowRunTask.java b/nextflow/src/org/labkey/nextflow/pipeline/NextFlowRunTask.java index 2934ea2e..aea90fa8 100644 --- a/nextflow/src/org/labkey/nextflow/pipeline/NextFlowRunTask.java +++ b/nextflow/src/org/labkey/nextflow/pipeline/NextFlowRunTask.java @@ -6,7 +6,10 @@ import org.labkey.api.pipeline.AbstractTaskFactorySettings; import org.labkey.api.pipeline.PipelineJob; import org.labkey.api.pipeline.PipelineJobException; +import org.labkey.api.pipeline.RecordedAction; import org.labkey.api.pipeline.RecordedActionSet; +import org.labkey.api.pipeline.ToolExecutionException; +import org.labkey.api.pipeline.WorkDirectoryTask; import org.labkey.api.security.SecurityManager; import org.labkey.api.util.FileType; import org.labkey.nextflow.NextFlowConfiguration; @@ -14,22 +17,31 @@ import java.io.BufferedReader; import java.io.File; -import java.io.FileInputStream; import java.io.IOException; +import java.io.InputStream; import java.io.InputStreamReader; import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.stream.Stream; -public class NextFlowRunTask extends PipelineJob.Task +public class NextFlowRunTask extends WorkDirectoryTask { + public static final String SPECTRA_INPUT_ROLE = "Spectra"; + + public static final String ACTION_NAME = "NextFlow"; + public NextFlowRunTask(Factory factory, PipelineJob job) { super(factory, job); } + + @Override public @NotNull RecordedActionSet run() throws PipelineJobException { @@ -62,7 +74,18 @@ public NextFlowRunTask(Factory factory, PipelineJob job) ProcessBuilder executionPB = new ProcessBuilder(getArgs()); getJob().runSubProcess(executionPB, dir); log.info("Job Finished"); - return new RecordedActionSet(); + + RecordedAction action = new RecordedAction(ACTION_NAME); + for (Path inputFile : getJob().getInputFilePaths()) + { + action.addInput(inputFile.toFile(), SPECTRA_INPUT_ROLE); + } + addOutputs(action, getJob().getLogFilePath().getParent().resolve("reports")); + return new RecordedActionSet(action); + } + catch (IOException e) + { + throw new PipelineJobException(e); } finally { @@ -73,10 +96,28 @@ public NextFlowRunTask(Factory factory, PipelineJob job) } } - private boolean hasAwsSection(File configFile) throws PipelineJobException + private void addOutputs(RecordedAction action, Path path) throws IOException { - try (FileInputStream fIn = new FileInputStream(configFile); - InputStreamReader isReader = new InputStreamReader(fIn, StandardCharsets.UTF_8); + if (Files.isRegularFile(path)) + { + action.addOutput(path.toFile(), "Output", false); + } + else if (Files.isDirectory(path)) + { + try (Stream listing = Files.list(path)) + { + for (Path child : listing.toList()) + { + addOutputs(action, child); + } + } + } + } + + private boolean hasAwsSection(Path configFile) throws PipelineJobException + { + try (InputStream in = Files.newInputStream(configFile); + InputStreamReader isReader = new InputStreamReader(in, StandardCharsets.UTF_8); BufferedReader reader = new BufferedReader(isReader)) { String line; @@ -104,18 +145,7 @@ private boolean hasAwsSection(File configFile) throws PipelineJobException private @NotNull List getArgs() throws PipelineJobException { NextFlowConfiguration config = NextFlowManager.get().getConfiguration(); - String nextFlowConfigFilePath = config.getNextFlowConfigFilePath(); - - if (nextFlowConfigFilePath == null) - { - throw new PipelineJobException("No NextFlow config file specified"); - } - - File configFile = new File(nextFlowConfigFilePath); - if (!configFile.isFile()) - { - throw new PipelineJobException("NextFlow config file not found"); - } + Path configFile = getJob().getConfig(); boolean aws = hasAwsSection(configFile); @@ -135,7 +165,7 @@ private boolean hasAwsSection(File configFile) throws PipelineJobException args.add(s3Path); } args.add("-c"); - args.add(nextFlowConfigFilePath); + args.add(configFile.toAbsolutePath().toString()); return args; } @@ -153,7 +183,7 @@ public Factory() } @Override - public PipelineJob.Task createTask(PipelineJob job) + public NextFlowRunTask createTask(PipelineJob job) { return new NextFlowRunTask(this, job); } @@ -167,7 +197,7 @@ public List getInputTypes() @Override public List getProtocolActionNames() { - return Collections.emptyList(); + return List.of(ACTION_NAME); } @Override diff --git a/nextflow/webapp/WEB-INF/nextflow/nextflowContext.xml b/nextflow/webapp/WEB-INF/nextflow/nextflowContext.xml index 3cb859bd..eee32d08 100644 --- a/nextflow/webapp/WEB-INF/nextflow/nextflowContext.xml +++ b/nextflow/webapp/WEB-INF/nextflow/nextflowContext.xml @@ -19,10 +19,21 @@ org.labkey.nextflow.pipeline.NextFlowRunTask + + + + + + + + + + org.labkey.api.exp.pipeline.XarGeneratorId +