Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

1480 speedup npm install step for npm based formatters #1590

Merged
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion lib/src/main/java/com/diffplug/spotless/TimedLogger.java
Original file line number Diff line number Diff line change
@@ -126,7 +126,9 @@ private String durationString() {
if (duration < 1000) {
return duration + "ms";
} else if (duration < 1000 * 60) {
return (duration / 1000) + "s";
long seconds = duration / 1000;
long millis = duration - seconds * 1000;
return seconds + "." + millis + "s";
} else {
// output in the format 3m 4.321s
long minutes = duration / (1000 * 60);
Original file line number Diff line number Diff line change
@@ -0,0 +1,112 @@
/*
* Copyright 2023 DiffPlug
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.diffplug.spotless.npm;

import java.io.File;
import java.util.List;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.diffplug.spotless.ProcessRunner.Result;
import com.diffplug.spotless.TimedLogger;

public class NodeModulesCachingNpmProcessFactory implements NpmProcessFactory {

private static final Logger logger = LoggerFactory.getLogger(NodeModulesCachingNpmProcessFactory.class);

private static final TimedLogger timedLogger = TimedLogger.forLogger(logger);

private final File cacheDir;

private final ShadowCopy shadowCopy;

private NodeModulesCachingNpmProcessFactory(File cacheDir) {
this.cacheDir = cacheDir;
assertDir(cacheDir);
this.shadowCopy = new ShadowCopy(cacheDir);
}

private void assertDir(File cacheDir) {
if (cacheDir.exists() && !cacheDir.isDirectory()) {
throw new IllegalArgumentException("Cache dir must be a directory");
}
if (!cacheDir.exists()) {
if (!cacheDir.mkdirs()) {
throw new IllegalArgumentException("Cache dir could not be created.");
}
}
}

public static NodeModulesCachingNpmProcessFactory forCacheDir(File cacheDir) {
return new NodeModulesCachingNpmProcessFactory(cacheDir);
}

@Override
public NpmProcess createNpmInstallProcess(NodeServerLayout nodeServerLayout, NpmFormatterStepLocations formatterStepLocations) {
NpmProcess actualNpmInstallProcess = StandardNpmProcessFactory.INSTANCE.createNpmInstallProcess(nodeServerLayout, formatterStepLocations);
return new CachingNmpInstall(actualNpmInstallProcess, nodeServerLayout);
}

@Override
public NpmLongRunningProcess createNpmServeProcess(NodeServerLayout nodeServerLayout, NpmFormatterStepLocations formatterStepLocations) {
return StandardNpmProcessFactory.INSTANCE.createNpmServeProcess(nodeServerLayout, formatterStepLocations);
}

private class CachingNmpInstall implements NpmProcess {

private final NpmProcess actualNpmInstallProcess;
private final NodeServerLayout nodeServerLayout;

public CachingNmpInstall(NpmProcess actualNpmInstallProcess, NodeServerLayout nodeServerLayout) {
this.actualNpmInstallProcess = actualNpmInstallProcess;
this.nodeServerLayout = nodeServerLayout;
}

@Override
public Result waitFor() {
String entryName = entryName();
if (shadowCopy.entryExists(entryName, NodeServerLayout.NODE_MODULES)) {
timedLogger.withInfo("Using cached node_modules for {} from {}", entryName, cacheDir)
.run(() -> shadowCopy.copyEntryInto(entryName(), NodeServerLayout.NODE_MODULES, nodeServerLayout.nodeModulesDir()));
return new CachedResult();
} else {
Result result = actualNpmInstallProcess.waitFor();
assert result.exitCode() == 0;
// TODO: maybe spawn a thread to do this in the background?
timedLogger.withInfo("Caching node_modules for {} in {}", entryName, cacheDir)
.run(() -> shadowCopy.addEntry(entryName(), new File(nodeServerLayout.nodeModulesDir(), NodeServerLayout.NODE_MODULES)));
return result;
}
}

private String entryName() {
return nodeServerLayout.nodeModulesDir().getName();
}

@Override
public String describe() {
return String.format("Wrapper around [%s] to cache node_modules in [%s]", actualNpmInstallProcess.describe(), cacheDir.getAbsolutePath());
}
}

private class CachedResult extends Result {

public CachedResult() {
super(List.of("(from cache dir " + cacheDir + ")"), 0, new byte[0], new byte[0]);
}
}
}
Original file line number Diff line number Diff line change
@@ -27,6 +27,7 @@
class NodeServerLayout {

private static final Pattern PACKAGE_JSON_NAME_PATTERN = Pattern.compile("\"name\"\\s*:\\s*\"([^\"]+)\"");
static final String NODE_MODULES = "node_modules";

private final File nodeModulesDir;
private final File packageJsonFile;
@@ -55,7 +56,6 @@ private static String nodeModulesDirName(String packageJsonContent) {
}

File nodeModulesDir() {

return nodeModulesDir;
}

@@ -89,7 +89,7 @@ public boolean isLayoutPrepared() {
}

public boolean isNodeModulesPrepared() {
Path nodeModulesInstallDirPath = new File(nodeModulesDir(), "node_modules").toPath();
Path nodeModulesInstallDirPath = new File(nodeModulesDir(), NODE_MODULES).toPath();
if (!Files.isDirectory(nodeModulesInstallDirPath)) {
return false;
}
Original file line number Diff line number Diff line change
@@ -62,7 +62,7 @@ protected NpmFormatterStepStateBase(String stepName, NpmConfig npmConfig, NpmFor
this.npmConfig = requireNonNull(npmConfig);
this.locations = locations;
this.nodeServerLayout = new NodeServerLayout(locations.buildDir(), npmConfig.getPackageJsonContent());
this.nodeServeApp = new NodeServeApp(nodeServerLayout, npmConfig, new StandardNpmProcessFactory(), locations);
this.nodeServeApp = new NodeServeApp(nodeServerLayout, npmConfig, NodeModulesCachingNpmProcessFactory.forCacheDir(new File(locations.buildDir(), "spotless-npm-cache"))/*StandardNpmProcessFactory.INSTANCE*/, locations);
}

protected void prepareNodeServerLayout() throws IOException {
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
/*
* Copyright 2023 DiffPlug
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.diffplug.spotless.npm;

import com.diffplug.spotless.ProcessRunner.LongRunningProcess;

interface NpmLongRunningProcess {

String describe();

LongRunningProcess start();

}
29 changes: 1 addition & 28 deletions lib/src/main/java/com/diffplug/spotless/npm/NpmProcess.java
Original file line number Diff line number Diff line change
@@ -15,39 +15,12 @@
*/
package com.diffplug.spotless.npm;

import java.util.concurrent.ExecutionException;

import com.diffplug.spotless.ProcessRunner.LongRunningProcess;
import com.diffplug.spotless.ProcessRunner.Result;

interface NpmProcess {

String describe();

LongRunningProcess start();

default Result waitFor() {
try (LongRunningProcess npmProcess = start()) {
if (npmProcess.waitFor() != 0) {
throw new NpmProcessException("Running npm command '" + describe() + "' failed with exit code: " + npmProcess.exitValue() + "\n\n" + npmProcess.result());
}
return npmProcess.result();
} catch (InterruptedException e) {
throw new NpmProcessException("Running npm command '" + describe() + "' was interrupted.", e);
} catch (ExecutionException e) {
throw new NpmProcessException("Running npm command '" + describe() + "' failed.", e);
}
}

class NpmProcessException extends RuntimeException {
private static final long serialVersionUID = 6424331316676759525L;

public NpmProcessException(String message) {
super(message);
}
Result waitFor();

public NpmProcessException(String message, Throwable cause) {
super(message, cause);
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
/*
* Copyright 2023 DiffPlug
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.diffplug.spotless.npm;

public class NpmProcessException extends RuntimeException {
private static final long serialVersionUID = 6424331316676759525L;

public NpmProcessException(String message) {
super(message);
}

public NpmProcessException(String message, Throwable cause) {
super(message, cause);
}
}
Original file line number Diff line number Diff line change
@@ -18,7 +18,7 @@
public interface NpmProcessFactory {
NpmProcess createNpmInstallProcess(NodeServerLayout nodeServerLayout, NpmFormatterStepLocations formatterStepLocations);

NpmProcess createNpmServeProcess(NodeServerLayout nodeServerLayout, NpmFormatterStepLocations formatterStepLocations);
NpmLongRunningProcess createNpmServeProcess(NodeServerLayout nodeServerLayout, NpmFormatterStepLocations formatterStepLocations);

default String describe() {
return getClass().getSimpleName();
4 changes: 4 additions & 0 deletions lib/src/main/java/com/diffplug/spotless/npm/ShadowCopy.java
Original file line number Diff line number Diff line change
@@ -116,6 +116,10 @@ public File copyEntryInto(String key, String origName, File targetParentFolder)
return target;
}

public boolean entryExists(String key, String origName) {
return entry(key, origName).exists();
}

private static class CopyDirectoryRecursively extends SimpleFileVisitor<Path> {
private final File target;
private final File orig;
Original file line number Diff line number Diff line change
@@ -19,21 +19,29 @@
import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutionException;

import com.diffplug.spotless.ProcessRunner;

public class StandardNpmProcessFactory implements NpmProcessFactory {

public static final StandardNpmProcessFactory INSTANCE = new StandardNpmProcessFactory();

private StandardNpmProcessFactory() {
// only one instance neeeded
}

@Override
public NpmProcess createNpmInstallProcess(NodeServerLayout nodeServerLayout, NpmFormatterStepLocations formatterStepLocations) {
return new NpmInstall(nodeServerLayout.nodeModulesDir(), formatterStepLocations);
}

@Override
public NpmProcess createNpmServeProcess(NodeServerLayout nodeServerLayout, NpmFormatterStepLocations formatterStepLocations) {
public NpmLongRunningProcess createNpmServeProcess(NodeServerLayout nodeServerLayout, NpmFormatterStepLocations formatterStepLocations) {
return new NpmServe(nodeServerLayout.nodeModulesDir(), formatterStepLocations);
}

private static abstract class AbstractStandardNpmProcess implements NpmProcess {
private static abstract class AbstractStandardNpmProcess {
protected final ProcessRunner processRunner = ProcessRunner.usingRingBuffersOfCapacity(100 * 1024); // 100kB

protected final File workingDir;
@@ -55,22 +63,22 @@ protected Map<String, String> environmentVariables() {
"PATH", formatterStepLocations.nodeExecutable().getParentFile().getAbsolutePath() + File.pathSeparator + System.getenv("PATH"));
}

@Override
public ProcessRunner.LongRunningProcess start() {
protected ProcessRunner.LongRunningProcess doStart() {
try {
return processRunner.start(workingDir, environmentVariables(), null, true, commandLine());
} catch (IOException e) {
throw new NpmProcessException("Failed to launch npm command '" + describe() + "'.", e);
}
}

@Override
public String describe() {
protected abstract String describe();

public String doDescribe() {
return String.format("%s in %s [%s]", getClass().getSimpleName(), workingDir, String.join(" ", commandLine()));
}
}

private static class NpmInstall extends AbstractStandardNpmProcess {
private static class NpmInstall extends AbstractStandardNpmProcess implements NpmProcess {

public NpmInstall(File workingDir, NpmFormatterStepLocations formatterStepLocations) {
super(workingDir, formatterStepLocations);
@@ -85,9 +93,28 @@ protected List<String> commandLine() {
"--no-fund",
"--prefer-offline");
}

@Override
public String describe() {
return doDescribe();
}

@Override
public ProcessRunner.Result waitFor() {
try (ProcessRunner.LongRunningProcess npmProcess = doStart()) {
if (npmProcess.waitFor() != 0) {
throw new NpmProcessException("Running npm command '" + describe() + "' failed with exit code: " + npmProcess.exitValue() + "\n\n" + npmProcess.result());
}
return npmProcess.result();
} catch (InterruptedException e) {
throw new NpmProcessException("Running npm command '" + describe() + "' was interrupted.", e);
} catch (ExecutionException e) {
throw new NpmProcessException("Running npm command '" + describe() + "' failed.", e);
}
}
}

private static class NpmServe extends AbstractStandardNpmProcess {
private static class NpmServe extends AbstractStandardNpmProcess implements NpmLongRunningProcess {

public NpmServe(File workingDir, NpmFormatterStepLocations formatterStepLocations) {
super(workingDir, formatterStepLocations);
@@ -100,5 +127,15 @@ protected List<String> commandLine() {
"start",
"--scripts-prepend-node-path=true");
}

@Override
public String describe() {
return doDescribe();
}

@Override
public ProcessRunner.LongRunningProcess start() {
return doStart();
}
}
}
Loading