Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add timeout settings for potentially slow tasks #312

Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 11 additions & 9 deletions docs/goal/image.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,14 @@

Import images into k3s containerd.

| Name | User Property | Description | Default |
| -----| ------------- | ----------- | ------- |
| `ctrImages` | `k3s.ctrImages` | Download given images via `ctr image pull` inside k3s container. | [] |
| `tarFiles` | `k3s.tarFiles` | Import given tar files via `ctr image import` inside k3s container. | [] |
| `dockerImages` | `k3s.dockerImages` | Copy given images from docker deamon via `ctr image import` inside k3s container. | [] |
| `dockerPullAlways` | `k3s.dockerPullAlways` | Always pull docker images or only if not present. | false |
| `pullTimeout` | `k3s.pullTimeout` | Timout for `ctr image pull` or `docker pull` in seconds. | 1200 |
| `skipImage` | `k3s.skipImage` | Skip image handling. | false |
| `debug` | `k3s.debug` | Stream logs of docker and kubectl. | false |
| Name | User Property | Description | Default |
| -----|------------------------| ----------- |---------|
| `ctrImages` | `k3s.ctrImages` | Download given images via `ctr image pull` inside k3s container. | [] |
| `tarFiles` | `k3s.tarFiles` | Import given tar files via `ctr image import` inside k3s container. | [] |
| `dockerImages` | `k3s.dockerImages` | Copy given images from docker deamon via `ctr image import` inside k3s container. | [] |
| `dockerPullAlways` | `k3s.dockerPullAlways` | Always pull docker images or only if not present. | false |
| `pullTimeout` | `k3s.pullTimeout` | Timout for `ctr image pull` or `docker pull` in seconds. | 1200 |
| `copyToContainerTimeout` | `k3s.copyToContainerTimeout` | Timout for saving image to tar in seconds. | 60 |
| `saveImageTimeout` | `k3s.saveImageTimeout` | Timout for saving image to tar in seconds. | 60 |
| `skipImage` | `k3s.skipImage` | Skip image handling. | false |
| `debug` | `k3s.debug` | Stream logs of docker and kubectl. | false |
16 changes: 9 additions & 7 deletions src/main/java/io/kokuwa/maven/k3s/mojo/ApplyMojo.java
Original file line number Diff line number Diff line change
Expand Up @@ -84,25 +84,26 @@ public void execute() throws MojoExecutionException {

// verify container and copy manifests

if (getDocker().getContainer().isEmpty()) {
if (getDocker().getContainer(getDefaultTaskTimeout()).isEmpty()) {
throw new MojoExecutionException("No k3s container found");
}
getDocker().copyToContainer(manifests, toLinuxPath(path));
getDocker().copyToContainer(manifests, toLinuxPath(path), getDefaultTaskTimeout());

// wait for service account, see https://github.com/kubernetes/kubernetes/issues/66689

var serviceAccount = new String[] { "kubectl", "get", "sa", "default", "--ignore-not-found", "--output=name" };
if (getDocker().exec(serviceAccount).isEmpty()) {
if (getDocker().exec(getDefaultTaskTimeout(), serviceAccount).isEmpty()) {
log.info("");
log.info("No service account found, waiting for sa ...");
Await.await(log, "k3s service account ready").until(() -> !getDocker().exec(serviceAccount).isEmpty());
Await.await(log, "k3s service account ready")
.until(() -> !getDocker().exec(getDefaultTaskTimeout(), serviceAccount).isEmpty());
log.info("Service account found, continue ...");
log.info("");
}

// wait for node getting ready

getDocker().exec("kubectl", "wait", "--for=condition=Ready", "node", "k3s");
getDocker().exec(getDefaultTaskTimeout(), "kubectl", "wait", "--for=condition=Ready", "node", "k3s");

// execute command

Expand Down Expand Up @@ -182,7 +183,7 @@ private Stream<Entry<String, Callable<Boolean>>> waitFor(Entry<String, List<Stri

var kind = entry.getKey();
var resources = getDocker()
.exec("kubectl", "get", kind,
.exec(getDefaultTaskTimeout(), "kubectl", "get", kind,
"--all-namespaces",
"--no-headers",
"--output=custom-columns=:.metadata.namespace,:.metadata.name")
Expand Down Expand Up @@ -212,7 +213,8 @@ private Stream<Entry<String, Callable<Boolean>>> waitFor(Entry<String, List<Stri
log.info("{} {} ... ready", kind, representation);
return true;
} catch (MojoExecutionException e) {
getDocker().exec("kubectl", "get", "--output=yaml", "--namespace=" + namespace, kind, name);
getDocker().exec(getDefaultTaskTimeout(), "kubectl", "get", "--output=yaml",
"--namespace=" + namespace, kind, name);
return false;
}
});
Expand Down
45 changes: 35 additions & 10 deletions src/main/java/io/kokuwa/maven/k3s/mojo/ImageMojo.java
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,22 @@ public class ImageMojo extends K3sMojo {
@Parameter(property = "k3s.pullTimeout", defaultValue = "1200")
private Duration pullTimeout;

/**
* Timout for copying resources to the container in seconds.
*
* @since 1.5.0
*/
@Parameter(property = "k3s.copyToContainerTimeout", defaultValue = "60")
private Duration copyToContainerTimeout;

/**
* Timout for saving image to tar in seconds.
*
* @since 1.5.0
*/
@Parameter(property = "k3s.saveImageTimeout", defaultValue = "60")
private Duration saveImageTimeout;

/**
* Skip starting of k3s container.
*
Expand All @@ -93,7 +109,7 @@ public void execute() throws MojoExecutionException {

// verify container

if (getDocker().getContainer().isEmpty()) {
if (getDocker().getContainer(getDefaultTaskTimeout()).isEmpty()) {
throw new MojoExecutionException("No k3s container found");
}

Expand Down Expand Up @@ -154,12 +170,13 @@ private boolean tar(Map<String, Map<String, ?>> existingImages, Path tarFile) {
var destination = "/tmp/" + tarFile.getFileName() + "_" + System.nanoTime();
var outputPattern = Pattern.compile("^unpacking (?<image>.*) \\(sha256:[0-9a-f]{64}\\).*$");

getDocker().copyToContainer(tarFile, destination);
getDocker().copyToContainer(tarFile, destination, copyToContainerTimeout);
for (var output : getDocker().exec(pullTimeout, "ctr", "image", "import", destination.toString())) {
var matcher = outputPattern.matcher(output);
if (matcher.matches()) {
getDocker().exec("ctr", "image", "label", matcher.group("image"), labelPath + "=" + tarFile);
getDocker().exec("ctr", "image", "label", matcher.group("image"),
getDocker().exec(getDefaultTaskTimeout(), "ctr", "image", "label", matcher.group("image"),
labelPath + "=" + tarFile);
getDocker().exec(getDefaultTaskTimeout(), "ctr", "image", "label", matcher.group("image"),
labelChecksum + "=" + newChecksum);
} else {
log.warn("Tar {} import output cannot be parsed: {}", tarFile, output);
Expand Down Expand Up @@ -194,7 +211,7 @@ private boolean docker(Map<String, Map<String, ?>> existingImages, String image)

// pull image

var digest = getDocker().getImage(image).map(ContainerImage::getDigest).orElse(null);
var digest = getDocker().getImage(image, getDefaultTaskTimeout()).map(ContainerImage::getDigest).orElse(null);
if (dockerPullAlways || digest == null) {
if (digest != null) {
log.debug("Image {} found in docker, pull always ...", image);
Expand All @@ -207,7 +224,7 @@ private boolean docker(Map<String, Map<String, ?>> existingImages, String image)
log.error("Failed to pull docker image {}", image, e);
return false;
}
digest = getDocker().getImage(image).map(ContainerImage::getDigest).orElse(null);
digest = getDocker().getImage(image, getDefaultTaskTimeout()).map(ContainerImage::getDigest).orElse(null);
} else {
log.debug("Image {} found in docker", image);
}
Expand All @@ -232,10 +249,10 @@ private boolean docker(Map<String, Map<String, ?>> existingImages, String image)
var source = Paths.get(System.getProperty("java.io.tmpdir")).resolve(filename);
var destination = "/tmp/" + filename;
try {
getDocker().saveImage(image, source);
getDocker().copyToContainer(source, destination);
getDocker().saveImage(image, source, saveImageTimeout);
getDocker().copyToContainer(source, destination, copyToContainerTimeout);
getDocker().exec(pullTimeout, "ctr", "image", "import", destination.toString());
getDocker().exec("ctr", "image", "label", normalizedImage, label + "=" + digest);
getDocker().exec(getDefaultTaskTimeout(), "ctr", "image", "label", normalizedImage, label + "=" + digest);
} catch (MojoExecutionException e) {
log.error("Failed to import tar {}", source, e);
return false;
Expand All @@ -251,7 +268,7 @@ private boolean docker(Map<String, Map<String, ?>> existingImages, String image)
* @return Image name with labels.
*/
private Map<String, Map<String, ?>> getCtrImages() throws MojoExecutionException {
return getDocker().exec("ctr", "image", "list").stream().map(String::strip)
return getDocker().exec(getDefaultTaskTimeout(), "ctr", "image", "list").stream().map(String::strip)
.filter(row -> !row.startsWith("REF") && !row.startsWith("sha256:"))
.map(row -> row.split("(\\s)+"))
.filter(parts -> {
Expand Down Expand Up @@ -290,6 +307,14 @@ public void setPullTimeout(int pullTimeout) {
this.pullTimeout = Duration.ofSeconds(pullTimeout);
}

public void setCopyToContainerTimeout(int copyToContainerTimeout) {
this.copyToContainerTimeout = Duration.ofSeconds(copyToContainerTimeout);
}

public void setSaveImageTimeout(int saveImageTimeout) {
this.saveImageTimeout = Duration.ofSeconds(saveImageTimeout);
}

public void setSkipImage(boolean skipImage) {
this.skipImage = skipImage;
}
Expand Down
17 changes: 17 additions & 0 deletions src/main/java/io/kokuwa/maven/k3s/mojo/K3sMojo.java
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

import java.io.File;
import java.nio.file.Path;
import java.time.Duration;

import org.apache.maven.plugin.AbstractMojo;
import org.apache.maven.plugin.logging.Log;
Expand All @@ -28,6 +29,14 @@ public abstract class K3sMojo extends AbstractMojo {
@Parameter(property = "k3s.debug", defaultValue = "false")
private boolean debug;

/**
* Default timout for plugin tasks in seconds.
*
* @since 1.5.0
*/
@Parameter(property = "k3s.defaultTaskTimeout", defaultValue = "30")
private Duration defaultTaskTimeout;

/**
* Skip plugin.
*
Expand Down Expand Up @@ -61,6 +70,10 @@ public Marker getMarker() {
return marker;
}

public Duration getDefaultTaskTimeout() {
return defaultTaskTimeout;
}

@Deprecated
@Override
public Log getLog() {
Expand All @@ -86,6 +99,10 @@ public void setDebug(boolean debug) {
this.debug = debug;
}

public void setDefaultTaskTimeout(int defaultTaskTimeout) {
this.defaultTaskTimeout = Duration.ofSeconds(defaultTaskTimeout);
}

public void setSkip(boolean skip) {
this.skip = skip;
}
Expand Down
4 changes: 2 additions & 2 deletions src/main/java/io/kokuwa/maven/k3s/mojo/RemoveMojo.java
Original file line number Diff line number Diff line change
Expand Up @@ -36,9 +36,9 @@ public void execute() throws MojoExecutionException {
return;
}

getDocker().removeContainer();
getDocker().removeContainer(getDefaultTaskTimeout());
if (includeCache) {
getDocker().removeVolume();
getDocker().removeVolume(getDefaultTaskTimeout());
log.info("Deleted cache volume.");
}
}
Expand Down
11 changes: 7 additions & 4 deletions src/main/java/io/kokuwa/maven/k3s/mojo/RestartMojo.java
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ public void execute() throws MojoExecutionException {

// verify container

if (getDocker().getContainer().isEmpty()) {
if (getDocker().getContainer(getDefaultTaskTimeout()).isEmpty()) {
throw new MojoExecutionException("No k3s container found");
}

Expand Down Expand Up @@ -118,14 +118,17 @@ private Callable<Boolean> restart(String resoure) {

return () -> {
try {
getDocker().exec("kubectl", "rollout", "restart", kind, name, "--namespace=" + namespace);
getDocker().exec(timeout, "kubectl", "rollout", "restart", kind, name, "--namespace=" + namespace);
log.info("{} {}/{} restarted", kind, namespace, name);
getDocker().exec("kubectl", "rollout", "status", kind, name, "--namespace=" + namespace,
// Docker exec timeout should be greater than "inner" K3s timeout
getDocker().exec(timeout.plusSeconds(10), "kubectl", "rollout", "status", kind, name,
"--namespace=" + namespace,
"--timeout=" + timeout.getSeconds() + "s");
log.info("{} {}/{} restart finished", kind, namespace, name);
return true;
} catch (MojoExecutionException e) {
getDocker().exec("kubectl", "get", "--output=yaml", "--namespace=" + namespace, kind, name);
getDocker().exec(getDefaultTaskTimeout(), "kubectl", "get", "--output=yaml", "--namespace=" + namespace,
kind, name);
return false;
}
};
Expand Down
15 changes: 8 additions & 7 deletions src/main/java/io/kokuwa/maven/k3s/mojo/RunMojo.java
Original file line number Diff line number Diff line change
Expand Up @@ -244,14 +244,14 @@ public void execute() throws MojoExecutionException {

var create = true;
var restart = false;
var container = getDocker().getContainer().orElse(null);
var container = getDocker().getContainer(getDefaultTaskTimeout()).orElse(null);
if (container != null) {
if (failIfExists) {
throw new MojoExecutionException("Container with id '" + container.id
+ "' found. Please remove that container or set 'k3s.failIfExists' to false.");
} else if (replaceIfExists) {
log.info("Container with id '{}' found, replacing", container.id);
getDocker().removeContainer();
getDocker().removeContainer(getDefaultTaskTimeout());
} else if (!container.isRunning()) {
log.warn("Container with id '{}' found in stopped state, restart container", container.id);
create = false;
Expand All @@ -269,20 +269,21 @@ public void execute() throws MojoExecutionException {
if (create) {
createAndStartK3sContainer();
} else if (restart) {
getDocker().startContainer();
getDocker().startContainer(getDefaultTaskTimeout());
}

// wait for k3s api to be ready

var await = Await.await(log, "k3s api available").timeout(nodeTimeout);
getDocker().waitForLog(await, output -> output.stream().anyMatch(l -> l.contains("k3s is up and running")));
getDocker().waitForLog(await, output -> output.stream().anyMatch(l -> l.contains("k3s is up and running")),
getDefaultTaskTimeout());

// write file that k3s started

getMarker().writeStarted();
}

getDocker().copyFromContainer("/etc/rancher/k3s/k3s.yaml", kubeconfig);
getDocker().copyFromContainer("/etc/rancher/k3s/k3s.yaml", kubeconfig, getDefaultTaskTimeout());
log.info("k3s ready: KUBECONFIG={} kubectl get all --all-namespaces", kubeconfig);
}

Expand Down Expand Up @@ -340,8 +341,8 @@ private void createAndStartK3sContainer() throws MojoExecutionException {
}
var ports = new ArrayList<>(portBindings);
ports.add(portKubeApi + ":" + portKubeApi);
getDocker().createContainer(image, ports, command, registries);
getDocker().createVolume();
getDocker().createContainer(image, ports, command, registries, getDefaultTaskTimeout());
getDocker().createVolume(getDefaultTaskTimeout());
}

// setter
Expand Down
Loading