Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Honor SCM checkout retry count #110

Merged
merged 3 commits into from
Mar 15, 2024
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -25,14 +25,17 @@
package org.jenkinsci.plugins.workflow.steps.scm;

import com.google.common.collect.ImmutableSet;
import hudson.AbortException;
import hudson.FilePath;
import hudson.Functions;
import hudson.Launcher;
import hudson.model.Run;
import hudson.model.TaskListener;
import hudson.model.listeners.SCMListener;
import hudson.scm.SCM;
import hudson.scm.SCMRevisionState;
import java.io.File;
import java.io.InterruptedIOException;
import java.nio.file.FileSystems;
import java.nio.file.Files;
import java.nio.file.attribute.PosixFilePermissions;
Expand All @@ -42,6 +45,7 @@
import java.util.logging.Level;
import java.util.logging.Logger;
import edu.umd.cs.findbugs.annotations.NonNull;
import jenkins.model.Jenkins;
import org.jenkinsci.plugins.workflow.steps.Step;
import org.jenkinsci.plugins.workflow.steps.StepContext;
import org.jenkinsci.plugins.workflow.steps.StepDescriptor;
Expand Down Expand Up @@ -126,7 +130,31 @@ public final void checkout(Run<?,?> run, FilePath workspace, TaskListener listen
}
}
}
scm.checkout(run, launcher, workspace, listener, changelogFile, baseline);

for (int retryCount = Jenkins.get().getScmCheckoutRetryCount(); retryCount >= 0; retryCount--) {
try {
scm.checkout(run, launcher, workspace, listener, changelogFile, baseline);
break;
} catch (AbortException e) {
// abort exception might have a null message.
// If so, just skip echoing it.
if (e.getMessage() != null) {
listener.error(e.getMessage());
}
} catch (InterruptedIOException e) {
throw e;
} catch (Exception e) {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This block should include:

if (e instanceof FlowInterruptedException && ((FlowInterruptedException)e).isActualInterruption()) {
    throw e;
}

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

// checkout error not yet reported
Functions.printStackTrace(e, listener.error("Checkout failed"));
}

if (retryCount == 0) // all attempts failed
throw new AbortException("Maximum checkout retry attempts reached, aborting");
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ideally we would throw up the original exception if this is the last attempt, rather than constructing a fresh exception.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fixed


listener.getLogger().println("Retrying after 10 seconds");
Thread.sleep(10000);
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This does not look right to me, should probably do something like what the sleep step does instead of blocking the thread here. Though I am not sure atm how a SynchronousNonBlockingStepExecution is supposed to behave in these situations.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Well, a background thread will sleep. Not terrible, under the assumption this code path is rarely used.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Shall I keep this change? Or, do I need to rewrite the whole StepExecutionImpl?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Probably good enough as is. This step used SynchronousNonBlockingStepExecution (or the equivalent) from the beginning since the SCM interface in Jenkins does not support durable execution, so unlike

sh 'git clone https://github.com/myorg/myrepo'

a slow checkout cannot survive a controller restart. As of jenkinsci/workflow-basic-steps-plugin#203 this matters less anyway since you can use the new retry system with the nonresumable condition to handle that case.

}

if (changelogFile != null && changelogFile.length() == 0
&& changelogOriginalModifiedDate != null && changelogFile.lastModified() == changelogOriginalModifiedDate) {
// JENKINS-57918/JENKINS-59560/FakeChangeLogSCM: Some SCMs don't write anything to the changelog file in some
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -238,4 +238,25 @@ public ChangeLogSet<? extends ChangeLogSet.Entry> parse(Run build, RepositoryBro
public static class DescriptorImpl extends NullSCM.DescriptorImpl { }
}



@Test
public void scmRetryFromFakeChangeLogSCM() {
rr.then(r -> {
r.jenkins.setScmCheckoutRetryCount(2);
WorkflowJob p = r.jenkins.createProject(WorkflowJob.class, "p");
p.setDefinition(new CpsFlowDefinition(
"import org.jenkinsci.plugins.workflow.steps.scm.UnstableSCM\n" +
"def testSCM = new UnstableSCM(2)\n" +
"testSCM.addChange().withAuthor(/alice$BUILD_NUMBER/)\n" +
"node() {\n" +
" checkout(testSCM)\n" +
"}", false));
WorkflowRun b = r.buildAndAssertSuccess(p);
assertThat(b.getCulpritIds(), Matchers.equalTo(Collections.singleton("alice1")));
r.assertLogContains("Checkout failed", b);
r.assertLogContains("IO Exception happens", b);
r.assertLogContains("Retrying after 10 seconds", b);
});
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
package org.jenkinsci.plugins.workflow.steps.scm;

import hudson.FilePath;
import hudson.Launcher;
import hudson.model.Run;
import hudson.model.TaskListener;
import hudson.scm.SCMRevisionState;

import java.io.File;
import java.io.IOException;

public class UnstableSCM extends org.jvnet.hudson.test.FakeChangeLogSCM {
private int failedCount;

public UnstableSCM(int failedCount) {
this.failedCount = failedCount;
}

@Override
public void checkout(Run<?, ?> build, Launcher launcher, FilePath remoteDir, TaskListener listener, File changeLogFile, SCMRevisionState baseline) throws IOException, InterruptedException {
try {
if (failedCount > 0) {
throw new IOException("IO Exception happens");
}
super.checkout(build, launcher, remoteDir, listener, changeLogFile, baseline);
} finally {
failedCount--;
}
}
}