Multiple swarms

src/main/java/org/jenkinsci/plugins/docker/swarm/BuildScheduler.java

@@ -17,14 +17,14 @@ public class BuildScheduler {
     private static final Logger LOGGER = Logger.getLogger(BuildScheduler.class.getName());
     private static AtomicInteger counter = new AtomicInteger(1);
 
-    public static void scheduleBuild(final Queue.BuildableItem bi) {
+    public static void scheduleBuild(final Queue.BuildableItem bi, final DockerSwarmCloud cloud) {
         try (ACLContext _ = ACL.as(ACL.SYSTEM)) {
             final DockerSwarmLabelAssignmentAction action = createLabelAssignmentAction(bi.task.getDisplayName());
             DockerSwarmAgentInfo dockerSwarmAgentInfo = new DockerSwarmAgentInfo(true);
             dockerSwarmAgentInfo.setAgentLabel(action.getLabel().toString());
             bi.replaceAction(dockerSwarmAgentInfo);
             bi.replaceAction(action);
-            final Node node = new DockerSwarmAgent(bi, action.getLabel().toString());
+            final Node node = new DockerSwarmAgent(bi, action.getLabel().toString(), cloud);
             Computer.threadPoolForRemoting.submit(() -> {
                 try {
                     Jenkins.getInstance().addNode(node); // locks queue

src/main/java/org/jenkinsci/plugins/docker/swarm/DeadAgentServiceReaperActor.java

@@ -5,6 +5,7 @@
 import java.util.concurrent.TimeUnit;
 import java.util.logging.Logger;
 
+import hudson.slaves.Cloud;
 import org.jenkinsci.plugins.docker.swarm.docker.api.response.ApiException;
 import org.jenkinsci.plugins.docker.swarm.docker.api.response.SerializationException;
 import org.jenkinsci.plugins.docker.swarm.docker.api.service.DeleteServiceRequest;
@@ -36,27 +37,28 @@ private void reapDeadAgentServices() throws IOException {
         try {
             final DockerSwarmPlugin swarmPlugin = Jenkins.getInstance().getPlugin(DockerSwarmPlugin.class);
             final ActorSystem as = swarmPlugin.getActorSystem();
-            if (DockerSwarmCloud.get() != null) {
-                String dockerSwarmApiUrl = DockerSwarmCloud.get().getDockerSwarmApiUrl();
-                final Object result = new ListServicesRequest(dockerSwarmApiUrl, "label", "ROLE=jenkins-agent").execute();
-                for (ScheduledService service : (List<ScheduledService>) getResult(result, List.class)) {
-                    Object tasks = new ListTasksRequest(dockerSwarmApiUrl, "service", service.Spec.Name).execute();
-                    if (tasks != null) {
-                        for (Task task : (List<Task>) getResult(tasks, List.class)) {
-                            if (task.isComplete()) {
-                                LOGGER.info("Reaping service: " + service.Spec.Name);
-                                new DeleteServiceRequest(dockerSwarmApiUrl, service.Spec.Name).execute();
-                                break;
+            for (Cloud cloud : Jenkins.getInstance().clouds) {
+                if (cloud instanceof DockerSwarmCloud) {
+                    final DockerSwarmCloud swarmCloud = (DockerSwarmCloud)cloud;
+                    String dockerSwarmApiUrl = swarmCloud.getDockerSwarmApiUrl();
+                    final Object result = new ListServicesRequest(dockerSwarmApiUrl, "label", "ROLE=jenkins-agent").execute();
+                    for (ScheduledService service : (List<ScheduledService>) getResult(result, List.class)) {
+                        Object tasks = new ListTasksRequest(dockerSwarmApiUrl, "service", service.Spec.Name).execute();
+                        if (tasks != null) {
+                            for (Task task : (List<Task>) getResult(tasks, List.class)) {
+                                if (task.isComplete()) {
+                                    LOGGER.info("Reaping service: " + service.Spec.Name);
+                                    new DeleteServiceRequest(swarmCloud.name, service.Spec.Name).execute();
+                                    break;
+                                }
                             }
                         }
                     }
-
                 }
             }
         } finally {
             reschedule();
         }
-
     }
 
     private <T> T getResult(Object result, Class<T> clazz) {

src/main/java/org/jenkinsci/plugins/docker/swarm/DockerSwarmAgent.java

@@ -25,19 +25,33 @@ public class DockerSwarmAgent extends AbstractCloudSlave implements EphemeralNod
 
     private static final Logger LOGGER = Logger.getLogger(DockerSwarmAgent.class.getName());
 
-    public DockerSwarmAgent(final Queue.BuildableItem bi, final String labelString)
+    private final String cloudName;
+    private final String labelName;
+
+    public DockerSwarmAgent(final Queue.BuildableItem bi, final String labelString, final DockerSwarmCloud cloud)
             throws Descriptor.FormException, IOException {
         super(labelString, "Docker swarm agent for building " + bi.task.getFullDisplayName(),
-                DockerSwarmCloud.get().getLabelConfiguration(bi.task.getAssignedLabel().getName()).getWorkingDir(), 1,
-                Mode.EXCLUSIVE, labelString, new DockerSwarmComputerLauncher(bi),
+                cloud.getLabelConfiguration(bi.task.getAssignedLabel().getName()).getWorkingDir(), 1,
+                Mode.EXCLUSIVE, labelString, new DockerSwarmComputerLauncher(bi, cloud),
                 new DockerSwarmAgentRetentionStrategy(1), Collections.emptyList());
+
+        this.labelName = bi.task.getAssignedLabel().getName();
+        this.cloudName = cloud.name;
         LOGGER.log(Level.FINE, "Created docker swarm agent: {0}", labelString);
     }
 
+    public String getCloudName() {
+        return cloudName;
+    }
+
     public DockerSwarmComputer createComputer() {
         return new DockerSwarmComputer(this);
     }
 
+    public DockerSwarmAgentTemplate getTemplate() {
+        return ((DockerSwarmCloud)Jenkins.getInstance().clouds.getByName(cloudName)).getLabelConfiguration(labelName);
+    }
+
     @Override
     protected void _terminate(final TaskListener listener) throws IOException, InterruptedException {
     }
@@ -67,7 +81,7 @@ public void terminate() throws IOException {
         try {
             DockerSwarmPlugin swarmPlugin = Jenkins.getInstance().getPlugin(DockerSwarmPlugin.class);
             ActorRef agentLauncherRef = swarmPlugin.getActorSystem().actorFor("/user/" + getComputer().getName());
-            agentLauncherRef.tell(new DeleteServiceRequest(getComputer().getName()), ActorRef.noSender());
+            agentLauncherRef.tell(new DeleteServiceRequest(getCloudName(), getComputer().getName()), ActorRef.noSender());
         } finally {
             try {
                 Jenkins.getInstance().removeNode(this);

src/main/java/org/jenkinsci/plugins/docker/swarm/DockerSwarmAgentLauncherActor.java

@@ -73,8 +73,8 @@ private void serviceLogResponse(ApiSuccess apiSuccess) {
     }
 
     private void createService(ServiceSpec createRequest) throws IOException {
-        logger.println(String.format("[%s] Creating Service with Name : %s",
-                DateFormat.getTimeInstance().format(new Date()), createRequest.Name));
+        logger.println(String.format("[%s] Creating Service with Name : %s on swarm: %s",
+                DateFormat.getTimeInstance().format(new Date()), createRequest.Name, createRequest.SwarmName));
         this.createRequest = createRequest;
         handleServiceResponse(createRequest.execute());
     }
@@ -96,7 +96,7 @@ private void createServiceSuccess(CreateServiceResponse createServiceResponse) t
         if (StringUtils.isNotEmpty(createServiceResponse.Warning)) {
             logger.println("ServiceSpec creation warning : " + createServiceResponse.Warning);
         }
-        Object result = apiRequestWithErrorHandling(new ServiceLogRequest(createServiceResponse.ID));
+        Object result = apiRequestWithErrorHandling(new ServiceLogRequest(createRequest.SwarmName, createServiceResponse.ID));
         if (result instanceof ApiSuccess) {
             serviceLogResponse((ApiSuccess) result);
         }

src/main/java/org/jenkinsci/plugins/docker/swarm/DockerSwarmCloud.java

@@ -65,10 +65,10 @@ public class DockerSwarmCloud extends Cloud {
     private DockerServerEndpoint dockerHost;
 
     @DataBoundConstructor
-    public DockerSwarmCloud(DockerServerEndpoint dockerHost, String dockerSwarmApiUrl, String jenkinsUrl,
+    public DockerSwarmCloud(String name, DockerServerEndpoint dockerHost, String dockerSwarmApiUrl, String jenkinsUrl,
             String swarmNetwork, String cacheDriverName, String tunnel, List<DockerSwarmAgentTemplate> agentTemplates,
             long timeoutMinutes) {
-        super(DOCKER_SWARM_CLOUD_NAME);
+        super(name);
         this.jenkinsUrl = jenkinsUrl;
         this.swarmNetwork = swarmNetwork;
         this.cacheDriverName = cacheDriverName;
@@ -135,7 +135,10 @@ public FormValidation doValidateTestDockerApiConnection(@QueryParameter("uri") S
             if (uri.endsWith("/")) {
                 return FormValidation.error("URI must not have trailing /");
             }
-            Object response = new PingRequest(uri).execute();
+            if(credentialsId.equals("")) {
+                credentialsId = null;
+            }
+            Object response = new PingRequest(uri, credentialsId).execute();
             if (response instanceof ApiException) {
                 return FormValidation.error(((ApiException) response).getCause(),
                         "Couldn't ping docker api: " + uri + "/_ping");
@@ -151,7 +154,7 @@ public String getDockerSwarmApiUrl() {
         return dockerHost.getUri();
     }
 
-    private static SSLConfig toSSlConfig(String credentialsId) {
+    public static SSLConfig toSSlConfig(String credentialsId) {
         if (credentialsId == null)
             return null;
 
@@ -211,8 +214,8 @@ public List<String> getLabels() {
         return Lists.newArrayList(labels);
     }
 
-    public static DockerSwarmCloud get() {
-        return (DockerSwarmCloud) Jenkins.getInstance().getCloud(DOCKER_SWARM_CLOUD_NAME);
+    public static DockerSwarmCloud get(String name) {
+        return (DockerSwarmCloud) Jenkins.getInstance().getCloud(name);
     }
 
     public void save() {
@@ -228,7 +231,7 @@ public static void initFromYaml() throws IOException {
             ObjectMapper mapper = new ObjectMapper(new YAMLFactory());
             try (InputStream in = new BufferedInputStream(new FileInputStream(swarmConfigYaml))) {
                 DockerSwarmCloud configuration = mapper.readValue(in, DockerSwarmCloud.class);
-                DockerSwarmCloud existingCloud = DockerSwarmCloud.get();
+                DockerSwarmCloud existingCloud = DockerSwarmCloud.get(configuration.name);
                 if (existingCloud != null) {
                     Jenkins.getInstance().clouds.remove(existingCloud);
                 }

src/main/java/org/jenkinsci/plugins/docker/swarm/DockerSwarmComputerLauncher.java

@@ -44,8 +44,8 @@ public class DockerSwarmComputerLauncher extends JNLPLauncher {
 
     private static final Logger LOGGER = Logger.getLogger(DockerSwarmComputerLauncher.class.getName());
 
-    public DockerSwarmComputerLauncher(final Queue.BuildableItem bi) {
-        super(DockerSwarmCloud.get().getTunnel(), null, new RemotingWorkDirSettings(false, "/tmp", null, false));
+    public DockerSwarmComputerLauncher(final Queue.BuildableItem bi, final DockerSwarmCloud cloud) {
+        super(cloud.getTunnel(), null, new RemotingWorkDirSettings(false, "/tmp", null, false));
         this.bi = bi;
         this.label = bi.task.getAssignedLabel().getName();
         this.jobName = bi.task instanceof AbstractProject ? ((AbstractProject) bi.task).getFullName()
@@ -66,8 +66,13 @@ public void launch(final SlaveComputer computer, final TaskListener listener) {
     }
 
     private void launch(final DockerSwarmComputer computer, final TaskListener listener) throws IOException {
-        final DockerSwarmCloud configuration = DockerSwarmCloud.get();
-        final DockerSwarmAgentTemplate dockerSwarmAgentTemplate = configuration.getLabelConfiguration(this.label);
+        final DockerSwarmAgent agent = computer.getNode();
+
+        final DockerSwarmCloud configuration = DockerSwarmCloud.get(agent.getCloudName());
+        final DockerSwarmAgentTemplate dockerSwarmAgentTemplate = agent.getTemplate();
+        if(dockerSwarmAgentTemplate == null) {
+            throw new RuntimeException("dockerSwarmAgentTemplate is null");
+        }
 
         this.agentInfo = this.bi.getAction(DockerSwarmAgentInfo.class);
         this.agentInfo.setDockerImage(dockerSwarmAgentTemplate.getImage());
@@ -130,8 +135,8 @@ public void launchContainer(String[] commands, DockerSwarmCloud configuration, S
         setLimitsAndReservations(dockerSwarmAgentTemplate, crReq);
         setHostBinds(dockerSwarmAgentTemplate, crReq);
         setHostNamedPipes(dockerSwarmAgentTemplate, crReq);
-        setSecrets(dockerSwarmAgentTemplate, crReq);
-        setConfigs(dockerSwarmAgentTemplate, crReq);
+        setSecrets(configuration.name, dockerSwarmAgentTemplate, crReq);
+        setConfigs(configuration.name, dockerSwarmAgentTemplate, crReq);
         setNetwork(configuration, crReq);
         setCacheDirs(configuration, dockerSwarmAgentTemplate, listener, computer, crReq);
         setTmpfs(dockerSwarmAgentTemplate, crReq);
@@ -174,9 +179,9 @@ private ServiceSpec createCreateServiceRequest(String[] commands, DockerSwarmClo
                     : String.format("docker run --rm --privileged --network %s %s sh -xc '%s' ",
                             configuration.getSwarmNetwork(), dockerSwarmAgentTemplate.getImage(), commands[2]);
 
-            crReq = new ServiceSpec(computer.getName(), "docker:17.12", commands, envVars, dir, user, hosts);
+            crReq = new ServiceSpec(configuration.name, computer.getName(), "docker:17.12", commands, envVars, dir, user, hosts);
         } else {
-            crReq = new ServiceSpec(computer.getName(), dockerSwarmAgentTemplate.getImage(), commands, envVars, dir,
+            crReq = new ServiceSpec(configuration.name, computer.getName(), dockerSwarmAgentTemplate.getImage(), commands, envVars, dir,
                     user, hosts);
         }
         return crReq;
@@ -229,11 +234,11 @@ private void setHostNamedPipes(DockerSwarmAgentTemplate dockerSwarmAgentTemplate
         }
     }
 
-    private void setSecrets(DockerSwarmAgentTemplate dockerSwarmAgentTemplate, ServiceSpec crReq) {
+    private void setSecrets(String swarmName, DockerSwarmAgentTemplate dockerSwarmAgentTemplate, ServiceSpec crReq) {
         String[] secrets = dockerSwarmAgentTemplate.getSecretsConfig();
         if (secrets.length > 0)
             try {
-                final Object secretList = new ListSecretsRequest().execute();
+                final Object secretList = new ListSecretsRequest(swarmName).execute();
                 for (int i = 0; i < secrets.length; i++) {
                     String secret = secrets[i];
                     String[] split = secret.split(":");
@@ -254,11 +259,11 @@ private void setSecrets(DockerSwarmAgentTemplate dockerSwarmAgentTemplate, Servi
             }
     }
 
-    private void setConfigs(DockerSwarmAgentTemplate dockerSwarmAgentTemplate, ServiceSpec crReq) {
+    private void setConfigs(String swarmName, DockerSwarmAgentTemplate dockerSwarmAgentTemplate, ServiceSpec crReq) {
         String[] configs = dockerSwarmAgentTemplate.getConfigsConfig();
         if (configs.length > 0)
             try {
-                final Object configList = new ListConfigsRequest().execute();
+                final Object configList = new ListConfigsRequest(swarmName).execute();
                 for (int i = 0; i < configs.length; i++) {
                     String config = configs[i];
                     String[] split = config.split(":");

src/main/java/org/jenkinsci/plugins/docker/swarm/OneShotProvisionQueueListener.java

@@ -7,9 +7,11 @@
 
 import hudson.Extension;
 import hudson.model.Computer;
+import hudson.model.Label;
 import hudson.model.Node;
 import hudson.model.Queue;
 import hudson.model.queue.QueueListener;
+import hudson.slaves.Cloud;
 import jenkins.model.Jenkins;
 
 @Extension
@@ -19,11 +21,12 @@ public class OneShotProvisionQueueListener extends QueueListener {
 
     @Override
     public void onEnterBuildable(final Queue.BuildableItem bi) {
-        final Queue.Task job = bi.task;
-        if (DockerSwarmCloud.get() != null) {
-            final List<String> labels = DockerSwarmCloud.get().getLabels();
-            if (job.getAssignedLabel() != null && labels.contains(job.getAssignedLabel().getName())) {
-                BuildScheduler.scheduleBuild(bi);
+        final Jenkins jenkins = Jenkins.getInstance();
+        final Label label = bi.getAssignedLabel();
+        for (Cloud cloud : jenkins.clouds) {
+            if (cloud instanceof DockerSwarmCloud && cloud.canProvision(label)) {
+                BuildScheduler.scheduleBuild(bi, (DockerSwarmCloud)cloud);
+                break;
             }
         }
     }

src/main/java/org/jenkinsci/plugins/docker/swarm/ResetStuckBuildsInQueueActor.java

@@ -4,14 +4,17 @@
 import java.util.Date;
 import java.util.Optional;
 import java.util.concurrent.TimeUnit;
+import java.util.logging.Level;
 import java.util.logging.Logger;
 
 import akka.actor.AbstractActor;
 import akka.actor.ActorRef;
 import akka.actor.ActorSystem;
 import akka.actor.Props;
+import hudson.model.Label;
 import hudson.model.Node;
 import hudson.model.Queue;
+import hudson.slaves.Cloud;
 import jenkins.model.Jenkins;
 import scala.concurrent.duration.Duration;
 
@@ -31,7 +34,15 @@ public static Props props() {
 
     private void resetStuckBuildsInQueue() throws IOException {
         try {
-            long resetMinutes = Optional.ofNullable(DockerSwarmCloud.get().getTimeoutMinutes()).orElse(DEFAULT_RESET_MINUTES);
+            //FIXME getting timeout from first cloud
+            DockerSwarmCloud swarmCloud = null;
+            for (final Cloud cloud: Jenkins.getInstance().clouds) {
+                if(cloud instanceof DockerSwarmCloud) {
+                    swarmCloud = (DockerSwarmCloud)cloud;
+                    break;
+                }
+            }
+            long resetMinutes = Optional.ofNullable(swarmCloud.getTimeoutMinutes()).orElse(DEFAULT_RESET_MINUTES);
             final Queue.Item[] items = Jenkins.getInstance().getQueue().getItems();
             for (int i = items.length - 1; i >= 0; i--) { // reverse order
                 final Queue.Item item = items[i];
@@ -47,7 +58,18 @@ private void resetStuckBuildsInQueue() throws IOException {
                         final Node provisionedNode = Jenkins.getInstance().getNode(computerName);
                         if (provisionedNode != null) {
                             LOGGER.info(String.format("Rescheduling %s and Deleting %s computer ", item, computerName));
-                            BuildScheduler.scheduleBuild((Queue.BuildableItem) item);
+                            //FIXME copy paste from onEnterBuildable
+                            //BuildScheduler.scheduleBuild((Queue.BuildableItem) item, DockerSwarmCloud.get());
+                            final Queue.BuildableItem bi = (Queue.BuildableItem) item;
+                            final Jenkins jenkins = Jenkins.getInstance();
+                            final Label label = bi.getAssignedLabel();
+                            for (Cloud cloud : jenkins.clouds) {
+                                LOGGER.log(Level.WARNING, "Checking cloud " + cloud.name + " url " + cloud.toString());
+                                if (cloud instanceof DockerSwarmCloud && cloud.canProvision(label)) {
+                                    BuildScheduler.scheduleBuild(bi, (DockerSwarmCloud)cloud);
+                                    break;
+                                }
+                            }
                             ((DockerSwarmAgent) provisionedNode).terminate();
                         }
                     }