update to two step instead of gate rush (#296)

Co-authored-by: zkofiro <[email protected]>
intuit · Feb 6, 2024 · b8b2eda · b8b2eda
1 parent aee1e7f
commit b8b2eda
Show file tree

Hide file tree

Showing 15 changed files with 125 additions and 21 deletions.
diff --git a/api/src/main/java/com/intuit/tank/vm/vmManager/JobRequest.java b/api/src/main/java/com/intuit/tank/vm/vmManager/JobRequest.java
@@ -93,6 +93,12 @@ public interface JobRequest extends Serializable {
      */
     public abstract boolean isUseEips();
 
+    /**
+     *
+     * @return
+     */
+    public abstract boolean isUseTwoStep();
+
     /**
      * @return the userIntervalIncrement
      */

diff --git a/api/src/main/java/com/intuit/tank/vm/vmManager/JobRequestImpl.java b/api/src/main/java/com/intuit/tank/vm/vmManager/JobRequestImpl.java
@@ -50,6 +50,7 @@ public final class JobRequestImpl implements Serializable, JobRequest {
     private int baselineVirtualUsers;
     private long simulationTime;
     private boolean useEips;
+    private boolean useTwoStep;
     private int userIntervalIncrement;
     private String reportingMode = TankConstants.RESULTS_NONE;
     private String loggingProfile = LoggingProfile.STANDARD.name();
@@ -87,6 +88,13 @@ public boolean isUseEips() {
         return useEips;
     }
 
+    /**
+     * @return the useTwoStep
+     */
+    public boolean isUseTwoStep() {
+        return useTwoStep;
+    }
+
     /**
      * @return the stopBehavior
      */
@@ -310,6 +318,7 @@ public String toString() {
                 .append("stopBehavior", StopBehavior.fromString(stopBehavior).getDisplay())
                 .append("simulationTime", simulationTime)
                 .append("useEips", useEips)
+                .append("useTwoStep", useTwoStep)
                 .append("baselineVirtualUsers", baselineVirtualUsers)
                 .append("userIntervalIncrement", userIntervalIncrement)
                 .append("endRate", endRate)
@@ -422,6 +431,12 @@ public GeneratorT withUseEips(boolean aValue) {
             return (GeneratorT) this;
         }
 
+        @SuppressWarnings("unchecked")
+        public GeneratorT withUseTwoStep(boolean aValue) {
+            instance.useTwoStep = aValue;
+            return (GeneratorT) this;
+        }
+
         @SuppressWarnings("unchecked")
         public GeneratorT withLoggingProfile(String aValue) {
             instance.loggingProfile = aValue;

diff --git a/api/src/test/java/com/intuit/tank/vm/vmManager/JobRequestImplCpTest.java b/api/src/test/java/com/intuit/tank/vm/vmManager/JobRequestImplCpTest.java
@@ -171,6 +171,7 @@ public void testToString() {
                 .append("stopBehavior", StopBehavior.fromString(request.getStopBehavior()).getDisplay())
                 .append("simulationTime", request.getSimulationTime())
                 .append("useEips", request.isUseEips())
+                .append("useTwoStep", request.isUseTwoStep())
                 .append("baselineVirtualUsers", request.getBaselineVirtualUsers())
                 .append("userIntervalIncrement", "0")
                 .append("endRate", "0.0")

diff --git a/data_model/src/main/java/com/intuit/tank/project/BaseJob.java b/data_model/src/main/java/com/intuit/tank/project/BaseJob.java
@@ -107,6 +107,9 @@ public abstract class BaseJob extends BaseEntity {
     @Column(name = "use_eips")
     private Boolean useEips;
 
+    @Column(name = "use_two_step")
+    private Boolean useTwoStep;
+
     /**
      * 
      */
@@ -137,6 +140,7 @@ public BaseJob(BaseJob copy) {
         this.numAgents = copy.numAgents;
         this.vmInstanceType = copy.vmInstanceType;
         this.useEips = copy.useEips;
+        this.useTwoStep = copy.useTwoStep;
         this.tankClientClass = copy.getTankClientClass();
     }
 
@@ -189,6 +193,21 @@ public void setUseEips(boolean useEips) {
         this.useEips = useEips;
     }
 
+    /**
+     * @return two-step job start setting
+     */
+    public boolean isUseTwoStep() {
+        return useTwoStep != null ? useTwoStep : false;
+    }
+
+    /**
+     * @param useTwoStep
+     *            enable/disable two-step job starts
+     */
+    public void setUseTwoStep(boolean useTwoStep) {
+        this.useTwoStep = useTwoStep;
+    }
+
     /**
      * 
      * @return

diff --git a/rest-mvc/src/main/java/com/intuit/tank/rest/mvc/rest/util/JobDetailFormatter.java b/rest-mvc/src/main/java/com/intuit/tank/rest/mvc/rest/util/JobDetailFormatter.java
@@ -80,6 +80,7 @@ protected static String buildDetails(JobValidator validator, Workload workload,
             addProperty(sb, "Tank Http Client", config.getAgentConfig().getTankClientName(proposedJobInstance.getTankClientClass()));
             addProperty(sb, "Agent VM Type", getVmDetails(config, proposedJobInstance.getVmInstanceType()));
             addProperty(sb, "Assign Elastic Ips", Boolean.toString(proposedJobInstance.isUseEips()));
+            addProperty(sb, "Enable Two-Step Job Start", Boolean.toString(proposedJobInstance.isUseTwoStep()));
             if(proposedJobInstance.getIncrementStrategy().equals(IncrementStrategy.standard)) {
                 addProperty(sb, "Number of Agents", Integer.toString(proposedJobInstance.getNumAgents()));
             } else {

diff --git a/tank_vmManager/src/main/java/com/intuit/tank/perfManager/workLoads/JobManager.java b/tank_vmManager/src/main/java/com/intuit/tank/perfManager/workLoads/JobManager.java
@@ -193,20 +193,23 @@ private void startTest(final JobInfo info) {
         try {
             Thread.sleep(RETRY_SLEEP);// 30 seconds
         } catch (InterruptedException ignored) { }
-        info.agentData // set agent status from pending to ready to run
-                .forEach(agentData -> {
-                    CloudVmStatus status = vmTracker.getStatus(agentData.getInstanceId());
-                    if (status != null) {
-                        status.setVmStatus(VMStatus.ready);
-                        vmTracker.setStatus(status);
-                    }
-                });
-        LOG.info(new ObjectMessage(ImmutableMap.of("Message","Waiting for start agents command to start test for job " + jobId)));
-        try {
-            jobInfoMapLocalCache.get(jobId).latch.await();
-        } catch (InterruptedException ignored) {}
-        LOG.info(new ObjectMessage(ImmutableMap.of("Message","Start agents command received - Sending start commands for job " + jobId + " asynchronously to following agents: " +
-                info.agentData.stream().collect(Collectors.toMap(AgentData::getInstanceId, AgentData::getInstanceUrl)))));
+        if(info.jobRequest.isUseTwoStep()) { // two-step job start - set agent status to ready and wait for command to start load
+            info.agentData // set agent status from pending to ready to run
+                    .forEach(agentData -> {
+                        CloudVmStatus status = vmTracker.getStatus(agentData.getInstanceId());
+                        if (status != null) {
+                            status.setVmStatus(VMStatus.ready);
+                            vmTracker.setStatus(status);
+                        }
+                    });
+            LOG.info(new ObjectMessage(ImmutableMap.of("Message", "Waiting for start agents command to start test for job " + jobId)));
+            try {
+                jobInfoMapLocalCache.get(jobId).latch.await();
+            } catch (InterruptedException ignored) {
+            }
+            LOG.info(new ObjectMessage(ImmutableMap.of("Message", "Start agents command received - Sending start commands for job " + jobId + " asynchronously to following agents: " +
+                    info.agentData.stream().collect(Collectors.toMap(AgentData::getInstanceId, AgentData::getInstanceUrl)))));
+        }
         info.agentData.parallelStream()
                 .map(agentData -> agentData.getInstanceUrl() + AgentCommand.start.getPath())
                 .map(URI::create)

diff --git a/tank_vmManager/src/main/java/com/intuit/tank/perfManager/workLoads/WorkLoadFactory.java b/tank_vmManager/src/main/java/com/intuit/tank/perfManager/workLoads/WorkLoadFactory.java
@@ -136,6 +136,7 @@ private JobRequest jobToJobRequest(JobInstance job) {
                 .withStopBehavior(job.getStopBehavior())
                 .withReportingMode(job.getReportingMode())
                 .withUseEips(job.isUseEips())
+                .withUseTwoStep(job.isUseTwoStep())
                 .withVmInstanceType(job.getVmInstanceType())
                 .withnumUsersPerAgent(job.getNumUsersPerAgent())
                 .withNumAgents(job.getNumAgents())

diff --git a/web/web_support/src/main/java/com/intuit/tank/job/ActJobNodeBean.java b/web/web_support/src/main/java/com/intuit/tank/job/ActJobNodeBean.java
@@ -42,6 +42,7 @@ public ActJobNodeBean(JobInstance job, boolean hasRights, FastDateFormat fmt) {
         this.setActiveUsers(String.valueOf(job.getBaselineVirtualUsers()));
         this.setTotalUsers(String.valueOf(job.getTotalVirtualUsers()));
         this.setTargetRampRate(String.valueOf(job.getTargetRampRate() * job.getNumAgents()));
+        this.setUseTwoStep(job.isUseTwoStep());
         this.jobDetails = job.getJobDetails();
         this.estimatedNonlinearSteadyStateUsers = estimateNonlinearSteadyStateUsers(job.getTargetRampRate(), job.getRampTime(), job.getNumAgents());
         this.setStartTime(job.getStartTime());

diff --git a/web/web_support/src/main/java/com/intuit/tank/job/JobNodeBean.java b/web/web_support/src/main/java/com/intuit/tank/job/JobNodeBean.java
@@ -50,6 +50,7 @@ public abstract class JobNodeBean implements Serializable {
     private Date startTime;
     private Date endTime;
     private boolean hasRights = false;
+    private Boolean useTwoStep = false;
     private Map<Date, List<UserDetail>> statusDetailMap;
     // private Map<Date, Map<String, TPSInfo>> tpsInfoMap;
     private int tps;
@@ -229,6 +230,21 @@ public void setTargetRampRate(String targetRampRate) {
         this.targetRampRate = targetRampRate;
     }
 
+    /**
+     * @return two-step job start setting
+     */
+    public boolean isUseTwoStep() {
+        return useTwoStep != null ? useTwoStep : false;
+    }
+
+    /**
+     * @param useTwoStep
+     *            two-step job start setting
+     */
+    public void setUseTwoStep(boolean useTwoStep) {
+        this.useTwoStep = useTwoStep;
+    }
+
     /**
      * @return the validation errors
      */

diff --git a/web/web_support/src/main/java/com/intuit/tank/job/JobQueueAction.java b/web/web_support/src/main/java/com/intuit/tank/job/JobQueueAction.java
@@ -63,7 +63,9 @@ public void startAgents(JobNodeBean node) {
         AWSXRay.getCurrentSegment().putAnnotation("job.action", "startAgents");
         AWSXRay.getCurrentSegment().putAnnotation("jobId", node.getJobId());
         if (node instanceof ActJobNodeBean) {
-            controller.startAgents(node.getId());
+            if(node.isUseTwoStep()) {
+                controller.startAgents(node.getId());
+            }
         }
     }
 

diff --git a/web/web_support/src/main/java/com/intuit/tank/project/JobDetailFormatter.java b/web/web_support/src/main/java/com/intuit/tank/project/JobDetailFormatter.java
@@ -86,6 +86,7 @@ protected static String buildDetails(JobValidator validator, Workload workload,
             addProperty(sb, "Tank Http Client", config.getAgentConfig().getTankClientName(proposedJobInstance.getTankClientClass()));
             addProperty(sb, "Agent VM Type", getVmDetails(config, proposedJobInstance.getVmInstanceType()));
             addProperty(sb, "Assign Elastic Ips", Boolean.toString(proposedJobInstance.isUseEips()));
+            addProperty(sb, "Enable Two-Step Job Start", Boolean.toString(proposedJobInstance.isUseTwoStep()));
             if(proposedJobInstance.getIncrementStrategy().equals(IncrementStrategy.standard)) {
                 addProperty(sb, "Number of Agents", Integer.toString(proposedJobInstance.getNumAgents()));
             } else {

diff --git a/web/web_support/src/main/java/com/intuit/tank/project/JobMaker.java b/web/web_support/src/main/java/com/intuit/tank/project/JobMaker.java
@@ -264,6 +264,22 @@ public void setUseEips(boolean b) {
             projectBean.getJobConfiguration().setUseEips(b);
     }
 
+    /**
+     *
+     * @return
+     */
+    public boolean isUseTwoStep() {
+        return projectBean.getJobConfiguration().isUseTwoStep();
+    }
+
+    /**
+     *
+     * @return
+     */
+    public void setUseTwoStep(boolean b) {
+        projectBean.getJobConfiguration().setUseTwoStep(b);
+    }
+
     /**
      * 
      * @param location
@@ -311,6 +327,7 @@ public void createJobInstance() {
             proposedJobInstance.setCreator(securityContext.getCallerPrincipal().getName());
             proposedJobInstance.setScheduledTime(new Date());
             proposedJobInstance.setUseEips(isUseEips());
+            proposedJobInstance.setUseTwoStep((isUseTwoStep()));
             proposedJobInstance.setTankClientClass(getTankClientClass());
             proposedJobInstance.setLocation(getLocation());
             proposedJobInstance.setVmInstanceType(getVmInstanceType());

diff --git a/web/web_ui/src/main/webapp/agents/index.xhtml b/web/web_ui/src/main/webapp/agents/index.xhtml
@@ -108,8 +108,9 @@
             <h:outputText value="#{node.getTotalSubNodesRunning()}/#{node.getCurrentSubNodes().size()} Jobs Running" rendered="#{not node.allSubNodesCompleted() and node.type eq 'project'}" />
             <h:outputText value="Completed" rendered="#{node.allSubNodesCompleted() and node.type eq 'project'}" />
             <h:outputText value="#{node.status}" rendered="#{not node.hasSubNodes() and node.type eq 'job'}" />
-            <h:outputText value="#{node.status} (#{node.getTotalSubNodesReady()}/#{node.getCurrentSubNodes().size()} Agents Ready)" rendered="#{node.hasSubNodes() and node.type eq 'job' and node.status eq 'Starting'}" />
-            <h:outputText value="#{node.status} (#{node.getTotalSubNodesRunning()}/#{node.getCurrentSubNodes().size()} Agents Running)" rendered="#{node.hasSubNodes() and node.type eq 'job' and node.status ne 'Starting'}" />
+            <h:outputText value="#{node.status} (#{node.getTotalSubNodesRunning()}/#{node.getCurrentSubNodes().size()} Agents Running)" rendered="#{node.hasSubNodes() and node.type eq 'job' and not node.useTwoStep}" />
+            <h:outputText value="#{node.status} (#{node.getTotalSubNodesReady()}/#{node.getCurrentSubNodes().size()} Agents Ready)" rendered="#{node.hasSubNodes() and node.type eq 'job' and node.status eq 'Starting' and node.useTwoStep}" />
+            <h:outputText value="#{node.status} (#{node.getTotalSubNodesRunning()}/#{node.getCurrentSubNodes().size()} Agents Running)" rendered="#{node.hasSubNodes() and node.type eq 'job' and node.status ne 'Starting' and node.useTwoStep}" />
             <h:outputText value="#{node.status}" rendered="#{node.type eq 'vm'}" />
           </p:column>
 
@@ -189,7 +190,7 @@
                 disabled="#{!node.isRunnable() or !jobQueueManager.canControlJob(node)}" ajax="true"
                 update=":mainForm:jobTableId" />
               <p:commandButton id="btnStart" action="#{jobQueueAction.startAgents(node)}" value="L" title="Start Load"
-                               onerror="window.location.reload()" rendered="#{node.type ne 'project'}"
+                               onerror="window.location.reload()" rendered="#{node.type ne 'project' and node.useTwoStep}"
                                disabled="#{!node.isStartable() or !jobQueueManager.canControlJob(node)}" ajax="true"
                                update=":mainForm:jobTableId" />
               <p:commandButton id="btnKill" action="#{jobQueueAction.kill(node)}" value="K" title="Kill"

diff --git a/web/web_ui/src/main/webapp/projects/addWorkloadToJobQueue.xhtml b/web/web_ui/src/main/webapp/projects/addWorkloadToJobQueue.xhtml
@@ -3,6 +3,7 @@
   xmlns:ui="http://xmlns.jcp.org/jsf/facelets"
   xmlns:f="http://xmlns.jcp.org/jsf/core"
   xmlns:h="http://xmlns.jcp.org/jsf/html"
+  xmlns:ts="http://xmlns.jcp.org/jsf/composite/tag"
   xmlns:p="http://primefaces.org/ui">
 
   <div id="addWorkloads">
@@ -117,6 +118,24 @@
                 value="#{jobMaker.useEips}" />
             </div>
           </div>
+
+          <div class="formRow">
+            <div class="formLabelDiv">
+              <h:outputLabel for="useTwoStepCB"
+                             value="Enable Two-Step Job Start"
+                             styleClass="formLabel required" />
+              <ts:tip tipId="TwoStepTip"
+                      text="Enabling Two Step Job Start will configure the job to start in a two step process: launching all the agents/load injectors before waiting on the command to start load.
+                            This mode will also enable the `(L) - Start Load` job command in the UI. Running the job with `(R) - Run` will launch and check in the agents for the job. Once all agents are ready to drive load, denoted by
+                            a new 'ready' agent status, the job will wait for the start load command (L) to start load. This feature is particularly useful in the context of gate rush/spike testing where load start times need to be exact.
+                            Note: Agents will need to receive the start load command within 5-10 minutes of checking in to ensure agent connectivity."/>
+            </div>
+            <div class="formInputDiv">
+              <p:selectBooleanCheckbox id="useTwoStepCB"
+                                       value="#{jobMaker.useTwoStep}" />
+
+            </div>
+          </div>
         </h:panelGroup>
 
         <ui:fragment rendered="#{not projectBean.canEditProject()}">

diff --git a/web/web_ui/src/main/webapp/projects/projectjobqueue.xhtml b/web/web_ui/src/main/webapp/projects/projectjobqueue.xhtml
@@ -113,8 +113,9 @@
         style="width:#{projectJobQueueManager.tablePrefs.getSize('statusColumn')}px;"
         rendered="#{projectJobQueueManager.tablePrefs.isVisible('statusColumn')}">
         <h:outputText value="#{node.status}" rendered="#{not node.hasSubNodes()}" />
-        <h:outputText value="#{node.status} (#{node.getTotalSubNodesReady()}/#{node.getCurrentSubNodes().size()} Agents Ready)" rendered="#{node.hasSubNodes() and node.type eq 'job' and node.status eq 'Starting'}" />
-        <h:outputText value="#{node.status} (#{node.getTotalSubNodesRunning()}/#{node.getCurrentSubNodes().size()} Agents Running)" rendered="#{node.hasSubNodes() and node.type eq 'job' and node.status ne 'Starting'}" />
+        <h:outputText value="#{node.status} (#{node.getTotalSubNodesRunning()}/#{node.getSubNodes().size()} Agents Running)" rendered="#{node.hasSubNodes()} and not node.useTwoStep}" />
+        <h:outputText value="#{node.status} (#{node.getTotalSubNodesReady()}/#{node.getCurrentSubNodes().size()} Agents Ready)" rendered="#{node.hasSubNodes() and node.type eq 'job' and node.status eq 'Starting' and node.useTwoStep}" />
+        <h:outputText value="#{node.status} (#{node.getTotalSubNodesRunning()}/#{node.getCurrentSubNodes().size()} Agents Running)" rendered="#{node.hasSubNodes() and node.type eq 'job' and node.status eq 'Starting' and node.useTwoStep}" />
       </p:column>
 
       <p:column id="regionColumn" headerText="Region" styleClass="ellipsis min-column-size"
@@ -193,7 +194,7 @@
             onerror="window.location.reload()" disabled="#{!node.isRunnable() or !projectJobQueueManager.canControlJob(node)}"
             ajax="true" update=":mainForm:projectTabPanelID:jobTableId" />
           <p:commandButton id="btnStart" action="#{jobQueueAction.startAgents(node)}" value="L" title="Start Load"
-                           onerror="window.location.reload()" rendered="#{node.type ne 'project'}"
+                           onerror="window.location.reload()" rendered="#{node.type ne 'project' and node.useTwoStep}"
                            disabled="#{!node.isStartable() or !projectJobQueueManager.canControlJob(node)}" ajax="true"
                            update=":mainForm:projectTabPanelID:jobTableId" />
           <p:commandButton oncomplete="PF('confirmJobQueueKill').show();" value="K" ajax="true"