diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/CommentDao.java b/cuebot/src/main/java/com/imageworks/spcue/dao/CommentDao.java index 5568bb2fd..08bea59b1 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/CommentDao.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/CommentDao.java @@ -23,6 +23,8 @@ import com.imageworks.spcue.HostInterface; import com.imageworks.spcue.JobInterface; +import java.util.List; + public interface CommentDao { /** @@ -32,6 +34,26 @@ public interface CommentDao { */ public void deleteComment(String id); + /** + * Deletes comments using host, user, and subject + * + * @param host + * @param user + * @param subject + * @return boolean: returns true if one or more comments where deleted + */ + public boolean deleteCommentByHostUserAndSubject(HostInterface host, String user, String subject); + + /** + * Get comments using host, user, and subject + * + * @param host + * @param user + * @param subject + * @return List + */ + public List getCommentsByHostUserAndSubject(HostInterface host, String user, String subject); + /** * Retrieves the specified comment. * diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/HostDao.java b/cuebot/src/main/java/com/imageworks/spcue/dao/HostDao.java index 768bcdbd2..5ed18947e 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/HostDao.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/HostDao.java @@ -78,6 +78,14 @@ public interface HostDao { */ void updateHostState(HostInterface host, HardwareState state); + /** + * updates a host with the passed free temporary directory + * + * @param host + * @param freeTempDir + */ + void updateHostFreeTempDir(HostInterface host, Long freeTempDir); + /** * returns a full host detail * diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/CommentDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/CommentDaoJdbc.java index 9587e41db..ea61f07bb 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/CommentDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/CommentDaoJdbc.java @@ -21,6 +21,7 @@ import java.sql.ResultSet; import java.sql.SQLException; +import java.util.List; import java.util.Map; import org.springframework.jdbc.core.RowMapper; @@ -71,6 +72,18 @@ public CommentDetail mapRow(ResultSet rs, int row) throws SQLException { } }; + public boolean deleteCommentByHostUserAndSubject(HostInterface host, String user, String subject) { + return getJdbcTemplate().update( + "DELETE FROM comments WHERE pk_host=? AND str_user=? AND str_subject=?", + host.getHostId(), user, subject) > 0; + } + + public List getCommentsByHostUserAndSubject(HostInterface host, String user, String subject) { + return getJdbcTemplate().query( + "SELECT * FROM comments WHERE pk_host=? AND str_user=? AND str_subject=?", + COMMENT_DETAIL_MAPPER, host.getHostId(), user, subject); + } + public CommentDetail getCommentDetail(String id) { return getJdbcTemplate().queryForObject( "SELECT * FROM comments WHERE pk_comment=?", diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/HostDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/HostDaoJdbc.java index 5c106335c..6fe898b44 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/HostDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/HostDaoJdbc.java @@ -523,6 +523,13 @@ public void updateHostState(HostInterface host, HardwareState state) { state.toString(), host.getHostId()); } + @Override + public void updateHostFreeTempDir(HostInterface host, Long freeTempDir) { + getJdbcTemplate().update( + "UPDATE host_stat SET int_mcp_free=? WHERE pk_host=?", + freeTempDir, host.getHostId()); + } + @Override public void updateHostSetAllocation(HostInterface host, AllocationInterface alloc) { diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HostReportHandler.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HostReportHandler.java index d763cce53..2adef34fb 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HostReportHandler.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HostReportHandler.java @@ -29,10 +29,13 @@ import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.env.Environment; import org.springframework.core.task.TaskRejectedException; import org.springframework.dao.DataAccessException; import org.springframework.dao.EmptyResultDataAccessException; +import com.imageworks.spcue.CommentDetail; import com.imageworks.spcue.DispatchHost; import com.imageworks.spcue.FrameInterface; import com.imageworks.spcue.JobEntity; @@ -57,6 +60,7 @@ import com.imageworks.spcue.rqd.RqdClient; import com.imageworks.spcue.rqd.RqdClientException; import com.imageworks.spcue.service.BookingManager; +import com.imageworks.spcue.service.CommentManager; import com.imageworks.spcue.service.HostManager; import com.imageworks.spcue.service.JobManager; import com.imageworks.spcue.service.JobManagerSupport; @@ -80,6 +84,14 @@ public class HostReportHandler { private JobManagerSupport jobManagerSupport; private JobDao jobDao; private LayerDao layerDao; + @Autowired + private Environment env; + @Autowired + private CommentManager commentManager; + // Comment constants + private static final String SUBJECT_COMMENT_FULL_TEMP_DIR = "Host set to REPAIR for not having enough storage " + + "space on the temporary directory (mcp)"; + private static final String CUEBOT_COMMENT_USER = "cuebot"; /** * Boolean to toggle if this class is accepting data or not. @@ -156,7 +168,7 @@ public void handleHostReport(HostReport report, boolean isBoot) { rhost.getLoad(), new Timestamp(rhost.getBootTime() * 1000l), rhost.getAttributesMap().get("SP_OS")); - changeHardwareState(host, report.getHost().getState(), isBoot); + changeHardwareState(host, report.getHost().getState(), isBoot, report.getHost().getFreeMcp()); changeNimbyState(host, report.getHost()); /** @@ -221,7 +233,14 @@ public void handleHostReport(HostReport report, boolean isBoot) { } } - if (host.idleCores < Dispatcher.CORE_POINTS_RESERVED_MIN) { + // The minimum amount of free space in the temporary directory to book a host + Long minBookableFreeTempDir = env.getRequiredProperty("dispatcher.min_bookable_free_temp_dir_kb", Long.class); + + if (minBookableFreeTempDir != -1 && report.getHost().getFreeMcp() < minBookableFreeTempDir) { + msg = String.format("%s doens't have enough free space in the temporary directory (mcp), %dMB needs %dMB", + host.name, (report.getHost().getFreeMcp()/1024), (minBookableFreeTempDir/1024)); + } + else if (host.idleCores < Dispatcher.CORE_POINTS_RESERVED_MIN) { msg = String.format("%s doesn't have enough idle cores, %d needs %d", host.name, host.idleCores, Dispatcher.CORE_POINTS_RESERVED_MIN); } @@ -231,7 +250,7 @@ else if (host.idleMemory < Dispatcher.MEM_RESERVED_MIN) { } else if (report.getHost().getFreeMem() < CueUtil.MB512) { msg = String.format("%s doens't have enough free system mem, %d needs %d", - host.name, report.getHost().getFreeMem(), Dispatcher.MEM_RESERVED_MIN); + host.name, report.getHost().getFreeMem(), Dispatcher.MEM_RESERVED_MIN); } else if(!host.hardwareState.equals(HardwareState.UP)) { msg = host + " is not in the Up state."; @@ -309,13 +328,61 @@ else if (!dispatchSupport.isCueBookable(host)) { * updated with a boot report. If the state is Repair, then state is * never updated via RQD. * + * + * Prevent cue frames from booking on hosts with full temporary directories. + * + * Change host state to REPAIR or UP according the amount of free space + * in the temporary directory: + * - Set the host state to REPAIR, when the amount of free space in the + * temporary directory is less than the minimum required. Add a comment with + * subject: SUBJECT_COMMENT_FULL_TEMP_DIR + * - Set the host state to UP, when the amount of free space in the temporary directory + * is greater or equals to the minimum required and the host has a comment with + * subject: SUBJECT_COMMENT_FULL_TEMP_DIR + * * @param host * @param reportState * @param isBoot + * @param freeTempDir */ - private void changeHardwareState(DispatchHost host, - HardwareState reportState, boolean isBoot) { + private void changeHardwareState(DispatchHost host, HardwareState reportState, boolean isBoot, long freeTempDir) { + + // The minimum amount of free space in the temporary directory to book a host + Long minBookableFreeTempDir = env.getRequiredProperty("dispatcher.min_bookable_free_temp_dir_kb", Long.class); + + // Prevent cue frames from booking on hosts with full temporary directories + if (minBookableFreeTempDir != -1) { + if (host.hardwareState == HardwareState.UP && freeTempDir < minBookableFreeTempDir) { + + // Insert a comment indicating that the Host status = Repair with reason = Full temporary directory + CommentDetail c = new CommentDetail(); + c.subject = SUBJECT_COMMENT_FULL_TEMP_DIR; + c.user = CUEBOT_COMMENT_USER; + c.timestamp = null; + c.message = "Host " + host.getName() + " marked as REPAIR. The current amount of free space in the " + + "temporary directory (mcp) is " + (freeTempDir/1024) + "MB. It must have at least " + + (minBookableFreeTempDir/1024) + "MB of free space in temporary directory"; + commentManager.addComment(host, c); + // Set the host state to REPAIR + hostManager.setHostState(host, HardwareState.REPAIR); + host.hardwareState = HardwareState.REPAIR; + + return; + } else if (host.hardwareState == HardwareState.REPAIR && freeTempDir >= minBookableFreeTempDir) { + // Check if the host with REPAIR status has comments with subject=SUBJECT_COMMENT_FULL_TEMP_DIR and + // user=CUEBOT_COMMENT_USER and delete the comments, if they exists + boolean commentsDeleted = commentManager.deleteCommentByHostUserAndSubject(host, + CUEBOT_COMMENT_USER, SUBJECT_COMMENT_FULL_TEMP_DIR); + + if (commentsDeleted) { + // Set the host state to UP + hostManager.setHostState(host, HardwareState.UP); + host.hardwareState = HardwareState.UP; + return; + } + } + } // If the states are the same there is no reason to do this update. if (host.hardwareState.equals(reportState)) { @@ -374,7 +441,7 @@ private void changeNimbyState(DispatchHost host, RenderHost rh) { * locked if all cores are locked. * * @param host DispatchHost - * @param renderHost RenderHost + * @param coreInfo CoreDetail */ private void changeLockState(DispatchHost host, CoreDetail coreInfo) { if (host.lockState == LockState.LOCKED) { diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/CommentManager.java b/cuebot/src/main/java/com/imageworks/spcue/service/CommentManager.java index 10533c542..faee9dff9 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/service/CommentManager.java +++ b/cuebot/src/main/java/com/imageworks/spcue/service/CommentManager.java @@ -23,6 +23,8 @@ import com.imageworks.spcue.HostInterface; import com.imageworks.spcue.JobInterface; +import java.util.List; + public interface CommentManager { /** @@ -47,6 +49,26 @@ public interface CommentManager { */ public void deleteComment(String id); + /** + * Deletes comments using host, user, and subject + * + * @param host + * @param user + * @param subject + * @return boolean: returns true if one or more comments where deleted + */ + public boolean deleteCommentByHostUserAndSubject(HostInterface host, String user, String subject); + + /** + * Get comments using host, user, and subject + * + * @param host + * @param user + * @param subject + * @return List + */ + public List getCommentsByHostUserAndSubject(HostInterface host, String user, String subject); + /** * * @param id diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/CommentManagerService.java b/cuebot/src/main/java/com/imageworks/spcue/service/CommentManagerService.java index cc9a016ef..b6d4430ec 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/service/CommentManagerService.java +++ b/cuebot/src/main/java/com/imageworks/spcue/service/CommentManagerService.java @@ -28,6 +28,8 @@ import com.imageworks.spcue.ShowEntity; import com.imageworks.spcue.dao.CommentDao; +import java.util.List; + @Transactional public class CommentManagerService implements CommentManager { @@ -55,6 +57,16 @@ public void deleteComment(String id) { commentDao.deleteComment(id); } + @Transactional(propagation = Propagation.REQUIRED) + public boolean deleteCommentByHostUserAndSubject(HostInterface host, String user, String subject) { + return commentDao.deleteCommentByHostUserAndSubject(host, user, subject); + } + + @Transactional(propagation = Propagation.REQUIRED) + public List getCommentsByHostUserAndSubject(HostInterface host, String user, String subject) { + return commentDao.getCommentsByHostUserAndSubject(host, user, subject); + } + @Transactional(propagation = Propagation.REQUIRED) public void setCommentSubject(String id, String subject) { commentDao.updateCommentSubject(id, subject); diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/HostManager.java b/cuebot/src/main/java/com/imageworks/spcue/service/HostManager.java index 8b176c77e..aaf401688 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/service/HostManager.java +++ b/cuebot/src/main/java/com/imageworks/spcue/service/HostManager.java @@ -62,6 +62,14 @@ public interface HostManager { */ void setHostState(HostInterface host, HardwareState state); + /** + * Updates the free temporary directory (mcp) of a host. + * + * @param host HostInterface + * @param freeTempDir Long + */ + void setHostFreeTempDir(HostInterface host, Long freeTempDir); + /** * Return true if the host is swapping hard enough * that killing frames will save the entire machine. diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/HostManagerService.java b/cuebot/src/main/java/com/imageworks/spcue/service/HostManagerService.java index a7c5b0729..a1533d695 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/service/HostManagerService.java +++ b/cuebot/src/main/java/com/imageworks/spcue/service/HostManagerService.java @@ -93,6 +93,11 @@ public void setHostState(HostInterface host, HardwareState state) { hostDao.updateHostState(host, state); } + @Override + public void setHostFreeTempDir(HostInterface host, Long freeTempDir) { + hostDao.updateHostFreeTempDir(host, freeTempDir); + } + @Override @Transactional(propagation = Propagation.REQUIRED, readOnly=true) public boolean isSwapping(HostInterface host) { diff --git a/cuebot/src/main/resources/opencue.properties b/cuebot/src/main/resources/opencue.properties index a08522eb1..6b2875899 100644 --- a/cuebot/src/main/resources/opencue.properties +++ b/cuebot/src/main/resources/opencue.properties @@ -110,6 +110,12 @@ dispatcher.report_queue.max_pool_size=8 # Queue capacity for handling Host Report. dispatcher.report_queue.queue_capacity=1000 +# The minimum amount of free space in the temporary directory (mcp) to book a host. +# E.g: 1G = 1048576 kB => dispatcher.min_bookable_free_temp_dir_kb=1048576 +# Default = -1 (deactivated) +# If equals to -1, it means the feature is turned off +dispatcher.min_bookable_free_temp_dir_kb=-1 + # Number of threads to keep in the pool for kill frame operation. dispatcher.kill_queue.core_pool_size=6 # Maximum number of threads to allow in the pool for kill frame operation. diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/criteria/ProcSearchTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/criteria/ProcSearchTests.java index 59f00df9a..78a13b321 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/criteria/ProcSearchTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dao/criteria/ProcSearchTests.java @@ -50,6 +50,7 @@ import com.imageworks.spcue.service.HostManager; import com.imageworks.spcue.service.JobLauncher; import com.imageworks.spcue.service.JobManager; +import com.imageworks.spcue.util.CueUtil; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.Assert.assertEquals; @@ -209,11 +210,12 @@ private void launchJobs() { private RenderHost.Builder buildRenderHost() { return RenderHost.newBuilder() .setBootTime(1192369572) - .setFreeMcp(76020) + // The minimum amount of free space in the temporary directory to book a host. + .setFreeMcp(CueUtil.GB) .setFreeMem(53500) .setFreeSwap(20760) .setLoad(1) - .setTotalMcp(195430) + .setTotalMcp(CueUtil.GB4) .setTotalMem(8173264) .setTotalSwap(20960) .setNimbyEnabled(false) diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/BookingDaoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/BookingDaoTests.java index c6c03d604..577b53eac 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/BookingDaoTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/BookingDaoTests.java @@ -96,11 +96,12 @@ public DispatchHost createHost() { RenderHost host = RenderHost.newBuilder() .setName("test_host") .setBootTime(1192369572) - .setFreeMcp(76020) + // The minimum amount of free space in the temporary directory to book a host. + .setFreeMcp(CueUtil.GB) .setFreeMem(53500) .setFreeSwap(20760) .setLoad(1) - .setTotalMcp(195430) + .setTotalMcp(CueUtil.GB4) .setTotalMem((int) CueUtil.GB16) .setTotalSwap((int) CueUtil.GB16) .setNimbyEnabled(false) diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/CommentDaoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/CommentDaoTests.java index 668e666e9..9282d7b79 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/CommentDaoTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/CommentDaoTests.java @@ -140,11 +140,12 @@ public void testInsertCommentOnHost() { RenderHost host = RenderHost.newBuilder() .setName("boo") .setBootTime(1192369572) - .setFreeMcp(76020) + // The minimum amount of free space in the temporary directory to book a host. + .setFreeMcp(CueUtil.GB) .setFreeMem(15290520) .setFreeSwap(2076) .setLoad(1) - .setTotalMcp(19543) + .setTotalMcp(CueUtil.GB4) .setTotalMem(15290520) .setTotalSwap(2096) .setNimbyEnabled(false) diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/DeedDaoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/DeedDaoTests.java index a04e7e5e6..962b669bb 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/DeedDaoTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/DeedDaoTests.java @@ -73,11 +73,12 @@ public DispatchHost createHost() { RenderHost host = RenderHost.newBuilder() .setName("test_host") .setBootTime(1192369572) - .setFreeMcp(76020) + // The minimum amount of free space in the temporary directory to book a host. + .setFreeMcp(CueUtil.GB) .setFreeMem(15290520) .setFreeSwap(2076) .setLoad(1) - .setTotalMcp(19543) + .setTotalMcp(CueUtil.GB4) .setTotalMem((int) CueUtil.GB16) .setTotalSwap((int) CueUtil.GB16) .setNimbyEnabled(false) diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/DispatcherDaoFifoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/DispatcherDaoFifoTests.java index c34396709..4f6db1072 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/DispatcherDaoFifoTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/DispatcherDaoFifoTests.java @@ -56,6 +56,7 @@ import com.imageworks.spcue.service.JobLauncher; import com.imageworks.spcue.service.JobManager; import com.imageworks.spcue.test.AssumingPostgresEngine; +import com.imageworks.spcue.util.CueUtil; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -147,11 +148,12 @@ public void createHost() { RenderHost host = RenderHost.newBuilder() .setName(HOSTNAME) .setBootTime(1192369572) - .setFreeMcp(76020) + // The minimum amount of free space in the temporary directory to book a host. + .setFreeMcp(CueUtil.GB) .setFreeMem(53500) .setFreeSwap(20760) .setLoad(1) - .setTotalMcp(195430) + .setTotalMcp(CueUtil.GB4) .setTotalMem(8173264) .setTotalSwap(20960) .setNimbyEnabled(false) diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/DispatcherDaoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/DispatcherDaoTests.java index 900f50afe..99fe2543a 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/DispatcherDaoTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/DispatcherDaoTests.java @@ -146,11 +146,12 @@ public void createHost() { RenderHost host = RenderHost.newBuilder() .setName(HOSTNAME) .setBootTime(1192369572) - .setFreeMcp(76020) + // The minimum amount of free space in the temporary directory to book a host. + .setFreeMcp(CueUtil.GB) .setFreeMem(53500) .setFreeSwap(20760) .setLoad(1) - .setTotalMcp(195430) + .setTotalMcp(CueUtil.GB4) .setTotalMem(8173264) .setTotalSwap(20960) .setNimbyEnabled(false) diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/FrameDaoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/FrameDaoTests.java index 8d64d918e..6312e6502 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/FrameDaoTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/FrameDaoTests.java @@ -114,11 +114,12 @@ public void create() { RenderHost host = RenderHost.newBuilder() .setName(HOST) .setBootTime(1192369572) - .setFreeMcp(76020) + // The minimum amount of free space in the temporary directory to book a host. + .setFreeMcp(CueUtil.GB) .setFreeMem(53500) .setFreeSwap(20760) .setLoad(1) - .setTotalMcp(195430) + .setTotalMcp(CueUtil.GB4) .setTotalMem(8173264) .setTotalSwap(20960) .setNimbyEnabled(false) diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/HostDaoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/HostDaoTests.java index df965893b..a6261e464 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/HostDaoTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/HostDaoTests.java @@ -86,12 +86,13 @@ public static RenderHost buildRenderHost(String name) { RenderHost host = RenderHost.newBuilder() .setName(name) .setBootTime(1192369572) - .setFreeMcp(7602) + // The minimum amount of free space in the temporary directory to book a host. + .setFreeMcp(CueUtil.GB) .setFreeMem(15290520) .setFreeSwap((int) CueUtil.MB512) .setLoad(1) .setNimbyEnabled(false) - .setTotalMcp(19543) + .setTotalMcp(CueUtil.GB4) .setTotalMem((int) CueUtil.GB16) .setTotalSwap((int) CueUtil.GB2) .setNimbyEnabled(false) diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/ProcDaoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/ProcDaoTests.java index ab95e7e1a..f6cabc23a 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/ProcDaoTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/ProcDaoTests.java @@ -118,11 +118,12 @@ public DispatchHost createHost() { RenderHost host = RenderHost.newBuilder() .setName("beta") .setBootTime(1192369572) - .setFreeMcp(76020) + // The minimum amount of free space in the temporary directory to book a host. + .setFreeMcp(CueUtil.GB) .setFreeMem(53500) .setFreeSwap(20760) .setLoad(1) - .setTotalMcp(195430) + .setTotalMcp(CueUtil.GB4) .setTotalMem((int) CueUtil.GB32) .setTotalSwap(20960) .setNimbyEnabled(false) diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/ShowDaoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/ShowDaoTests.java index e44393ab6..ea0ed67b8 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/ShowDaoTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/ShowDaoTests.java @@ -71,11 +71,12 @@ public DispatchHost createHost() { RenderHost host = RenderHost.newBuilder() .setName("test_host") .setBootTime(1192369572) - .setFreeMcp(76020) + // The minimum amount of free space in the temporary directory to book a host. + .setFreeMcp(CueUtil.GB) .setFreeMem(53500) .setFreeSwap(20760) .setLoad(1) - .setTotalMcp(195430) + .setTotalMcp(CueUtil.GB4) .setTotalMem((int) CueUtil.GB16) .setTotalSwap((int) CueUtil.GB16) .setNimbyEnabled(false) diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/WhiteboardDaoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/WhiteboardDaoTests.java index d419b6ce9..293359f8d 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/WhiteboardDaoTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/WhiteboardDaoTests.java @@ -266,11 +266,12 @@ public RenderHost getRenderHost() { RenderHost host = RenderHost.newBuilder() .setName(HOST) .setBootTime(1192369572) - .setFreeMcp(7602) + // The minimum amount of free space in the temporary directory to book a host. + .setFreeMcp(CueUtil.GB) .setFreeMem((int) Dispatcher.MEM_RESERVED_MIN * 4) .setFreeSwap(2076) .setLoad(1) - .setTotalMcp(19543) + .setTotalMcp(CueUtil.GB4) .setTotalMem((int) Dispatcher.MEM_RESERVED_MIN * 4) .setTotalSwap(2096) .setNimbyEnabled(true) diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/CoreUnitDispatcherGpuJobTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/CoreUnitDispatcherGpuJobTests.java index 4cc1c1f03..55bd44463 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/CoreUnitDispatcherGpuJobTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/CoreUnitDispatcherGpuJobTests.java @@ -100,11 +100,12 @@ public void createHost() { RenderHost host = RenderHost.newBuilder() .setName(HOSTNAME) .setBootTime(1192369572) - .setFreeMcp(76020) + // The minimum amount of free space in the temporary directory to book a host. + .setFreeMcp(CueUtil.GB) .setFreeMem((int) CueUtil.GB8) .setFreeSwap(20760) .setLoad(1) - .setTotalMcp(195430) + .setTotalMcp(CueUtil.GB4) .setTotalMem((int) CueUtil.GB8) .setTotalSwap((int) CueUtil.GB2) .setNimbyEnabled(false) diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/CoreUnitDispatcherGpuTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/CoreUnitDispatcherGpuTests.java index 0a4f6b74a..c61c9553f 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/CoreUnitDispatcherGpuTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/CoreUnitDispatcherGpuTests.java @@ -100,11 +100,12 @@ public void createHost() { RenderHost host = RenderHost.newBuilder() .setName(HOSTNAME) .setBootTime(1192369572) - .setFreeMcp(76020) + // The minimum amount of free space in the temporary directory to book a host. + .setFreeMcp(CueUtil.GB) .setFreeMem((int) CueUtil.GB8) .setFreeSwap(20760) .setLoad(1) - .setTotalMcp(195430) + .setTotalMcp(CueUtil.GB4) .setTotalMem((int) CueUtil.GB8) .setTotalSwap((int) CueUtil.GB2) .setNimbyEnabled(false) diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/CoreUnitDispatcherGpusJobTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/CoreUnitDispatcherGpusJobTests.java index 4972b8f9b..e2d1cb564 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/CoreUnitDispatcherGpusJobTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/CoreUnitDispatcherGpusJobTests.java @@ -112,11 +112,12 @@ public void createHost() { RenderHost host = RenderHost.newBuilder() .setName(HOSTNAME) .setBootTime(1192369572) - .setFreeMcp(76020) + // The minimum amount of free space in the temporary directory to book a host. + .setFreeMcp(CueUtil.GB) .setFreeMem((int) CueUtil.GB8) .setFreeSwap(20760) .setLoad(0) - .setTotalMcp(195430) + .setTotalMcp(CueUtil.GB4) .setTotalMem(CueUtil.GB8) .setTotalSwap(CueUtil.GB2) .setNimbyEnabled(false) diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/CoreUnitDispatcherTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/CoreUnitDispatcherTests.java index adb6c404d..89112dd69 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/CoreUnitDispatcherTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/CoreUnitDispatcherTests.java @@ -45,6 +45,7 @@ import com.imageworks.spcue.service.JobLauncher; import com.imageworks.spcue.service.JobManager; import com.imageworks.spcue.test.TransactionalTest; +import com.imageworks.spcue.util.CueUtil; import static org.junit.Assert.assertEquals; @@ -99,11 +100,12 @@ public void createHost() { RenderHost host = RenderHost.newBuilder() .setName(HOSTNAME) .setBootTime(1192369572) - .setFreeMcp(76020) + // The minimum amount of free space in the temporary directory to book a host. + .setFreeMcp(CueUtil.GB) .setFreeMem(53500) .setFreeSwap(20760) .setLoad(1) - .setTotalMcp(195430) + .setTotalMcp(CueUtil.GB4) .setTotalMem(8173264) .setTotalSwap(20960) .setNimbyEnabled(false) diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/DispatchSupportTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/DispatchSupportTests.java index 98c60fd9c..55a7806c0 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/DispatchSupportTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/DispatchSupportTests.java @@ -96,11 +96,12 @@ public void createHost() { RenderHost host = RenderHost.newBuilder() .setName(HOSTNAME) .setBootTime(1192369572) - .setFreeMcp(76020) + // The minimum amount of free space in the temporary directory to book a host. + .setFreeMcp(CueUtil.GB) .setFreeMem(53500) .setFreeSwap(20760) .setLoad(0) - .setTotalMcp(195430) + .setTotalMcp(CueUtil.GB4) .setTotalMem(8173264) .setTotalSwap(20960) .setNimbyEnabled(false) diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/FrameCompleteHandlerTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/FrameCompleteHandlerTests.java index f022fc687..d313c5293 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/FrameCompleteHandlerTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/FrameCompleteHandlerTests.java @@ -115,11 +115,12 @@ public void createHost() { RenderHost host = RenderHost.newBuilder() .setName(HOSTNAME) .setBootTime(1192369572) - .setFreeMcp(76020) + // The minimum amount of free space in the temporary directory to book a host. + .setFreeMcp(CueUtil.GB) .setFreeMem((int) CueUtil.GB8) .setFreeSwap(20760) .setLoad(0) - .setTotalMcp(195430) + .setTotalMcp(CueUtil.GB4) .setTotalMem(CueUtil.GB8) .setTotalSwap(CueUtil.GB2) .setNimbyEnabled(false) @@ -139,11 +140,12 @@ public void createHost() { RenderHost host2 = RenderHost.newBuilder() .setName(HOSTNAME2) .setBootTime(1192369572) - .setFreeMcp(76020) + // The minimum amount of free space in the temporary directory to book a host. + .setFreeMcp(CueUtil.GB) .setFreeMem((int) CueUtil.GB4) .setFreeSwap((int) CueUtil.GB4) .setLoad(0) - .setTotalMcp(195430) + .setTotalMcp(CueUtil.GB4) .setTotalMem((int) CueUtil.GB8) .setTotalSwap((int) CueUtil.GB8) .setNimbyEnabled(false) diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/HistoryControlTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/HistoryControlTests.java index 138a3f33c..de67ff26a 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/HistoryControlTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/HistoryControlTests.java @@ -102,11 +102,12 @@ public void createHost() { RenderHost host = RenderHost.newBuilder() .setName(HOSTNAME) .setBootTime(1192369572) - .setFreeMcp(76020) + // The minimum amount of free space in the temporary directory to book a host. + .setFreeMcp(CueUtil.GB) .setFreeMem((int) CueUtil.GB8) .setFreeSwap(20760) .setLoad(0) - .setTotalMcp(195430) + .setTotalMcp(CueUtil.GB4) .setTotalMem(CueUtil.GB8) .setTotalSwap(CueUtil.GB2) .setNimbyEnabled(false) diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/HostReportHandlerGpuTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/HostReportHandlerGpuTests.java index dee9d0792..ce1e98ae1 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/HostReportHandlerGpuTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/HostReportHandlerGpuTests.java @@ -81,11 +81,12 @@ private static RenderHost getRenderHost() { return RenderHost.newBuilder() .setName(HOSTNAME) .setBootTime(1192369572) - .setFreeMcp(76020) + // The minimum amount of free space in the temporary directory to book a host. + .setFreeMcp(CueUtil.GB) .setFreeMem(53500) .setFreeSwap(20760) .setLoad(0) - .setTotalMcp(195430) + .setTotalMcp(CueUtil.GB4) .setTotalMem(1048576L * 4096) .setTotalSwap(20960) .setNimbyEnabled(false) diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/HostReportHandlerTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/HostReportHandlerTests.java index d27f76c32..970b97a95 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/HostReportHandlerTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/HostReportHandlerTests.java @@ -22,6 +22,7 @@ import java.io.File; import java.sql.Timestamp; import java.util.List; +import java.util.concurrent.ThreadPoolExecutor; import javax.annotation.Resource; import org.junit.Before; @@ -31,6 +32,7 @@ import org.springframework.transaction.annotation.Transactional; import com.imageworks.spcue.AllocationEntity; +import com.imageworks.spcue.CommentDetail; import com.imageworks.spcue.DispatchHost; import com.imageworks.spcue.dispatcher.Dispatcher; import com.imageworks.spcue.dispatcher.HostReportHandler; @@ -43,6 +45,7 @@ import com.imageworks.spcue.grpc.report.RenderHost; import com.imageworks.spcue.grpc.report.RunningFrameInfo; import com.imageworks.spcue.service.AdminManager; +import com.imageworks.spcue.service.CommentManager; import com.imageworks.spcue.service.HostManager; import com.imageworks.spcue.service.JobLauncher; import com.imageworks.spcue.service.JobManager; @@ -50,7 +53,10 @@ import com.imageworks.spcue.util.CueUtil; import com.imageworks.spcue.VirtualProc; +import java.util.UUID; + import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; @ContextConfiguration public class HostReportHandlerTests extends TransactionalTest { @@ -73,8 +79,16 @@ public class HostReportHandlerTests extends TransactionalTest { @Resource JobManager jobManager; + @Resource + CommentManager commentManager; + private static final String HOSTNAME = "beta"; private static final String NEW_HOSTNAME = "gamma"; + private String hostname; + private String hostname2; + private static final String SUBJECT_COMMENT_FULL_TEMP_DIR = "Host set to REPAIR for not having enough storage " + + "space on the temporary directory (mcp)"; + private static final String CUEBOT_COMMENT_USER = "cuebot"; @Before public void setTestMode() { @@ -83,7 +97,11 @@ public void setTestMode() { @Before public void createHost() { - hostManager.createHost(getRenderHost(), + hostname = UUID.randomUUID().toString().substring(0, 8); + hostname2 = UUID.randomUUID().toString().substring(0, 8); + hostManager.createHost(getRenderHost(hostname, HardwareState.UP), + adminManager.findAllocationDetail("spi","general")); + hostManager.createHost(getRenderHost(hostname2, HardwareState.UP), adminManager.findAllocationDetail("spi","general")); } @@ -96,26 +114,51 @@ private static CoreDetail getCoreDetail(int total, int idle, int booked, int loc .build(); } - private DispatchHost getHost() { - return hostManager.findDispatchHost(HOSTNAME); + private DispatchHost getHost(String hostname) { + return hostManager.findDispatchHost(hostname); } - private static RenderHost getRenderHost() { + private static RenderHost getRenderHost(String hostname, HardwareState state) { return RenderHost.newBuilder() - .setName(HOSTNAME) + .setName(hostname) .setBootTime(1192369572) - .setFreeMcp(76020) + // The minimum amount of free space in the temporary directory to book a host. + .setFreeMcp(CueUtil.GB) .setFreeMem((int) CueUtil.GB8) .setFreeSwap(20760) .setLoad(0) - .setTotalMcp(195430) + .setTotalMcp(CueUtil.GB4) .setTotalMem(CueUtil.GB8) .setTotalSwap(CueUtil.GB2) .setNimbyEnabled(false) .setNumProcs(2) .setCoresPerProc(100) .addTags("test") - .setState(HardwareState.UP) + .setState(state) + .setFacility("spi") + .putAttributes("SP_OS", "Linux") + .setFreeGpuMem((int) CueUtil.MB512) + .setTotalGpuMem((int) CueUtil.MB512) + .build(); + } + + private static RenderHost getRenderHost(String hostname, HardwareState state, Long freeTempDir) { + return RenderHost.newBuilder() + .setName(hostname) + .setBootTime(1192369572) + // The minimum amount of free space in the temporary directory to book a host. + .setFreeMcp(freeTempDir) + .setFreeMem((int) CueUtil.GB8) + .setFreeSwap(20760) + .setLoad(0) + .setTotalMcp(freeTempDir * 4) + .setTotalMem(CueUtil.GB8) + .setTotalSwap(CueUtil.GB2) + .setNimbyEnabled(false) + .setNumProcs(2) + .setCoresPerProc(100) + .addTags("test") + .setState(state) .setFacility("spi") .putAttributes("SP_OS", "Linux") .setFreeGpuMem((int) CueUtil.MB512) @@ -127,11 +170,12 @@ private static RenderHost getNewRenderHost(String tags) { return RenderHost.newBuilder() .setName(NEW_HOSTNAME) .setBootTime(1192369572) - .setFreeMcp(76020) + // The minimum amount of free space in the temporary directory to book a host. + .setFreeMcp(CueUtil.GB) .setFreeMem(53500) .setFreeSwap(20760) .setLoad(0) - .setTotalMcp(195430) + .setTotalMcp(CueUtil.GB4) .setTotalMem(8173264) .setTotalSwap(20960) .setNimbyEnabled(false) @@ -149,17 +193,40 @@ private static RenderHost getNewRenderHost(String tags) { @Test @Transactional @Rollback(true) - public void testHandleHostReport() { - boolean isBoot = false; + public void testHandleHostReport() throws InterruptedException { CoreDetail cores = getCoreDetail(200, 200, 0, 0); - HostReport report = HostReport.newBuilder() - .setHost(getRenderHost()) + HostReport report1 = HostReport.newBuilder() + .setHost(getRenderHost(hostname, HardwareState.UP)) .setCoreInfo(cores) .build(); + HostReport report2 = HostReport.newBuilder() + .setHost(getRenderHost(hostname2, HardwareState.UP)) + .setCoreInfo(cores) + .build(); + HostReport report1_2 = HostReport.newBuilder() + .setHost(getRenderHost(hostname, HardwareState.UP)) + .setCoreInfo(getCoreDetail(200, 200, 100, 0)) + .build(); - hostReportHandler.handleHostReport(report, isBoot); - DispatchHost host = getHost(); - assertEquals(host.lockState, LockState.OPEN); + hostReportHandler.handleHostReport(report1, false); + DispatchHost host = getHost(hostname); + assertEquals(LockState.OPEN, host.lockState); + assertEquals(HardwareState.UP, host.hardwareState); + hostReportHandler.handleHostReport(report1_2, false); + host = getHost(hostname); + assertEquals(HardwareState.UP, host.hardwareState); + + // Test Queue thread handling + ThreadPoolExecutor queue = hostReportHandler.getReportQueue(); + // Make sure jobs flow normally without any nullpointer exception + // Expecting results from a ThreadPool based class on JUnit is tricky + // A future test will be developed in the future to better address the behavior of + // this feature + hostReportHandler.queueHostReport(report1); // HOSTNAME + hostReportHandler.queueHostReport(report2); // HOSTNAME2 + hostReportHandler.queueHostReport(report1); // HOSTNAME + hostReportHandler.queueHostReport(report1); // HOSTNAME + hostReportHandler.queueHostReport(report1_2); // HOSTNAME } @Test @@ -228,6 +295,138 @@ public void testHandleHostReportWithNonExistentTags() { assertEquals(host.getAllocationId(), alloc.id); } + @Test + @Transactional + @Rollback(true) + public void testHandleHostReportWithFullTemporaryDirectories() { + // Create CoreDetail + CoreDetail cores = getCoreDetail(200, 200, 0, 0); + + /* + * Test 1: + * Precondition: + * - HardwareState=UP + * Action: + * - Receives a HostReport with freeTempDir < dispatcher.min_bookable_free_temp_dir_kb (opencue.properties) + * Postcondition: + * - Host hardwareState changes to REPAIR + * - A comment is created with subject=SUBJECT_COMMENT_FULL_TEMP_DIR and user=CUEBOT_COMMENT_USER + * */ + // Create HostReport + HostReport report1 = HostReport.newBuilder() + .setHost(getRenderHost(hostname, HardwareState.UP, 1024L)) + .setCoreInfo(cores) + .build(); + // Call handleHostReport() => Create the comment with subject=SUBJECT_COMMENT_FULL_TEMP_DIR and change the + // host's hardwareState to REPAIR + hostReportHandler.handleHostReport(report1, false); + // Get host + DispatchHost host = getHost(hostname); + // Get list of comments by host, user, and subject + List comments = commentManager.getCommentsByHostUserAndSubject(host, CUEBOT_COMMENT_USER, + SUBJECT_COMMENT_FULL_TEMP_DIR); + // Check if there is 1 comment + assertEquals(comments.size(), 1); + // Get host comment + CommentDetail comment = comments.get(0); + // Check if the comment has the user = CUEBOT_COMMENT_USER + assertEquals(comment.user, CUEBOT_COMMENT_USER); + // Check if the comment has the subject = SUBJECT_COMMENT_FULL_TEMP_DIR + assertEquals(comment.subject, SUBJECT_COMMENT_FULL_TEMP_DIR); + // Check host lock state + assertEquals(LockState.OPEN, host.lockState); + // Check if host hardware state is REPAIR + assertEquals(HardwareState.REPAIR, host.hardwareState); + // Test Queue thread handling + ThreadPoolExecutor queue = hostReportHandler.getReportQueue(); + // Make sure jobs flow normally without any nullpointer exception + hostReportHandler.queueHostReport(report1); // HOSTNAME + hostReportHandler.queueHostReport(report1); // HOSTNAME + + /* + * Test 2: + * Precondition: + * - HardwareState=REPAIR + * - There is a comment for the host with subject=SUBJECT_COMMENT_FULL_TEMP_DIR and user=CUEBOT_COMMENT_USER + * Action: + * - Receives a HostReport with freeTempDir >= dispatcher.min_bookable_free_temp_dir_kb (opencue.properties) + * Postcondition: + * - Host hardwareState changes to UP + * - Comment with subject=SUBJECT_COMMENT_FULL_TEMP_DIR and user=CUEBOT_COMMENT_USER gets deleted + * */ + // Set the host freeTempDir to the minimum size required = 1GB (1048576 KB) + HostReport report2 = HostReport.newBuilder() + .setHost(getRenderHost(hostname, HardwareState.UP, 1048576L)) + .setCoreInfo(cores) + .build(); + // Call handleHostReport() => Delete the comment with subject=SUBJECT_COMMENT_FULL_TEMP_DIR and change the + // host's hardwareState to UP + hostReportHandler.handleHostReport(report2, false); + // Get host + host = getHost(hostname); + // Get list of comments by host, user, and subject + comments = commentManager.getCommentsByHostUserAndSubject(host, CUEBOT_COMMENT_USER, + SUBJECT_COMMENT_FULL_TEMP_DIR); + // Check if there is no comment associated with the host + assertEquals(comments.size(), 0); + // Check host lock state + assertEquals(LockState.OPEN, host.lockState); + // Check if host hardware state is UP + assertEquals(HardwareState.UP, host.hardwareState); + // Test Queue thread handling + queue = hostReportHandler.getReportQueue(); + // Make sure jobs flow normally without any nullpointer exception + hostReportHandler.queueHostReport(report1); // HOSTNAME + hostReportHandler.queueHostReport(report1); // HOSTNAME + } + + @Test + @Transactional + @Rollback(true) + public void testHandleHostReportWithHardwareStateRepairNotRelatedToFullTempDir() { + // Create CoreDetail + CoreDetail cores = getCoreDetail(200, 200, 0, 0); + + /* + * Test if host.hardwareState == HardwareState.REPAIR + * (Not related to freeMcp < dispatcher.min_bookable_free_mcp_kb (opencue.properties)) + * + * - There is no comment with subject=SUBJECT_COMMENT_FULL_MCP_DIR and user=CUEBOT_COMMENT_USER associated with + * the host + * The host.hardwareState continue as HardwareState.REPAIR + * */ + // Create HostReport + HostReport report = HostReport.newBuilder() + .setHost(getRenderHost(hostname, HardwareState.UP, 1048576L)) + .setCoreInfo(cores) + .build(); + // Get host + DispatchHost host = getHost(hostname); + // Host's HardwareState set to REPAIR + hostManager.setHostState(host, HardwareState.REPAIR); + host.hardwareState = HardwareState.REPAIR; + // Get list of comments by host, user, and subject + List hostComments = commentManager.getCommentsByHostUserAndSubject(host, CUEBOT_COMMENT_USER, + SUBJECT_COMMENT_FULL_TEMP_DIR); + // Check if there is no comment + assertEquals(hostComments.size(), 0); + // There is no comment to delete + boolean commentsDeleted = commentManager.deleteCommentByHostUserAndSubject(host, + CUEBOT_COMMENT_USER, SUBJECT_COMMENT_FULL_TEMP_DIR); + assertFalse(commentsDeleted); + // Call handleHostReport() + hostReportHandler.handleHostReport(report, false); + // Check host lock state + assertEquals(LockState.OPEN, host.lockState); + // Check if host hardware state is REPAIR + assertEquals(HardwareState.REPAIR, host.hardwareState); + // Test Queue thread handling + ThreadPoolExecutor queueThread = hostReportHandler.getReportQueue(); + // Make sure jobs flow normally without any nullpointer exception + hostReportHandler.queueHostReport(report); // HOSTNAME + hostReportHandler.queueHostReport(report); // HOSTNAME + } + @Test @Transactional @Rollback(true) @@ -235,7 +434,7 @@ public void testMemoryAndLlu() { jobLauncher.testMode = true; jobLauncher.launch(new File("src/test/resources/conf/jobspec/jobspec_simple.xml")); - DispatchHost host = getHost(); + DispatchHost host = getHost(hostname); List procs = dispatcher.dispatchHost(host); assertEquals(1, procs.size()); VirtualProc proc = procs.get(0); @@ -252,7 +451,7 @@ public void testMemoryAndLlu() { .setMaxRss(420000) .build(); HostReport report = HostReport.newBuilder() - .setHost(getRenderHost()) + .setHost(getRenderHost(hostname, HardwareState.UP)) .setCoreInfo(cores) .addFrames(info) .build(); diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/LocalDispatcherTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/LocalDispatcherTests.java index 97a270085..a7218b47a 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/LocalDispatcherTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/LocalDispatcherTests.java @@ -95,11 +95,12 @@ public void createHost() { RenderHost host = RenderHost.newBuilder() .setName(HOSTNAME) .setBootTime(1192369572) - .setFreeMcp(76020) + // The minimum amount of free space in the temporary directory to book a host. + .setFreeMcp(CueUtil.GB) .setFreeMem(53500) .setFreeSwap(20760) .setLoad(0) - .setTotalMcp(195430) + .setTotalMcp(CueUtil.GB4) .setTotalMem(8173264) .setTotalSwap(20960) .setNimbyEnabled(false) diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/RedirectManagerTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/RedirectManagerTests.java index 25ccf69c5..70e3db4af 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/RedirectManagerTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/RedirectManagerTests.java @@ -60,6 +60,7 @@ import com.imageworks.spcue.service.RedirectService; import com.imageworks.spcue.service.Whiteboard; import com.imageworks.spcue.util.Convert; +import com.imageworks.spcue.util.CueUtil; import static org.hamcrest.Matchers.emptyString; import static org.hamcrest.Matchers.is; @@ -137,11 +138,12 @@ public void createHost() { RenderHost host = RenderHost.newBuilder() .setName(HOSTNAME) .setBootTime(1192369572) - .setFreeMcp(76020) + // The minimum amount of free space in the temporary directory to book a host. + .setFreeMcp(CueUtil.GB) .setFreeMem(53500) .setFreeSwap(20760) .setLoad(1) - .setTotalMcp(195430) + .setTotalMcp(CueUtil.GB4) .setTotalMem(8173264) .setTotalSwap(20960) .setNimbyEnabled(false) diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/StrandedCoreTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/StrandedCoreTests.java index 4211c9866..7d02d44e8 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/StrandedCoreTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/StrandedCoreTests.java @@ -43,6 +43,7 @@ import com.imageworks.spcue.service.JobLauncher; import com.imageworks.spcue.service.JobManager; import com.imageworks.spcue.test.TransactionalTest; +import com.imageworks.spcue.util.CueUtil; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -98,11 +99,12 @@ public void createHost() { RenderHost host = RenderHost.newBuilder() .setName(HOSTNAME) .setBootTime(1192369572) - .setFreeMcp(76020) + // The minimum amount of free space in the temporary directory to book a host. + .setFreeMcp(CueUtil.GB) .setFreeMem(53500) .setFreeSwap(20760) .setLoad(1) - .setTotalMcp(195430) + .setTotalMcp(CueUtil.GB4) .setTotalMem(8173264) .setTotalSwap(20960) .setNimbyEnabled(false) diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/TestBookingQueue.java b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/TestBookingQueue.java index 74b21c102..7654570a0 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/TestBookingQueue.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/TestBookingQueue.java @@ -64,11 +64,12 @@ public void create() { RenderHost host = RenderHost.newBuilder() .setName(HOSTNAME) .setBootTime(1192369572) - .setFreeMcp(76020) + // The minimum amount of free space in the temporary directory to book a host. + .setFreeMcp(CueUtil.GB) .setFreeMem(53500) .setFreeSwap(20760) .setLoad(1) - .setTotalMcp(195430) + .setTotalMcp(CueUtil.GB4) .setTotalMem(8173264) .setTotalSwap(20960) .setNimbyEnabled(false) diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/service/BookingManagerTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/service/BookingManagerTests.java index 9b6813c33..1e894eb1c 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/service/BookingManagerTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/service/BookingManagerTests.java @@ -112,11 +112,12 @@ public DispatchHost createHost() { RenderHost host = RenderHost.newBuilder() .setName("test_host") .setBootTime(1192369572) - .setFreeMcp(76020) + // The minimum amount of free space in the temporary directory to book a host. + .setFreeMcp(CueUtil.GB) .setFreeMem(53500) .setFreeSwap(20760) .setLoad(1) - .setTotalMcp(195430) + .setTotalMcp(CueUtil.GB4) .setTotalMem((int) CueUtil.GB16) .setTotalSwap((int) CueUtil.GB16) .setNimbyEnabled(false) diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/service/HostManagerTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/service/HostManagerTests.java index cf86e5362..29970441d 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/service/HostManagerTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/service/HostManagerTests.java @@ -101,11 +101,12 @@ public DispatchHost createHost() { RenderHost host = RenderHost.newBuilder() .setName(HOST_NAME) .setBootTime(1192369572) - .setFreeMcp(7602) + // The minimum amount of free space in the temporary directory to book a host. + .setFreeMcp(CueUtil.GB) .setFreeMem(15290520) .setFreeSwap(2076) .setLoad(1) - .setTotalMcp(19543) + .setTotalMcp(CueUtil.GB4) .setTotalMem((int) CueUtil.GB16) .setTotalSwap(2076) .setNimbyEnabled(true) diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/service/JobManagerTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/service/JobManagerTests.java index 3be56bf06..2ea9b5dde 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/service/JobManagerTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/service/JobManagerTests.java @@ -129,11 +129,12 @@ public DispatchHost createHost() { RenderHost host = RenderHost.newBuilder() .setName("test_host") .setBootTime(1192369572) - .setFreeMcp(76020) + // The minimum amount of free space in the temporary directory to book a host. + .setFreeMcp(CueUtil.GB) .setFreeMem(53500) .setFreeSwap(20760) .setLoad(1) - .setTotalMcp(195430) + .setTotalMcp(CueUtil.GB4) .setTotalMem((int) CueUtil.GB16) .setTotalSwap((int) CueUtil.GB16) .setNimbyEnabled(false) diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/service/OwnerManagerTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/service/OwnerManagerTests.java index 224dcac75..51dcafec4 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/service/OwnerManagerTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/service/OwnerManagerTests.java @@ -69,11 +69,12 @@ public DispatchHost createHost() { RenderHost host = RenderHost.newBuilder() .setName("test_host") .setBootTime(1192369572) - .setFreeMcp(76020) + // The minimum amount of free space in the temporary directory to book a host. + .setFreeMcp(CueUtil.GB) .setFreeMem(53500) .setFreeSwap(20760) .setLoad(1) - .setTotalMcp(195430) + .setTotalMcp(CueUtil.GB4) .setTotalMem((int) CueUtil.GB16) .setTotalSwap((int) CueUtil.GB16) .setNimbyEnabled(true) diff --git a/cuebot/src/test/resources/opencue.properties b/cuebot/src/test/resources/opencue.properties index 334408470..00d0c4463 100644 --- a/cuebot/src/test/resources/opencue.properties +++ b/cuebot/src/test/resources/opencue.properties @@ -64,3 +64,9 @@ dispatcher.kill_queue.queue_capacity=1000 dispatcher.booking_queue.core_pool_size=6 dispatcher.booking_queue.max_pool_size=6 dispatcher.booking_queue.queue_capacity=1000 + +# The minimum amount of free space in the temporary directory (mcp) to book a host. +# E.g: 1G = 1048576 kB => dispatcher.min_bookable_free_temp_dir_kb=1048576 +# Default = 1G = 1048576 kB +# If equals to -1, it means the feature is turned off +dispatcher.min_bookable_free_temp_dir_kb=1048576 \ No newline at end of file