diff --git a/hystrix-contrib/hystrix-javanica/src/test/java/com/netflix/hystrix/contrib/javanica/test/common/observable/BasicObservableTest.java b/hystrix-contrib/hystrix-javanica/src/test/java/com/netflix/hystrix/contrib/javanica/test/common/observable/BasicObservableTest.java index a6e88517e..9125b6fb9 100644 --- a/hystrix-contrib/hystrix-javanica/src/test/java/com/netflix/hystrix/contrib/javanica/test/common/observable/BasicObservableTest.java +++ b/hystrix-contrib/hystrix-javanica/src/test/java/com/netflix/hystrix/contrib/javanica/test/common/observable/BasicObservableTest.java @@ -53,7 +53,6 @@ public void setUp() throws Exception { public void testGetUserByIdSuccess() { // blocking Observable observable = userService.getUser("1", "name: "); - assertObservableExecutionMode(observable, ObservableExecutionMode.EAGER); assertEquals("name: 1", observable.toBlocking().single().getName()); // non-blocking @@ -96,7 +95,6 @@ public void call(User user) { public void testGetUserWithRegularFallback() { final User exUser = new User("def", "def"); Observable userObservable = userService.getUserRegularFallback(" ", ""); - assertObservableExecutionMode(userObservable, ObservableExecutionMode.LAZY); // blocking assertEquals(exUser, userObservable.toBlocking().single()); assertEquals(1, HystrixRequestLog.getCurrentRequest().getAllExecutedCommands().size()); @@ -127,7 +125,6 @@ public void testGetUserWithRxCommandFallback() { // blocking Observable userObservable = userService.getUserRxCommandFallback(" ", ""); - assertObservableExecutionMode(userObservable, ObservableExecutionMode.LAZY); assertEquals(exUser, userObservable.toBlocking().single()); assertEquals(2, HystrixRequestLog.getCurrentRequest().getAllExecutedCommands().size()); com.netflix.hystrix.HystrixInvokableInfo getUserRxCommandFallback = getHystrixCommandByKey("getUserRxCommandFallback"); @@ -140,20 +137,6 @@ public void testGetUserWithRxCommandFallback() { } - private static void assertObservableExecutionMode(Observable observable, ObservableExecutionMode mode) { - // todo find better way to figure it out - boolean eager = observable instanceof ReplaySubject; - if (ObservableExecutionMode.EAGER == mode) { - if (!eager) { - throw new AssertionError("observable must be instance of ReplaySubject"); - } - } else { - if (eager) { - throw new AssertionError("observable must not be instance of ReplaySubject"); - } - } - } - public static class UserService { private User regularFallback(String id, String name) { diff --git a/hystrix-core/src/main/java/com/netflix/hystrix/AbstractCommand.java b/hystrix-core/src/main/java/com/netflix/hystrix/AbstractCommand.java index 3a66bca30..0b4a23c3c 100644 --- a/hystrix-core/src/main/java/com/netflix/hystrix/AbstractCommand.java +++ b/hystrix-core/src/main/java/com/netflix/hystrix/AbstractCommand.java @@ -37,9 +37,9 @@ import org.slf4j.LoggerFactory; import rx.Notification; import rx.Observable; -import rx.Observable.OnSubscribe; import rx.Observable.Operator; import rx.Subscriber; +import rx.Subscription; import rx.functions.Action0; import rx.functions.Action1; import rx.functions.Func0; @@ -93,15 +93,28 @@ protected static enum TimedOutStatus { protected final AtomicReference> timeoutTimer = new AtomicReference>(); - protected AtomicBoolean started = new AtomicBoolean(); + protected final AtomicBoolean commandStarted = new AtomicBoolean(); + protected volatile boolean isExecutionComplete = false; - /* result of execution (if this command instance actually gets executed, which may not occur due to request caching) */ - protected volatile ExecutionResult executionResult = ExecutionResult.EMPTY; + /* + * {@link ExecutionResult} refers to what happened as the user-provided code ran. If request-caching is used, + * then multiple command instances will have a reference to the same {@link ExecutionResult}. So all values there + * should be the same, even in the presence of request-caching. + * + * If some values are not properly shareable, then they belong on the command instance, so they are not visible to + * other commands. + * + * Examples: RESPONSE_FROM_CACHE, CANCELLED HystrixEventTypes + */ + protected volatile ExecutionResult executionResult = ExecutionResult.EMPTY; //state on shared execution + + protected volatile boolean isResponseFromCache = false; + protected volatile ExecutionResult executionResultAtTimeOfCancellation; + protected volatile long commandStartTimestamp = -1L; /* If this command executed and timed-out */ protected final AtomicReference isCommandTimedOut = new AtomicReference(TimedOutStatus.NOT_EXECUTED); - protected final AtomicBoolean isExecutionComplete = new AtomicBoolean(false); - protected final AtomicReference endCurrentThreadExecutingCommand = new AtomicReference(); // don't like how this is being done + protected volatile Action0 endCurrentThreadExecutingCommand; /** * Instance of RequestCache logic @@ -132,8 +145,6 @@ protected static enum TimedOutStatus { return name; } - - protected AbstractCommand(HystrixCommandGroupKey group, HystrixCommandKey key, HystrixThreadPoolKey threadPoolKey, HystrixCircuitBreaker circuitBreaker, HystrixThreadPool threadPool, HystrixCommandProperties.Setter commandPropertiesDefaults, HystrixThreadPoolProperties.Setter threadPoolPropertiesDefaults, HystrixCommandMetrics metrics, TryableSemaphore fallbackSemaphore, TryableSemaphore executionSemaphore, @@ -306,9 +317,14 @@ public Observable observe() { // us a ReplaySubject to buffer the eagerly subscribed-to Observable ReplaySubject subject = ReplaySubject.create(); // eagerly kick off subscription - toObservable().subscribe(subject); + final Subscription sourceSubscription = toObservable().subscribe(subject); // return the subject that can be subscribed to later while the execution has already started - return subject; + return subject.doOnUnsubscribe(new Action0() { + @Override + public void call() { + sourceSubscription.unsubscribe(); + } + }); } protected abstract Observable getExecutionObservable(); @@ -339,234 +355,152 @@ public Observable observe() { */ public Observable toObservable() { /* this is a stateful object so can only be used once */ - if (!started.compareAndSet(false, true)) { + if (!commandStarted.compareAndSet(false, true)) { throw new IllegalStateException("This instance can only be executed once. Please instantiate a new instance."); } - final HystrixInvokable _this = this; - final boolean requestCacheEnabled = isRequestCachingEnabled(); + commandStartTimestamp = System.currentTimeMillis(); - /* try from cache first */ - if (requestCacheEnabled) { - Observable fromCache = requestCache.get(getCacheKey()); - if (fromCache != null) { - long latency = System.currentTimeMillis() - executionResult.getStartTimestamp(); - executionResult = executionResult.markUserThreadCompletion((int) latency); - executionResult = executionResult.addEvent((int) latency, HystrixEventType.RESPONSE_FROM_CACHE); - metrics.markCommandDone(executionResult, commandKey, threadPoolKey); - eventNotifier.markEvent(HystrixEventType.RESPONSE_FROM_CACHE, commandKey); - isExecutionComplete.set(true); - try { - executionHook.onCacheHit(this); - } catch (Throwable hookEx) { - logger.warn("Error calling HystrixCommandExecutionHook.onCacheHit", hookEx); - } - return new CachedObservableResponse((CachedObservableOriginal) fromCache, this); + if (properties.requestLogEnabled().get()) { + // log this command execution regardless of what happened + if (currentRequestLog != null) { + currentRequestLog.addExecutedCommand(this); } } - // create an Observable that will lazily execute when subscribed to - Observable o = Observable.create(new OnSubscribe() { + final boolean requestCacheEnabled = isRequestCachingEnabled(); - @Override - public void call(Subscriber observer) { - // async record keeping - recordExecutedCommand(); - - // mark that we're starting execution on the ExecutionHook - // if this hook throws an exception, then a fast-fail occurs with no fallback. No state is left inconsistent - executionHook.onStart(_this); - - /* determine if we're allowed to execute */ - if (circuitBreaker.allowRequest()) { - final TryableSemaphore executionSemaphore = getExecutionSemaphore(); - final AtomicBoolean semaphoreHasBeenReleased = new AtomicBoolean(false); - // acquire a permit - if (executionSemaphore.tryAcquire()) { - try { - /* used to track userThreadExecutionTime */ - executionResult = executionResult.setInvocationStartTime(System.currentTimeMillis()); - getRunObservableDecoratedForMetricsAndErrorHandling() - .doOnTerminate(new Action0() { - - @Override - public void call() { - // release the semaphore - // this is done here instead of below so that the acquire/release happens where it is guaranteed - // and not affected by the conditional circuit-breaker checks, timeouts, etc - if (semaphoreHasBeenReleased.compareAndSet(false, true)) { - executionSemaphore.release(); - } - - } - }) - .doOnUnsubscribe(new Action0() { - @Override - public void call() { - if (semaphoreHasBeenReleased.compareAndSet(false, true)) { - executionSemaphore.release(); - } - } - }) - .unsafeSubscribe(observer); - } catch (RuntimeException e) { - observer.onError(e); - } - } else { - Exception semaphoreRejectionException = new RuntimeException("could not acquire a semaphore for execution"); - executionResult = executionResult.setExecutionException(semaphoreRejectionException); - eventNotifier.markEvent(HystrixEventType.SEMAPHORE_REJECTED, commandKey); - logger.debug("HystrixCommand Execution Rejection by Semaphore."); // debug only since we're throwing the exception and someone higher will do something with it - // retrieve a fallback or throw an exception if no fallback available - getFallbackOrThrowException(HystrixEventType.SEMAPHORE_REJECTED, FailureType.REJECTED_SEMAPHORE_EXECUTION, - "could not acquire a semaphore for execution", semaphoreRejectionException) - .lift(new DeprecatedOnCompleteWithValueHookApplication(_this)) - .unsafeSubscribe(observer); - } - } else { - // record that we are returning a short-circuited fallback - eventNotifier.markEvent(HystrixEventType.SHORT_CIRCUITED, commandKey); - // short-circuit and go directly to fallback (or throw an exception if no fallback implemented) - Exception shortCircuitException = new RuntimeException("Hystrix circuit short-circuited and is OPEN"); - executionResult = executionResult.setExecutionException(shortCircuitException); - try { - getFallbackOrThrowException(HystrixEventType.SHORT_CIRCUITED, FailureType.SHORTCIRCUIT, - "short-circuited", shortCircuitException) - .lift(new DeprecatedOnCompleteWithValueHookApplication(_this)) - .unsafeSubscribe(observer); - } catch (Exception e) { - observer.onError(e); - } - } + final AbstractCommand _cmd = this; + + /* try from cache first */ + if (requestCacheEnabled) { + HystrixCachedObservable fromCache = requestCache.get(getCacheKey()); + if (fromCache != null) { + isResponseFromCache = true; + return handleRequestCacheHitAndEmitValues(fromCache); } - }); + } - //apply all lifecycle hooks - o = o.lift(new CommandHookApplication(this)); + // ensure that cleanup code only runs exactly once + final AtomicBoolean commandCleanupExecuted = new AtomicBoolean(false); - // error handling at very end (this means fallback didn't exist or failed) - o = o.onErrorResumeNext(new Func1>() { + //doOnCompleted handler already did all of the SUCCESS work + //doOnError handler already did all of the FAILURE/TIMEOUT/REJECTION/BAD_REQUEST work + final Action0 terminateCommandCleanup = new Action0() { @Override - public Observable call(Throwable t) { - // count that we are throwing an exception and re-throw it - eventNotifier.markEvent(HystrixEventType.EXCEPTION_THROWN, commandKey); - return Observable.error(t); + public void call() { + if (commandCleanupExecuted.compareAndSet(false, true)) { + isExecutionComplete = true; + handleCommandEnd(); + } } + }; - }); - - final AtomicBoolean commandCleanupExecuted = new AtomicBoolean(false); - final Action0 commandCleanup = new Action0() { - + //mark the command as CANCELLED and store the latency (in addition to standard cleanup) + final Action0 unsubscribeCommandCleanup = new Action0() { @Override public void call() { if (commandCleanupExecuted.compareAndSet(false, true)) { - Reference tl = timeoutTimer.get(); - if (tl != null) { - tl.clear(); - } - - long userThreadLatency = System.currentTimeMillis() - executionResult.getStartTimestamp(); - executionResult = executionResult.markUserThreadCompletion((int) userThreadLatency); - metrics.markCommandDone(executionResult, commandKey, threadPoolKey); - // record that we're completed - isExecutionComplete.set(true); + eventNotifier.markEvent(HystrixEventType.CANCELLED, commandKey); + executionResultAtTimeOfCancellation = executionResult + .addEvent((int) (System.currentTimeMillis() - commandStartTimestamp), HystrixEventType.CANCELLED); + handleCommandEnd(); } } + }; + final Func0> applyHystrixSemantics = new Func0>() { + @Override + public Observable call() { + return applyHystrixSemantics(_cmd); + } }; - // any final cleanup needed - o = o.doOnTerminate(commandCleanup).doOnUnsubscribe(commandCleanup); + Observable hystrixObservable = + Observable.defer(applyHystrixSemantics). + lift(new CommandHookApplication(this)); + + Observable afterCache; // put in cache if (requestCacheEnabled) { // wrap it for caching - o = new CachedObservableOriginal(o.cache(), this); - Observable fromCache = requestCache.putIfAbsent(getCacheKey(), o); + HystrixCachedObservable toCache = HystrixCachedObservable.from(hystrixObservable, this); + HystrixCachedObservable fromCache = requestCache.putIfAbsent(getCacheKey(), toCache); if (fromCache != null) { // another thread beat us so we'll use the cached value instead - o = new CachedObservableResponse((CachedObservableOriginal) fromCache, this); + toCache.originalSubscription.unsubscribe(); + isResponseFromCache = true; + return handleRequestCacheHitAndEmitValues(fromCache); + } else { + // we just created an ObservableCommand so we cast and return it + afterCache = toCache.toObservable(); } - // we just created an ObservableCommand so we cast and return it - return o; } else { - // no request caching so a simple wrapper just to pass 'this' along with the Observable - return new ObservableCommand(o, this); + afterCache = hystrixObservable; } - } - /** - * This decorate "Hystrix" functionality around the run() Observable. - * - * @return R - */ - private Observable getRunObservableDecoratedForMetricsAndErrorHandling() { - final AbstractCommand _self = this; + return afterCache + .doOnTerminate(terminateCommandCleanup) // perform cleanup once (either on normal terminal state (this line), or unsubscribe (next line)) + .doOnUnsubscribe(unsubscribeCommandCleanup); // perform cleanup once + } - final HystrixRequestContext currentRequestContext = HystrixRequestContext.getContextForCurrentThread(); - Observable run; - if (properties.executionIsolationStrategy().get().equals(ExecutionIsolationStrategy.THREAD)) { - // mark that we are executing in a thread (even if we end up being rejected we still were a THREAD execution and not SEMAPHORE) - run = Observable.create(new OnSubscribe() { + private Observable applyHystrixSemantics(final AbstractCommand _cmd) { + // mark that we're starting execution on the ExecutionHook + // if this hook throws an exception, then a fast-fail occurs with no fallback. No state is left inconsistent + executionHook.onStart(_cmd); + /* determine if we're allowed to execute */ + if (circuitBreaker.allowRequest()) { + final TryableSemaphore executionSemaphore = getExecutionSemaphore(); + final AtomicBoolean semaphoreHasBeenReleased = new AtomicBoolean(false); + final Action0 singleSemaphoreRelease = new Action0() { @Override - public void call(Subscriber s) { - metrics.markCommandStart(commandKey, threadPoolKey, ExecutionIsolationStrategy.THREAD); - - if (isCommandTimedOut.get() == TimedOutStatus.TIMED_OUT) { - // the command timed out in the wrapping thread so we will return immediately - // and not increment any of the counters below or other such logic - s.onError(new RuntimeException("timed out before executing run()")); - } else { - // not timed out so execute - HystrixCounters.incrementGlobalConcurrentThreads(); - threadPool.markThreadExecution(); - // store the command that is being run - endCurrentThreadExecutingCommand.set(Hystrix.startCurrentThreadExecutingCommand(getCommandKey())); - executionResult = executionResult.setExecutedInThread(); - /** - * If any of these hooks throw an exception, then it appears as if the actual execution threw an error - */ - try { - executionHook.onThreadStart(_self); - executionHook.onRunStart(_self); - executionHook.onExecutionStart(_self); - getExecutionObservableWithLifecycle().unsafeSubscribe(s); - } catch (Throwable ex) { - s.onError(ex); - } + public void call() { + if (semaphoreHasBeenReleased.compareAndSet(false, true)) { + executionSemaphore.release(); } } - }).subscribeOn(threadPool.getScheduler(new Func0() { + }; + final Action1 markExceptionThrown = new Action1() { @Override - public Boolean call() { - return properties.executionIsolationThreadInterruptOnTimeout().get() && _self.isCommandTimedOut.get().equals(TimedOutStatus.TIMED_OUT); + public void call(Throwable t) { + eventNotifier.markEvent(HystrixEventType.EXCEPTION_THROWN, commandKey); } - })); - } else { - metrics.markCommandStart(commandKey, threadPoolKey, ExecutionIsolationStrategy.SEMAPHORE); - // semaphore isolated - // store the command that is being run - endCurrentThreadExecutingCommand.set(Hystrix.startCurrentThreadExecutingCommand(getCommandKey())); - try { - executionHook.onRunStart(_self); - executionHook.onExecutionStart(_self); - run = getExecutionObservableWithLifecycle(); //the getExecutionObservableWithLifecycle method already wraps sync exceptions, so this shouldn't throw - } catch (Throwable ex) { - //If the above hooks throw, then use that as the result of the run method - run = Observable.error(ex); + }; + + if (executionSemaphore.tryAcquire()) { + try { + /* used to track userThreadExecutionTime */ + executionResult = executionResult.setInvocationStartTime(System.currentTimeMillis()); + return executeCommandAndObserve(_cmd) + .doOnError(markExceptionThrown) + .doOnTerminate(singleSemaphoreRelease) + .doOnUnsubscribe(singleSemaphoreRelease); + } catch (RuntimeException e) { + return Observable.error(e); + } + } else { + return handleSemaphoreRejectionViaFallback(); } + } else { + return handleShortCircuitViaFallback(); } + } - if (properties.executionTimeoutEnabled().get()) { - run = run.lift(new HystrixObservableTimeoutOperator(_self)); - } - run = run.doOnNext(new Action1() { + /** + * This decorates "Hystrix" functionality around the run() Observable. + * + * @return R + */ + private Observable executeCommandAndObserve(final AbstractCommand _cmd) { + final HystrixRequestContext currentRequestContext = HystrixRequestContext.getContextForCurrentThread(); + + final Action1 markEmits = new Action1() { @Override public void call(R r) { if (shouldOutputOnNextEvents()) { @@ -574,8 +508,9 @@ public void call(R r) { eventNotifier.markEvent(HystrixEventType.EMIT, commandKey); } } - }).doOnCompleted(new Action0() { + }; + final Action0 markCompleted = new Action0() { @Override public void call() { long latency = System.currentTimeMillis() - executionResult.getStartTimestamp(); @@ -584,50 +519,19 @@ public void call() { circuitBreaker.markSuccess(); eventNotifier.markCommandExecution(getCommandKey(), properties.executionIsolationStrategy().get(), (int) latency, executionResult.getOrderedList()); } + }; - }).onErrorResumeNext(new Func1>() { - + final Func1> handleFallback = new Func1>() { @Override public Observable call(Throwable t) { Exception e = getExceptionFromThrowable(t); executionResult = executionResult.setExecutionException(e); if (e instanceof RejectedExecutionException) { - /** - * Rejection handling - */ - eventNotifier.markEvent(HystrixEventType.THREAD_POOL_REJECTED, commandKey); - threadPool.markThreadRejection(); - // use a fallback instead (or throw exception if not implemented) - return getFallbackOrThrowException(HystrixEventType.THREAD_POOL_REJECTED, FailureType.REJECTED_THREAD_EXECUTION, "could not be queued for execution", e); + return handleThreadPoolRejectionViaFallback(e); } else if (t instanceof HystrixTimeoutException) { - /** - * Timeout handling - * - * Callback is performed on the HystrixTimer thread. - */ - return getFallbackOrThrowException(HystrixEventType.TIMEOUT, FailureType.TIMEOUT, "timed-out", new TimeoutException()); + return handleTimeoutViaFallback(); } else if (t instanceof HystrixBadRequestException) { - /** - * BadRequest handling - */ - try { - long executionLatency = System.currentTimeMillis() - executionResult.getStartTimestamp(); - eventNotifier.markEvent(HystrixEventType.BAD_REQUEST, commandKey); - executionResult = executionResult.addEvent((int) executionLatency, HystrixEventType.BAD_REQUEST); - Exception decorated = executionHook.onError(_self, FailureType.BAD_REQUEST_EXCEPTION, (Exception) t); - - if (decorated instanceof HystrixBadRequestException) { - t = decorated; - } else { - logger.warn("ExecutionHook.onError returned an exception that was not an instance of HystrixBadRequestException so will be ignored.", decorated); - } - } catch (Exception hookEx) { - logger.warn("Error calling HystrixCommandExecutionHook.onError", hookEx); - } - /* - * HystrixBadRequestException is treated differently and allowed to propagate without any stats tracking or fallback logic - */ - return Observable.error(t); + return handleBadRequestByEmittingError(e); } else { /* * Treat HystrixBadRequestException from ExecutionHook like a plain HystrixBadRequestException. @@ -637,62 +541,103 @@ public Observable call(Throwable t) { return Observable.error(e); } - /** - * All other error handling - */ - logger.debug("Error executing HystrixCommand.run(). Proceeding to fallback logic ...", e); - - // report failure - eventNotifier.markEvent(HystrixEventType.FAILURE, commandKey); - - // record the exception - executionResult = executionResult.setException(e); - return getFallbackOrThrowException(HystrixEventType.FAILURE, FailureType.COMMAND_EXCEPTION, "failed", e); + return handleFailureViaFallback(e); } } - }).doOnEach(new Action1>() { - // setting again as the fallback could have lost the context + }; + + final Action1> setRequestContext = new Action1>() { @Override - public void call(Notification n) { + public void call(Notification rNotification) { setRequestContextIfNeeded(currentRequestContext); } + }; - }).doOnTerminate(new Action0() { + final Action0 handleThreadEndOnNonTimeout = new Action0() { @Override public void call() { if (!isCommandTimedOut.get().equals(TimedOutStatus.TIMED_OUT)) { handleThreadEnd(); } } - }).lift(new DeprecatedOnCompleteWithValueHookApplication(_self)); + }; - return run; - } + Observable execution; + if (properties.executionTimeoutEnabled().get()) { + execution = executeCommandWithSpecifiedIsolation(_cmd) + .lift(new HystrixObservableTimeoutOperator(_cmd)); + } else { + execution = executeCommandWithSpecifiedIsolation(_cmd); + } - private Observable getExecutionObservableWithLifecycle() { - final HystrixInvokable _self = this; + return execution.doOnNext(markEmits) + .doOnCompleted(markCompleted) + .onErrorResumeNext(handleFallback) + .doOnEach(setRequestContext) + .doOnTerminate(handleThreadEndOnNonTimeout) + .lift(new DeprecatedOnCompleteWithValueHookApplication(_cmd)); + } - Observable userObservable; + private Observable executeCommandWithSpecifiedIsolation(final AbstractCommand _cmd) { + if (properties.executionIsolationStrategy().get().equals(ExecutionIsolationStrategy.THREAD)) { + // mark that we are executing in a thread (even if we end up being rejected we still were a THREAD execution and not SEMAPHORE) + return Observable.defer(new Func0>() { + @Override + public Observable call() { + executionResult = executionResult.setExecutionOccurred(); + metrics.markCommandStart(commandKey, threadPoolKey, ExecutionIsolationStrategy.THREAD); - try { - userObservable = getExecutionObservable(); - } catch (Throwable ex) { - // the run() method is a user provided implementation so can throw instead of using Observable.onError - // so we catch it here and turn it into Observable.error - userObservable = Observable.error(ex); - } - return userObservable.lift(new ExecutionHookApplication(_self)) - .lift(new DeprecatedOnRunHookApplication(_self)) - .doOnTerminate(new Action0() { - @Override - public void call() { - //If the command timed out, then the calling thread has already walked away so we need - //to handle these markers. Otherwise, the calling thread will perform these for us. - if (isCommandTimedOut.get().equals(TimedOutStatus.TIMED_OUT)) { - handleThreadEnd(); + if (isCommandTimedOut.get() == TimedOutStatus.TIMED_OUT) { + // the command timed out in the wrapping thread so we will return immediately + // and not increment any of the counters below or other such logic + return Observable.error(new RuntimeException("timed out before executing run()")); + } else { + // not timed out so execute + HystrixCounters.incrementGlobalConcurrentThreads(); + threadPool.markThreadExecution(); + // store the command that is being run + endCurrentThreadExecutingCommand = Hystrix.startCurrentThreadExecutingCommand(getCommandKey()); + executionResult = executionResult.setExecutedInThread(); + /** + * If any of these hooks throw an exception, then it appears as if the actual execution threw an error + */ + try { + executionHook.onThreadStart(_cmd); + executionHook.onRunStart(_cmd); + executionHook.onExecutionStart(_cmd); + return getUserExecutionObservable(_cmd); + } catch (Throwable ex) { + return Observable.error(ex); } } - }); + } + }).subscribeOn(threadPool.getScheduler(new Func0() { + + @Override + public Boolean call() { + return properties.executionIsolationThreadInterruptOnTimeout().get() && _cmd.isCommandTimedOut.get().equals(TimedOutStatus.TIMED_OUT); + } + })); + } else { + return Observable.defer(new Func0>() { + @Override + public Observable call() { + executionResult = executionResult.setExecutionOccurred(); + metrics.markCommandStart(commandKey, threadPoolKey, ExecutionIsolationStrategy.SEMAPHORE); + // semaphore isolated + // store the command that is being run + endCurrentThreadExecutingCommand = Hystrix.startCurrentThreadExecutingCommand(getCommandKey()); + try { + executionHook.onRunStart(_cmd); + executionHook.onExecutionStart(_cmd); + return getUserExecutionObservable(_cmd); //the getUserExecutionObservable method already wraps sync exceptions, so this shouldn't throw + } catch (Throwable ex) { + //If the above hooks throw, then use that as the result of the run method + return Observable.error(ex); + } + } + }); + } } /** @@ -709,82 +654,36 @@ public void call() { * @throws HystrixRuntimeException * if getFallback() fails (throws an Exception) or is rejected by the semaphore */ - private Observable getFallbackOrThrowException(final HystrixEventType eventType, final FailureType failureType, final String message, final Exception originalException) { - final HystrixRequestContext currentRequestContext = HystrixRequestContext.getContextForCurrentThread(); + private Observable getFallbackOrThrowException(final AbstractCommand _cmd, final HystrixEventType eventType, final FailureType failureType, final String message, final Exception originalException) { + final HystrixRequestContext requestContext = HystrixRequestContext.getContextForCurrentThread(); long latency = System.currentTimeMillis() - executionResult.getStartTimestamp(); // record the executionResult // do this before executing fallback so it can be queried from within getFallback (see See https://github.com/Netflix/Hystrix/pull/144) executionResult = executionResult.addEvent((int) latency, eventType); - Observable fallbackLogicApplied; - if (isUnrecoverable(originalException)) { Exception e = originalException; logger.error("Unrecoverable Error for HystrixCommand so will throw HystrixRuntimeException and not apply fallback. ", e); /* executionHook for all errors */ e = wrapWithOnErrorHook(failureType, e); - fallbackLogicApplied = Observable. error(new HystrixRuntimeException(failureType, this.getClass(), getLogMessagePrefix() + " " + message + " and encountered unrecoverable error.", e, null)); + return Observable.error(new HystrixRuntimeException(failureType, this.getClass(), getLogMessagePrefix() + " " + message + " and encountered unrecoverable error.", e, null)); } else { if (isRecoverableError(originalException)) { logger.warn("Recovered from java.lang.Error by serving Hystrix fallback", originalException); } if (properties.fallbackEnabled().get()) { - /* fallback behavior is permitted so attempt */ - final AbstractCommand _cmd = this; + /* fallback behavior is permitted so attempt */ - final TryableSemaphore fallbackSemaphore = getFallbackSemaphore(); - final AtomicBoolean semaphoreHasBeenReleased = new AtomicBoolean(false); - - Observable fallbackExecutionChain; - - // acquire a permit - if (fallbackSemaphore.tryAcquire()) { - try { - if (isFallbackUserSupplied(this)) { - executionHook.onFallbackStart(this); - fallbackExecutionChain = getFallbackObservable(); - } else { - //same logic as above without the hook invocation - fallbackExecutionChain = getFallbackObservable(); - } - } catch(Throwable ex) { - //If hook or user-fallback throws, then use that as the result of the fallback lookup - fallbackExecutionChain = Observable.error(ex); + final Action1> setRequestContext = new Action1>() { + @Override + public void call(Notification rNotification) { + setRequestContextIfNeeded(requestContext); } + }; - fallbackExecutionChain = fallbackExecutionChain - .lift(new FallbackHookApplication(_cmd)) - .lift(new DeprecatedOnFallbackHookApplication(_cmd)) - .doOnTerminate(new Action0() { - - @Override - public void call() { - if (semaphoreHasBeenReleased.compareAndSet(false, true)) { - fallbackSemaphore.release(); - } - } - }) - .doOnUnsubscribe(new Action0() { - - @Override - public void call() { - if (semaphoreHasBeenReleased.compareAndSet(false, true)) { - fallbackSemaphore.release(); - } - } - }); - } else { - long latencyWithFallback = System.currentTimeMillis() - executionResult.getStartTimestamp(); - eventNotifier.markEvent(HystrixEventType.FALLBACK_REJECTION, commandKey); - executionResult = executionResult.addEvent((int) latencyWithFallback, HystrixEventType.FALLBACK_REJECTION); - logger.debug("HystrixCommand Fallback Rejection."); // debug only since we're throwing the exception and someone higher will do something with it - // if we couldn't acquire a permit, we "fail fast" by throwing an exception - return Observable.error(new HystrixRuntimeException(FailureType.REJECTED_SEMAPHORE_FALLBACK, this.getClass(), getLogMessagePrefix() + " fallback execution rejected.", null, null)); - } - - fallbackLogicApplied = fallbackExecutionChain.doOnNext(new Action1() { + final Action1 markFallbackEmit = new Action1() { @Override public void call(R r) { if (shouldOutputOnNextEvents()) { @@ -792,19 +691,18 @@ public void call(R r) { eventNotifier.markEvent(HystrixEventType.FALLBACK_EMIT, commandKey); } } - }).doOnCompleted(new Action0() { + }; + final Action0 markFallbackCompleted = new Action0() { @Override public void call() { long latency = System.currentTimeMillis() - executionResult.getStartTimestamp(); - // mark fallback on counter eventNotifier.markEvent(HystrixEventType.FALLBACK_SUCCESS, commandKey); - // record the executionResult executionResult = executionResult.addEvent((int) latency, HystrixEventType.FALLBACK_SUCCESS); } + }; - }).onErrorResumeNext(new Func1>() { - + final Func1> handleFallbackError = new Func1>() { @Override public Observable call(Throwable t) { Exception e = originalException; @@ -824,7 +722,6 @@ public Observable call(Throwable t) { long latency = System.currentTimeMillis() - executionResult.getStartTimestamp(); logger.debug("HystrixCommand execution " + failureType.name() + " and fallback failed.", fe); eventNotifier.markEvent(HystrixEventType.FALLBACK_FAILURE, commandKey); - // record the executionResult executionResult = executionResult.addEvent((int) latency, HystrixEventType.FALLBACK_FAILURE); /* executionHook for all errors */ @@ -833,28 +730,232 @@ public Observable call(Throwable t) { return Observable.error(new HystrixRuntimeException(failureType, _cmd.getClass(), getLogMessagePrefix() + " " + message + " and fallback failed.", e, fe)); } } + }; - }); - } else { - /* fallback is disabled so throw HystrixRuntimeException */ - Exception e = originalException; - logger.debug("Fallback disabled for HystrixCommand so will throw HystrixRuntimeException. ", e); // debug only since we're throwing the exception and someone higher will do something with it + final TryableSemaphore fallbackSemaphore = getFallbackSemaphore(); + final AtomicBoolean semaphoreHasBeenReleased = new AtomicBoolean(false); + final Action0 singleSemaphoreRelease = new Action0() { + @Override + public void call() { + if (semaphoreHasBeenReleased.compareAndSet(false, true)) { + fallbackSemaphore.release(); + } + } + }; + + Observable fallbackExecutionChain; + + // acquire a permit + if (fallbackSemaphore.tryAcquire()) { + try { + if (isFallbackUserSupplied(this)) { + executionHook.onFallbackStart(this); + fallbackExecutionChain = getFallbackObservable(); + } else { + //same logic as above without the hook invocation + fallbackExecutionChain = getFallbackObservable(); + } + } catch (Throwable ex) { + //If hook or user-fallback throws, then use that as the result of the fallback lookup + fallbackExecutionChain = Observable.error(ex); + } - /* executionHook for all errors */ - e = wrapWithOnErrorHook(failureType, e); - fallbackLogicApplied = Observable. error(new HystrixRuntimeException(failureType, this.getClass(), getLogMessagePrefix() + " " + message + " and fallback disabled.", e, null)); + return fallbackExecutionChain + .doOnEach(setRequestContext) + .lift(new FallbackHookApplication(_cmd)) + .lift(new DeprecatedOnFallbackHookApplication(_cmd)) + .doOnNext(markFallbackEmit) + .doOnCompleted(markFallbackCompleted) + .onErrorResumeNext(handleFallbackError) + .doOnTerminate(singleSemaphoreRelease) + .doOnUnsubscribe(singleSemaphoreRelease); + } else { + return handleFallbackRejectionByEmittingError(); + } + } else { + return handleFallbackDisabledByEmittingError(originalException, failureType, message); } } + } - return fallbackLogicApplied.doOnEach(new Action1>() { + private Observable getUserExecutionObservable(final AbstractCommand _cmd) { + Observable userObservable; + try { + userObservable = getExecutionObservable(); + } catch (Throwable ex) { + // the run() method is a user provided implementation so can throw instead of using Observable.onError + // so we catch it here and turn it into Observable.error + userObservable = Observable.error(ex); + } + return userObservable.lift(new ExecutionHookApplication(_cmd)) + .lift(new DeprecatedOnRunHookApplication(_cmd)) + .doOnTerminate(new Action0() { + @Override + public void call() { + //If the command timed out, then the calling thread has already walked away so we need + //to handle these markers. Otherwise, the calling thread will perform these for us. + if (isCommandTimedOut.get().equals(TimedOutStatus.TIMED_OUT)) { + handleThreadEnd(); + } + } + }); + } + + private Observable handleRequestCacheHitAndEmitValues(final HystrixCachedObservable fromCache) { + try { + executionHook.onCacheHit(this); + } catch (Throwable hookEx) { + logger.warn("Error calling HystrixCommandExecutionHook.onCacheHit", hookEx); + } + + final AtomicBoolean cleanupCompleted = new AtomicBoolean(false); + + return fromCache.toObservable(this).doOnTerminate(new Action0() { @Override - public void call(Notification n) { - setRequestContextIfNeeded(currentRequestContext); + public void call() { + if (!cleanupCompleted.get()) { + cleanUpAfterResponseFromCache(); + isExecutionComplete = true; + cleanupCompleted.set(true); + } + } + }).doOnUnsubscribe(new Action0() { + @Override + public void call() { + if (!cleanupCompleted.get()) { + cleanUpAfterResponseFromCache(); + cleanupCompleted.set(true); + } } }); } + private void cleanUpAfterResponseFromCache() { + Reference tl = timeoutTimer.get(); + if (tl != null) { + tl.clear(); + } + + final long latency = System.currentTimeMillis() - commandStartTimestamp; + executionResult = executionResult + .addEvent(-1, HystrixEventType.RESPONSE_FROM_CACHE) + .markUserThreadCompletion(latency) + .setNotExecutedInThread(); + ExecutionResult cacheOnlyForMetrics = ExecutionResult.from(HystrixEventType.RESPONSE_FROM_CACHE) + .markUserThreadCompletion(latency); + metrics.markCommandDone(cacheOnlyForMetrics, commandKey, threadPoolKey); + eventNotifier.markEvent(HystrixEventType.RESPONSE_FROM_CACHE, commandKey); + } + + private void handleCommandEnd() { + Reference tl = timeoutTimer.get(); + if (tl != null) { + tl.clear(); + } + + long userThreadLatency = System.currentTimeMillis() - commandStartTimestamp; + executionResult = executionResult.markUserThreadCompletion((int) userThreadLatency); + ExecutionResult cancelled = executionResultAtTimeOfCancellation; + if (cancelled == null) { + metrics.markCommandDone(executionResult, commandKey, threadPoolKey); + } else { + metrics.markCommandDone(cancelled, commandKey, threadPoolKey); + } + } + + private Observable handleSemaphoreRejectionViaFallback() { + Exception semaphoreRejectionException = new RuntimeException("could not acquire a semaphore for execution"); + executionResult = executionResult.setExecutionException(semaphoreRejectionException); + eventNotifier.markEvent(HystrixEventType.SEMAPHORE_REJECTED, commandKey); + logger.debug("HystrixCommand Execution Rejection by Semaphore."); // debug only since we're throwing the exception and someone higher will do something with it + // retrieve a fallback or throw an exception if no fallback available + return getFallbackOrThrowException(this, HystrixEventType.SEMAPHORE_REJECTED, FailureType.REJECTED_SEMAPHORE_EXECUTION, + "could not acquire a semaphore for execution", semaphoreRejectionException) + .lift(new DeprecatedOnCompleteWithValueHookApplication(this)); + } + + private Observable handleShortCircuitViaFallback() { + // record that we are returning a short-circuited fallback + eventNotifier.markEvent(HystrixEventType.SHORT_CIRCUITED, commandKey); + // short-circuit and go directly to fallback (or throw an exception if no fallback implemented) + Exception shortCircuitException = new RuntimeException("Hystrix circuit short-circuited and is OPEN"); + executionResult = executionResult.setExecutionException(shortCircuitException); + try { + return getFallbackOrThrowException(this, HystrixEventType.SHORT_CIRCUITED, FailureType.SHORTCIRCUIT, + "short-circuited", shortCircuitException) + .lift(new DeprecatedOnCompleteWithValueHookApplication(this)); + } catch (Exception e) { + return Observable.error(e); + } + } + + private Observable handleThreadPoolRejectionViaFallback(Exception underlying) { + eventNotifier.markEvent(HystrixEventType.THREAD_POOL_REJECTED, commandKey); + threadPool.markThreadRejection(); + // use a fallback instead (or throw exception if not implemented) + return getFallbackOrThrowException(this, HystrixEventType.THREAD_POOL_REJECTED, FailureType.REJECTED_THREAD_EXECUTION, "could not be queued for execution", underlying); + } + + private Observable handleTimeoutViaFallback() { + return getFallbackOrThrowException(this, HystrixEventType.TIMEOUT, FailureType.TIMEOUT, "timed-out", new TimeoutException()); + } + + private Observable handleBadRequestByEmittingError(Exception underlying) { + Exception toEmit = underlying; + + try { + long executionLatency = System.currentTimeMillis() - executionResult.getStartTimestamp(); + eventNotifier.markEvent(HystrixEventType.BAD_REQUEST, commandKey); + executionResult = executionResult.addEvent((int) executionLatency, HystrixEventType.BAD_REQUEST); + Exception decorated = executionHook.onError(this, FailureType.BAD_REQUEST_EXCEPTION, underlying); + + if (decorated instanceof HystrixBadRequestException) { + toEmit = decorated; + } else { + logger.warn("ExecutionHook.onError returned an exception that was not an instance of HystrixBadRequestException so will be ignored.", decorated); + } + } catch (Exception hookEx) { + logger.warn("Error calling HystrixCommandExecutionHook.onError", hookEx); + } + /* + * HystrixBadRequestException is treated differently and allowed to propagate without any stats tracking or fallback logic + */ + return Observable.error(toEmit); + } + + private Observable handleFailureViaFallback(Exception underlying) { + /** + * All other error handling + */ + logger.debug("Error executing HystrixCommand.run(). Proceeding to fallback logic ...", underlying); + + // report failure + eventNotifier.markEvent(HystrixEventType.FAILURE, commandKey); + + // record the exception + executionResult = executionResult.setException(underlying); + return getFallbackOrThrowException(this, HystrixEventType.FAILURE, FailureType.COMMAND_EXCEPTION, "failed", underlying); + } + + private Observable handleFallbackRejectionByEmittingError() { + long latencyWithFallback = System.currentTimeMillis() - executionResult.getStartTimestamp(); + eventNotifier.markEvent(HystrixEventType.FALLBACK_REJECTION, commandKey); + executionResult = executionResult.addEvent((int) latencyWithFallback, HystrixEventType.FALLBACK_REJECTION); + logger.debug("HystrixCommand Fallback Rejection."); // debug only since we're throwing the exception and someone higher will do something with it + // if we couldn't acquire a permit, we "fail fast" by throwing an exception + return Observable.error(new HystrixRuntimeException(FailureType.REJECTED_SEMAPHORE_FALLBACK, this.getClass(), getLogMessagePrefix() + " fallback execution rejected.", null, null)); + } + + private Observable handleFallbackDisabledByEmittingError(Exception underlying, FailureType failureType, String message) { + /* fallback is disabled so throw HystrixRuntimeException */ + logger.debug("Fallback disabled for HystrixCommand so will throw HystrixRuntimeException. ", underlying); // debug only since we're throwing the exception and someone higher will do something with it + + /* executionHook for all errors */ + Exception wrapped = wrapWithOnErrorHook(failureType, underlying); + return Observable.error(new HystrixRuntimeException(failureType, this.getClass(), getLogMessagePrefix() + " " + message + " and fallback disabled.", wrapped, null)); + } + /** * Returns true iff the t was caused by a java.lang.Error that is unrecoverable. Note: not all java.lang.Errors are unrecoverable. * @see for more context @@ -898,8 +999,8 @@ private boolean isRecoverableError(Throwable t) { } protected void handleThreadEnd() { - if (endCurrentThreadExecutingCommand.get() != null) { - endCurrentThreadExecutingCommand.get().call(); + if (endCurrentThreadExecutingCommand != null) { + endCurrentThreadExecutingCommand.call(); } if (executionResult.isExecutedInThread()) { HystrixCounters.decrementGlobalConcurrentThreads(); @@ -916,7 +1017,6 @@ protected void handleThreadEnd() { * * @return if onNext events should be reported on * This affects {@link HystrixRequestLog}, and {@link HystrixEventNotifier} currently. - * Metrics will be affected once they are in place */ protected boolean shouldOutputOnNextEvents() { return false; @@ -1107,114 +1207,6 @@ protected TryableSemaphore getExecutionSemaphore() { } } - protected static class ObservableCommand extends Observable { - private final AbstractCommand command; - - ObservableCommand(OnSubscribe func, final AbstractCommand command) { - super(func); - this.command = command; - } - - public AbstractCommand getCommand() { - return command; - } - - ObservableCommand(final Observable originalObservable, final AbstractCommand command) { - super(new OnSubscribe() { - - @Override - public void call(Subscriber observer) { - originalObservable.unsafeSubscribe(observer); - } - }); - this.command = command; - } - - } - - /** - * Wraps a source Observable and remembers the original HystrixCommand. - *

- * Used for request caching so multiple commands can respond from a single Observable but also get access to the originating HystrixCommand. - * - * @param - */ - protected static class CachedObservableOriginal extends ObservableCommand { - - final AbstractCommand originalCommand; - - CachedObservableOriginal(final Observable actual, AbstractCommand command) { - super(new OnSubscribe() { - - @Override - public void call(final Subscriber observer) { - actual.unsafeSubscribe(observer); - } - }, command); - this.originalCommand = command; - } - } - - /** - * Wraps a CachedObservableOriginal as it is being returned from cache. - *

- * As the Observable completes it copies state used for ExecutionResults - * and metrics that differentiate between the original and the de-duped "response from cache" command execution. - * - * @param - */ - protected static class CachedObservableResponse extends ObservableCommand { - final CachedObservableOriginal originalObservable; - - CachedObservableResponse(final CachedObservableOriginal originalObservable, final AbstractCommand commandOfDuplicateCall) { - super(new OnSubscribe() { - - @Override - public void call(final Subscriber observer) { - originalObservable.subscribe(new Subscriber() { - - @Override - public void onCompleted() { - completeCommand(); - observer.onCompleted(); - } - - @Override - public void onError(Throwable e) { - completeCommand(); - observer.onError(e); - } - - @Override - public void onNext(R v) { - observer.onNext(v); - } - - private void completeCommand() { - // when the observable completes we then update the execution results of the duplicate command - // set this instance to the result that is from cache - commandOfDuplicateCall.executionResult = originalObservable.originalCommand.executionResult; - // add that this came from cache - commandOfDuplicateCall.executionResult = commandOfDuplicateCall.executionResult.addEvent(HystrixEventType.RESPONSE_FROM_CACHE); - // set the execution time to 0 since we retrieved from cache - commandOfDuplicateCall.executionResult = commandOfDuplicateCall.executionResult.setExecutionLatency(-1); - // record that this command executed - commandOfDuplicateCall.recordExecutedCommand(); - } - }); - } - }, commandOfDuplicateCall); - this.originalObservable = originalObservable; - } - - /* - * This is a cached response so we want the command of the observable we're wrapping. - */ - public AbstractCommand getCommand() { - return originalObservable.originalCommand; - } - } - /** * @return {@link HystrixCommandGroupKey} used to group together multiple {@link AbstractCommand} objects. *

@@ -1262,20 +1254,6 @@ public HystrixCommandProperties getProperties() { return properties; } - /** - * Record that this command was executed in the HystrixRequestLog. - *

- * This can be treated as an async operation as it just adds a references to "this" in the log even if the current command is still executing. - */ - protected void recordExecutedCommand() { - if (properties.requestLogEnabled().get()) { - // log this command execution regardless of what happened - if (currentRequestLog != null) { - currentRequestLog.addExecutedCommand(this); - } - } - } - /* ******************************************************************************** */ /* ******************************************************************************** */ /* Operators that implement hook application */ @@ -1563,7 +1541,6 @@ private R wrapWithOnEmitHook(R r) { } } - /** * Take an Exception and determine whether to throw it, its cause or a new HystrixRuntimeException. *

@@ -1700,12 +1677,6 @@ public int getNumberOfPermitsUsed() { } - /* ******************************************************************************** */ - /* ******************************************************************************** */ - /* Result Status */ - /* ******************************************************************************** */ - /* ******************************************************************************** */ - /* ******************************************************************************** */ /* ******************************************************************************** */ /* RequestCache */ @@ -1766,7 +1737,7 @@ public boolean isCircuitBreakerOpen() { * @return boolean */ public boolean isExecutionComplete() { - return isExecutionComplete.get(); + return isExecutionComplete; } /** @@ -1779,7 +1750,7 @@ public boolean isExecutionComplete() { * @return boolean */ public boolean isExecutedInThread() { - return executionResult.isExecutedInThread(); + return getCommandResult().isExecutedInThread(); } /** @@ -1788,7 +1759,7 @@ public boolean isExecutedInThread() { * @return boolean */ public boolean isSuccessfulExecution() { - return executionResult.getEventCounts().contains(HystrixEventType.SUCCESS); + return getCommandResult().getEventCounts().contains(HystrixEventType.SUCCESS); } /** @@ -1797,7 +1768,7 @@ public boolean isSuccessfulExecution() { * @return boolean */ public boolean isFailedExecution() { - return executionResult.getEventCounts().contains(HystrixEventType.FAILURE); + return getCommandResult().getEventCounts().contains(HystrixEventType.FAILURE); } /** @@ -1840,7 +1811,7 @@ public Throwable getExecutionException() { * @return boolean */ public boolean isResponseFromFallback() { - return executionResult.getEventCounts().contains(HystrixEventType.FALLBACK_SUCCESS); + return getCommandResult().getEventCounts().contains(HystrixEventType.FALLBACK_SUCCESS); } /** @@ -1850,7 +1821,7 @@ public boolean isResponseFromFallback() { * @return boolean */ public boolean isResponseTimedOut() { - return executionResult.getEventCounts().contains(HystrixEventType.TIMEOUT); + return getCommandResult().getEventCounts().contains(HystrixEventType.TIMEOUT); } /** @@ -1860,7 +1831,7 @@ public boolean isResponseTimedOut() { * @return boolean */ public boolean isResponseShortCircuited() { - return executionResult.getEventCounts().contains(HystrixEventType.SHORT_CIRCUITED); + return getCommandResult().getEventCounts().contains(HystrixEventType.SHORT_CIRCUITED); } /** @@ -1869,7 +1840,7 @@ public boolean isResponseShortCircuited() { * @return boolean */ public boolean isResponseFromCache() { - return executionResult.getEventCounts().contains(HystrixEventType.RESPONSE_FROM_CACHE); + return isResponseFromCache; } /** @@ -1878,7 +1849,7 @@ public boolean isResponseFromCache() { * @return boolean */ public boolean isResponseSemaphoreRejected() { - return executionResult.isResponseSemaphoreRejected(); + return getCommandResult().isResponseSemaphoreRejected(); } /** @@ -1887,7 +1858,7 @@ public boolean isResponseSemaphoreRejected() { * @return boolean */ public boolean isResponseThreadPoolRejected() { - return executionResult.isResponseThreadPoolRejected(); + return getCommandResult().isResponseThreadPoolRejected(); } /** @@ -1896,7 +1867,7 @@ public boolean isResponseThreadPoolRejected() { * @return boolean */ public boolean isResponseRejected() { - return executionResult.isResponseRejected(); + return getCommandResult().isResponseRejected(); } /** @@ -1907,7 +1878,22 @@ public boolean isResponseRejected() { * @return {@code List} */ public List getExecutionEvents() { - return executionResult.getOrderedList(); + return getCommandResult().getOrderedList(); + } + + private ExecutionResult getCommandResult() { + ExecutionResult resultToReturn; + if (executionResultAtTimeOfCancellation == null) { + resultToReturn = executionResult; + } else { + resultToReturn = executionResultAtTimeOfCancellation; + } + + if (isResponseFromCache) { + resultToReturn = resultToReturn.addEvent(HystrixEventType.RESPONSE_FROM_CACHE); + } + + return resultToReturn; } /** @@ -1916,7 +1902,7 @@ public List getExecutionEvents() { */ @Override public int getNumberEmissions() { - return executionResult.getEventCounts().getCount(HystrixEventType.EMIT); + return getCommandResult().getEventCounts().getCount(HystrixEventType.EMIT); } /** @@ -1925,12 +1911,12 @@ public int getNumberEmissions() { */ @Override public int getNumberFallbackEmissions() { - return executionResult.getEventCounts().getCount(HystrixEventType.FALLBACK_EMIT); + return getCommandResult().getEventCounts().getCount(HystrixEventType.FALLBACK_EMIT); } @Override public int getNumberCollapsed() { - return executionResult.getEventCounts().getCount(HystrixEventType.COLLAPSED); + return getCommandResult().getEventCounts().getCount(HystrixEventType.COLLAPSED); } @Override @@ -1944,7 +1930,7 @@ public HystrixCollapserKey getOriginatingCollapserKey() { * @return int */ public int getExecutionTimeInMilliseconds() { - return executionResult.getExecutionLatency(); + return getCommandResult().getExecutionLatency(); } /** @@ -1959,7 +1945,7 @@ public long getCommandRunStartTimeInNanos() { @Override public ExecutionResult.EventCounts getEventCounts() { - return executionResult.getEventCounts(); + return getCommandResult().getEventCounts(); } protected Exception getExceptionFromThrowable(Throwable t) { diff --git a/hystrix-core/src/main/java/com/netflix/hystrix/ExecutionResult.java b/hystrix-core/src/main/java/com/netflix/hystrix/ExecutionResult.java index 669329b3b..94dff7359 100644 --- a/hystrix-core/src/main/java/com/netflix/hystrix/ExecutionResult.java +++ b/hystrix-core/src/main/java/com/netflix/hystrix/ExecutionResult.java @@ -201,9 +201,6 @@ private ExecutionResult(EventCounts eventCounts, long startTimestamp, int execut public static ExecutionResult from(HystrixEventType... eventTypes) { boolean didExecutionOccur = false; for (HystrixEventType eventType: eventTypes) { - /*if (isFallbackEvent(eventType)) { - - }*/ if (didExecutionOccur(eventType)) { didExecutionOccur = true; } @@ -217,10 +214,16 @@ private static boolean didExecutionOccur(HystrixEventType eventType) { case FAILURE: return true; case BAD_REQUEST: return true; case TIMEOUT: return true; + case CANCELLED: return true; default: return false; } } + public ExecutionResult setExecutionOccurred() { + return new ExecutionResult(eventCounts, startTimestamp, executionLatency, userThreadLatency, + failedExecutionException, executionException, true, isExecutedInThread, collapserKey); + } + public ExecutionResult setExecutionLatency(int executionLatency) { return new ExecutionResult(eventCounts, startTimestamp, executionLatency, userThreadLatency, failedExecutionException, executionException, executionOccurred, isExecutedInThread, collapserKey); @@ -246,6 +249,11 @@ public ExecutionResult setExecutedInThread() { failedExecutionException, executionException, executionOccurred, true, collapserKey); } + public ExecutionResult setNotExecutedInThread() { + return new ExecutionResult(eventCounts, startTimestamp, executionLatency, userThreadLatency, + failedExecutionException, executionException, executionOccurred, false, collapserKey); + } + public ExecutionResult markCollapsed(HystrixCollapserKey collapserKey, int sizeOfBatch) { return new ExecutionResult(eventCounts.plus(HystrixEventType.COLLAPSED, sizeOfBatch), startTimestamp, executionLatency, userThreadLatency, failedExecutionException, executionException, executionOccurred, isExecutedInThread, collapserKey); @@ -270,14 +278,14 @@ public ExecutionResult markUserThreadCompletion(long userThreadLatency) { public ExecutionResult addEvent(HystrixEventType eventType) { return new ExecutionResult(eventCounts.plus(eventType), startTimestamp, executionLatency, userThreadLatency, failedExecutionException, executionException, - executionOccurred ? executionOccurred : didExecutionOccur(eventType), isExecutedInThread, collapserKey); + executionOccurred, isExecutedInThread, collapserKey); } public ExecutionResult addEvent(int executionLatency, HystrixEventType eventType) { if (startTimestamp >= 0 && !isResponseRejected()) { return new ExecutionResult(eventCounts.plus(eventType), startTimestamp, executionLatency, userThreadLatency, failedExecutionException, executionException, - executionOccurred ? executionOccurred : didExecutionOccur(eventType), isExecutedInThread, collapserKey); + executionOccurred, isExecutedInThread, collapserKey); } else { return addEvent(eventType); } diff --git a/hystrix-core/src/main/java/com/netflix/hystrix/HystrixCachedObservable.java b/hystrix-core/src/main/java/com/netflix/hystrix/HystrixCachedObservable.java new file mode 100644 index 000000000..13834bbf1 --- /dev/null +++ b/hystrix-core/src/main/java/com/netflix/hystrix/HystrixCachedObservable.java @@ -0,0 +1,98 @@ +package com.netflix.hystrix; + +import rx.Observable; +import rx.Subscription; +import rx.functions.Action0; +import rx.functions.Action1; +import rx.subjects.ReplaySubject; + +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; + +public class HystrixCachedObservable { + final AbstractCommand originalCommand; + final Observable cachedObservable; + final Subscription originalSubscription; + final ReplaySubject replaySubject = ReplaySubject.create(); + final AtomicInteger outstandingSubscriptions = new AtomicInteger(0); + + /* package-private */ HystrixCachedObservable(Observable originalObservable, final AbstractCommand originalCommand) { + this.originalSubscription = originalObservable + .subscribe(replaySubject); + + this.cachedObservable = replaySubject + .doOnUnsubscribe(new Action0() { + @Override + public void call() { + if (outstandingSubscriptions.decrementAndGet() == 0) { + originalSubscription.unsubscribe(); + } + } + }) + .doOnSubscribe(new Action0() { + @Override + public void call() { + outstandingSubscriptions.getAndIncrement(); + } + }); + this.originalCommand = originalCommand; + } + + public static HystrixCachedObservable from(Observable o, AbstractCommand originalCommand) { + return new HystrixCachedObservable(o, originalCommand); + } + + public static HystrixCachedObservable from(Observable o, HystrixCollapser originalCollapser) { + return new HystrixCachedObservable(o, null); //??? + } + + public static HystrixCachedObservable from(Observable o, HystrixObservableCollapser originalCollapser) { + return new HystrixCachedObservable(o, null); //??? + } + + public Observable toObservable() { + return cachedObservable; + } + + public Observable toObservable(final AbstractCommand commandToCopyStateInto) { + final AtomicBoolean completionLogicRun = new AtomicBoolean(false); + + return cachedObservable + .doOnError(new Action1() { + @Override + public void call(Throwable throwable) { + if (!completionLogicRun.get()) { + commandCompleted(commandToCopyStateInto); + completionLogicRun.set(true); + } + } + }) + .doOnCompleted(new Action0() { + @Override + public void call() { + if (!completionLogicRun.get()) { + commandCompleted(commandToCopyStateInto); + completionLogicRun.set(true); + } + } + }) + .doOnUnsubscribe(new Action0() { + @Override + public void call() { + if (!completionLogicRun.get()) { + commandUnsubscribed(commandToCopyStateInto); + completionLogicRun.set(true); + } + } + }); + } + + private void commandCompleted(final AbstractCommand commandToCopyStateInto) { + commandToCopyStateInto.executionResult = originalCommand.executionResult; + } + + private void commandUnsubscribed(final AbstractCommand commandToCopyStateInto) { + commandToCopyStateInto.executionResult = commandToCopyStateInto.executionResult.addEvent(HystrixEventType.CANCELLED); + commandToCopyStateInto.executionResult = commandToCopyStateInto.executionResult.setExecutionLatency(-1); + } +} diff --git a/hystrix-core/src/main/java/com/netflix/hystrix/HystrixCollapser.java b/hystrix-core/src/main/java/com/netflix/hystrix/HystrixCollapser.java index 932c71d9d..abf7bed99 100644 --- a/hystrix-core/src/main/java/com/netflix/hystrix/HystrixCollapser.java +++ b/hystrix-core/src/main/java/com/netflix/hystrix/HystrixCollapser.java @@ -376,10 +376,10 @@ public Observable toObservable(Scheduler observeOn) { /* try from cache first */ if (isRequestCacheEnabled) { - Observable fromCache = requestCache.get(getCacheKey()); + HystrixCachedObservable fromCache = requestCache.get(getCacheKey()); if (fromCache != null) { metrics.markResponseFromCache(); - return fromCache; + return fromCache.toObservable(); } } @@ -396,12 +396,12 @@ public Observable toObservable(Scheduler observeOn) { * If this is an issue we can make a lazy-future that gets set in the cache * then only the winning 'put' will be invoked to actually call 'submitRequest' */ - Observable o = response.cache(); - Observable fromCache = requestCache.putIfAbsent(getCacheKey(), o); + HystrixCachedObservable toCache = HystrixCachedObservable.from(response, this); + HystrixCachedObservable fromCache = requestCache.putIfAbsent(getCacheKey(), toCache); if (fromCache == null) { - response = o; + return toCache.toObservable(); } else { - response = fromCache; + return fromCache.toObservable(); } } return response; diff --git a/hystrix-core/src/main/java/com/netflix/hystrix/HystrixCommand.java b/hystrix-core/src/main/java/com/netflix/hystrix/HystrixCommand.java index 2ab2944bc..b120bb00a 100755 --- a/hystrix-core/src/main/java/com/netflix/hystrix/HystrixCommand.java +++ b/hystrix-core/src/main/java/com/netflix/hystrix/HystrixCommand.java @@ -27,6 +27,7 @@ import com.netflix.hystrix.exception.HystrixRuntimeException.FailureType; import com.netflix.hystrix.strategy.executionhook.HystrixCommandExecutionHook; import com.netflix.hystrix.strategy.properties.HystrixPropertiesStrategy; +import rx.functions.Func0; /** * Used to wrap code that will execute potentially risky functionality (typically meaning a service call over the network) @@ -285,35 +286,29 @@ protected R getFallback() { @Override final protected Observable getExecutionObservable() { - return Observable.create(new OnSubscribe() { - + return Observable.defer(new Func0>() { @Override - public void call(Subscriber s) { + public Observable call() { try { - s.onNext(run()); - s.onCompleted(); - } catch (Throwable e) { - s.onError(e); + return Observable.just(run()); + } catch (Throwable ex) { + return Observable.error(ex); } } - }); } @Override final protected Observable getFallbackObservable() { - return Observable.create(new OnSubscribe() { - + return Observable.defer(new Func0>() { @Override - public void call(Subscriber s) { + public Observable call() { try { - s.onNext(getFallback()); - s.onCompleted(); - } catch (Throwable e) { - s.onError(e); + return Observable.just(getFallback()); + } catch (Throwable ex) { + return Observable.error(ex); } } - }); } diff --git a/hystrix-core/src/main/java/com/netflix/hystrix/HystrixCommandMetrics.java b/hystrix-core/src/main/java/com/netflix/hystrix/HystrixCommandMetrics.java index db62e2f21..e2bd8b006 100644 --- a/hystrix-core/src/main/java/com/netflix/hystrix/HystrixCommandMetrics.java +++ b/hystrix-core/src/main/java/com/netflix/hystrix/HystrixCommandMetrics.java @@ -355,19 +355,20 @@ public int getCurrentConcurrentExecutionCount() { * * This metrics should measure the actual health of a {@link HystrixCommand}. For that reason, the following are included: *

    - *
  • {@link HystrixRollingNumberEvent#SUCCESS} - *
  • {@link HystrixRollingNumberEvent#FAILURE} - *
  • {@link HystrixRollingNumberEvent#TIMEOUT} - *
  • {@link HystrixRollingNumberEvent#THREAD_POOL_REJECTED} - *
  • {@link HystrixRollingNumberEvent#SEMAPHORE_REJECTED} + *
  • {@link HystrixEventType#SUCCESS} + *
  • {@link HystrixEventType#FAILURE} + *
  • {@link HystrixEventType#TIMEOUT} + *
  • {@link HystrixEventType#THREAD_POOL_REJECTED} + *
  • {@link HystrixEventType#SEMAPHORE_REJECTED} *

* The following are not included in either attempts/failures: *

    - *
  • {@link HystrixRollingNumberEvent#BAD_REQUEST} - this event denotes bad arguments to the command and not a problem with the command - *
  • {@link HystrixRollingNumberEvent#SHORT_CIRCUITED} - this event measures a health problem in the past, not a problem with the current state + *
  • {@link HystrixEventType#BAD_REQUEST} - this event denotes bad arguments to the command and not a problem with the command + *
  • {@link HystrixEventType#SHORT_CIRCUITED} - this event measures a health problem in the past, not a problem with the current state + *
  • {@link HystrixEventType#CANCELLED} - this event denotes a user-cancelled command. It's not known if it would have been a success or failure, so it shouldn't count for either *
  • All Fallback metrics - *
  • {@link HystrixRollingNumberEvent#EMIT} - this event is not a terminal state for the command - *
  • {@link HystrixRollingNumberEvent#COLLAPSED} - this event is about the batching process, not the command execution + *
  • {@link HystrixEventType#EMIT} - this event is not a terminal state for the command + *
  • {@link HystrixEventType#COLLAPSED} - this event is about the batching process, not the command execution *

* * @return {@link HealthCounts} diff --git a/hystrix-core/src/main/java/com/netflix/hystrix/HystrixEventType.java b/hystrix-core/src/main/java/com/netflix/hystrix/HystrixEventType.java index 880d27684..05951310f 100644 --- a/hystrix-core/src/main/java/com/netflix/hystrix/HystrixEventType.java +++ b/hystrix-core/src/main/java/com/netflix/hystrix/HystrixEventType.java @@ -41,6 +41,7 @@ public enum HystrixEventType { FALLBACK_MISSING(true), EXCEPTION_THROWN(false), RESPONSE_FROM_CACHE(true), + CANCELLED(true), COLLAPSED(false); private final boolean isTerminal; diff --git a/hystrix-core/src/main/java/com/netflix/hystrix/HystrixObservableCollapser.java b/hystrix-core/src/main/java/com/netflix/hystrix/HystrixObservableCollapser.java index b2196ef3b..bfb80345a 100644 --- a/hystrix-core/src/main/java/com/netflix/hystrix/HystrixObservableCollapser.java +++ b/hystrix-core/src/main/java/com/netflix/hystrix/HystrixObservableCollapser.java @@ -437,10 +437,10 @@ public Observable toObservable(Scheduler observeOn) { /* try from cache first */ if (isRequestCacheEnabled) { - Observable fromCache = requestCache.get(getCacheKey()); + HystrixCachedObservable fromCache = requestCache.get(getCacheKey()); if (fromCache != null) { metrics.markResponseFromCache(); - return fromCache; + return fromCache.toObservable(); } } @@ -457,12 +457,12 @@ public Observable toObservable(Scheduler observeOn) { * If this is an issue we can make a lazy-future that gets set in the cache * then only the winning 'put' will be invoked to actually call 'submitRequest' */ - Observable o = response.cache(); - Observable fromCache = requestCache.putIfAbsent(getCacheKey(), o); + HystrixCachedObservable toCache = HystrixCachedObservable.from(response, this); + HystrixCachedObservable fromCache = requestCache.putIfAbsent(getCacheKey(), toCache); if (fromCache == null) { - response = o; + return toCache.toObservable(); } else { - response = fromCache; + return fromCache.toObservable(); } } return response; diff --git a/hystrix-core/src/main/java/com/netflix/hystrix/HystrixRequestCache.java b/hystrix-core/src/main/java/com/netflix/hystrix/HystrixRequestCache.java index 53648007e..916196c76 100644 --- a/hystrix-core/src/main/java/com/netflix/hystrix/HystrixRequestCache.java +++ b/hystrix-core/src/main/java/com/netflix/hystrix/HystrixRequestCache.java @@ -22,6 +22,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import rx.Observable; +import rx.internal.operators.CachedObservable; import java.util.concurrent.ConcurrentHashMap; @@ -45,15 +46,15 @@ public class HystrixRequestCache { *

* Key => CommandPrefix + CacheKey : Future from queue() */ - private static final HystrixRequestVariableHolder>> requestVariableForCache = new HystrixRequestVariableHolder>>(new HystrixRequestVariableLifecycle>>() { + private static final HystrixRequestVariableHolder>> requestVariableForCache = new HystrixRequestVariableHolder>>(new HystrixRequestVariableLifecycle>>() { @Override - public ConcurrentHashMap> initialValue() { - return new ConcurrentHashMap>(); + public ConcurrentHashMap> initialValue() { + return new ConcurrentHashMap>(); } @Override - public void shutdown(ConcurrentHashMap> value) { + public void shutdown(ConcurrentHashMap> value) { // nothing to shutdown } @@ -95,15 +96,15 @@ private static HystrixRequestCache getInstance(RequestCacheKey rcKey, HystrixCon */ // suppressing warnings because we are using a raw Future since it's in a heterogeneous ConcurrentHashMap cache @SuppressWarnings({ "unchecked" }) - /* package */ Observable get(String cacheKey) { + /* package */ HystrixCachedObservable get(String cacheKey) { ValueCacheKey key = getRequestCacheKey(cacheKey); if (key != null) { - ConcurrentHashMap> cacheInstance = requestVariableForCache.get(concurrencyStrategy); + ConcurrentHashMap> cacheInstance = requestVariableForCache.get(concurrencyStrategy); if (cacheInstance == null) { throw new IllegalStateException("Request caching is not available. Maybe you need to initialize the HystrixRequestContext?"); } /* look for the stored value */ - return (Observable) cacheInstance.get(key); + return (HystrixCachedObservable) cacheInstance.get(key); } return null; } @@ -122,15 +123,15 @@ private static HystrixRequestCache getInstance(RequestCacheKey rcKey, HystrixCon */ // suppressing warnings because we are using a raw Future since it's in a heterogeneous ConcurrentHashMap cache @SuppressWarnings({ "unchecked" }) - /* package */ Observable putIfAbsent(String cacheKey, Observable f) { + /* package */ HystrixCachedObservable putIfAbsent(String cacheKey, HystrixCachedObservable f) { ValueCacheKey key = getRequestCacheKey(cacheKey); if (key != null) { /* look for the stored value */ - ConcurrentHashMap> cacheInstance = requestVariableForCache.get(concurrencyStrategy); + ConcurrentHashMap> cacheInstance = requestVariableForCache.get(concurrencyStrategy); if (cacheInstance == null) { throw new IllegalStateException("Request caching is not available. Maybe you need to initialize the HystrixRequestContext?"); } - Observable alreadySet = (Observable) cacheInstance.putIfAbsent(key, f); + HystrixCachedObservable alreadySet = (HystrixCachedObservable) cacheInstance.putIfAbsent(key, f); if (alreadySet != null) { // someone beat us so we didn't cache this return alreadySet; diff --git a/hystrix-core/src/main/java/com/netflix/hystrix/strategy/concurrency/HystrixContextScheduler.java b/hystrix-core/src/main/java/com/netflix/hystrix/strategy/concurrency/HystrixContextScheduler.java index 5c870c63a..8f137fc2c 100644 --- a/hystrix-core/src/main/java/com/netflix/hystrix/strategy/concurrency/HystrixContextScheduler.java +++ b/hystrix-core/src/main/java/com/netflix/hystrix/strategy/concurrency/HystrixContextScheduler.java @@ -162,9 +162,6 @@ public Subscription schedule(final Action0 action) { return Subscriptions.unsubscribed(); } - //Schedulers.submitTo(executor, action, subscription, shouldInterrupt); - - // This is internal RxJava API but it is too useful. ScheduledAction sa = new ScheduledAction(action); diff --git a/hystrix-core/src/test/java/com/netflix/hystrix/CommonHystrixCommandTests.java b/hystrix-core/src/test/java/com/netflix/hystrix/CommonHystrixCommandTests.java index 008e16c55..c95308248 100644 --- a/hystrix-core/src/test/java/com/netflix/hystrix/CommonHystrixCommandTests.java +++ b/hystrix-core/src/test/java/com/netflix/hystrix/CommonHystrixCommandTests.java @@ -314,778 +314,6 @@ public boolean isQueueSpaceAvailable() { } - /** - *********************** THREAD-ISOLATED Execution Hook Tests ************************************** - */ - - /** - * Short-circuit? : NO - * Thread/semaphore: THREAD - * Thread Pool full? : NO - * Thread Pool Queue full?: NO - * Timeout: NO - * Execution Result: SUCCESS - */ - @Test - public void testExecutionHookThreadSuccess() { - assertHooksOnSuccess( - new Func0() { - @Override - public C call() { - return getCommand(ExecutionIsolationStrategy.THREAD, ExecutionResult.SUCCESS); - } - }, - new Action1() { - @Override - public void call(C command) { - TestableExecutionHook hook = command.getBuilder().executionHook; - assertTrue(hook.commandEmissionsMatch(1, 0, 1)); - assertTrue(hook.executionEventsMatch(1, 0, 1)); - assertTrue(hook.fallbackEventsMatch(0, 0, 0)); - assertEquals("onStart - onThreadStart - !onRunStart - onExecutionStart - onExecutionEmit - !onRunSuccess - !onComplete - onEmit - onExecutionSuccess - onThreadComplete - onSuccess - ", hook.executionSequence.toString()); - } - }); - } - - /** - * Short-circuit? : NO - * Thread/semaphore: THREAD - * Thread Pool full? : NO - * Thread Pool Queue full?: NO - * Timeout: NO - * Execution Result: synchronous HystrixBadRequestException - */ - @Test - public void testExecutionHookThreadBadRequestException() { - assertHooksOnFailure( - new Func0() { - @Override - public C call() { - return getCommand(ExecutionIsolationStrategy.THREAD, ExecutionResult.BAD_REQUEST); - } - }, - new Action1() { - @Override - public void call(C command) { - TestableExecutionHook hook = command.getBuilder().executionHook; - assertTrue(hook.commandEmissionsMatch(0, 1, 0)); - assertTrue(hook.executionEventsMatch(0, 1, 0)); - assertTrue(hook.fallbackEventsMatch(0, 0, 0)); - assertEquals(HystrixBadRequestException.class, hook.getCommandException().getClass()); - assertEquals(HystrixBadRequestException.class, hook.getExecutionException().getClass()); - assertEquals("onStart - onThreadStart - !onRunStart - onExecutionStart - onExecutionError - !onRunError - onError - onThreadComplete - ", hook.executionSequence.toString()); - } - }); - } - - - - /** - * Short-circuit? : NO - * Thread/semaphore: THREAD - * Thread Pool full? : NO - * Thread Pool Queue full?: NO - * Timeout: NO - * Execution Result: synchronous HystrixRuntimeException - * Fallback: UnsupportedOperationException - */ - @Test - public void testExecutionHookThreadExceptionNoFallback() { - assertHooksOnFailure( - new Func0() { - @Override - public C call() { - return getCommand(ExecutionIsolationStrategy.THREAD, ExecutionResult.FAILURE, 0, FallbackResult.UNIMPLEMENTED); - } - }, - new Action1() { - @Override - public void call(C command) { - TestableExecutionHook hook = command.getBuilder().executionHook; - assertTrue(hook.commandEmissionsMatch(0, 1, 0)); - assertTrue(hook.executionEventsMatch(0, 1, 0)); - assertTrue(hook.fallbackEventsMatch(0, 0, 0)); - assertEquals(RuntimeException.class, hook.getCommandException().getClass()); - assertEquals(RuntimeException.class, hook.getExecutionException().getClass()); - assertNull(hook.getFallbackException()); - assertEquals("onStart - onThreadStart - !onRunStart - onExecutionStart - onExecutionError - !onRunError - onError - onThreadComplete - ", hook.executionSequence.toString()); - } - }); - } - - /** - * Short-circuit? : NO - * Thread/semaphore: THREAD - * Thread Pool full? : NO - * Thread Pool Queue full?: NO - * Timeout: NO - * Execution Result: synchronous HystrixRuntimeException - * Fallback: SUCCESS - */ - @Test - public void testExecutionHookThreadExceptionSuccessfulFallback() { - assertHooksOnSuccess( - new Func0() { - @Override - public C call() { - return getCommand(ExecutionIsolationStrategy.THREAD, ExecutionResult.FAILURE, 0, FallbackResult.SUCCESS); - } - }, - new Action1() { - @Override - public void call(C command) { - TestableExecutionHook hook = command.getBuilder().executionHook; - assertTrue(hook.commandEmissionsMatch(1, 0, 1)); - assertTrue(hook.executionEventsMatch(0, 1, 0)); - assertTrue(hook.fallbackEventsMatch(1, 0, 1)); - assertEquals(RuntimeException.class, hook.getExecutionException().getClass()); - assertEquals("onStart - onThreadStart - !onRunStart - onExecutionStart - onExecutionError - !onRunError - onFallbackStart - onFallbackEmit - !onFallbackSuccess - !onComplete - onEmit - onFallbackSuccess - onThreadComplete - onSuccess - ", hook.executionSequence.toString()); - } - }); - } - - /** - * Short-circuit? : NO - * Thread/semaphore: THREAD - * Thread Pool full? : NO - * Thread Pool Queue full?: NO - * Timeout: NO - * Execution Result: synchronous HystrixRuntimeException - * Fallback: HystrixRuntimeException - */ - @Test - public void testExecutionHookThreadExceptionUnsuccessfulFallback() { - assertHooksOnFailure( - new Func0() { - @Override - public C call() { - return getCommand(ExecutionIsolationStrategy.THREAD, ExecutionResult.FAILURE, 0, FallbackResult.FAILURE); - } - }, - new Action1() { - @Override - public void call(C command) { - TestableExecutionHook hook = command.getBuilder().executionHook; - assertTrue(hook.commandEmissionsMatch(0, 1, 0)); - assertTrue(hook.executionEventsMatch(0, 1, 0)); - assertTrue(hook.fallbackEventsMatch(0, 1, 0)); - assertEquals(RuntimeException.class, hook.getCommandException().getClass()); - assertEquals(RuntimeException.class, hook.getExecutionException().getClass()); - assertEquals(RuntimeException.class, hook.getFallbackException().getClass()); - assertEquals("onStart - onThreadStart - !onRunStart - onExecutionStart - onExecutionError - !onRunError - onFallbackStart - onFallbackError - onError - onThreadComplete - ", hook.executionSequence.toString()); - } - }); - } - - /** - * Short-circuit? : NO - * Thread/semaphore: THREAD - * Thread Pool full? : NO - * Thread Pool Queue full?: NO - * Timeout: YES - * Execution Result: SUCCESS (but timeout prior) - * Fallback: UnsupportedOperationException - */ - @Test - public void testExecutionHookThreadTimeoutNoFallbackRunSuccess() { - assertHooksOnFailure( - new Func0() { - @Override - public C call() { - return getCommand(ExecutionIsolationStrategy.THREAD, ExecutionResult.SUCCESS, 150, FallbackResult.UNIMPLEMENTED, 50); - } - }, - new Action1() { - @Override - public void call(C command) { - TestableExecutionHook hook = command.getBuilder().executionHook; - assertTrue(hook.commandEmissionsMatch(0, 1, 0)); - assertTrue(hook.executionEventsMatch(0, 0, 0)); - assertTrue(hook.fallbackEventsMatch(0, 0, 0)); - assertEquals(TimeoutException.class, hook.getCommandException().getClass()); - assertNull(hook.getFallbackException()); - assertEquals("onStart - onThreadStart - !onRunStart - onExecutionStart - onError - ", hook.executionSequence.toString()); - - try { - Thread.sleep(300); - } catch (Exception ex) { - throw new RuntimeException(ex); - } - - assertTrue(hook.commandEmissionsMatch(0, 1, 0)); - assertTrue(hook.executionEventsMatch(1, 0, 1)); - assertTrue(hook.fallbackEventsMatch(0, 0, 0)); - assertEquals("onStart - onThreadStart - !onRunStart - onExecutionStart - onError - onExecutionEmit - !onRunSuccess - onExecutionSuccess - onThreadComplete - ", hook.executionSequence.toString()); - - } - }); - } - - /** - * Short-circuit? : NO - * Thread/semaphore: THREAD - * Thread Pool full? : NO - * Thread Pool Queue full?: NO - * Timeout: YES - * Execution Result: SUCCESS (but timeout prior) - * Fallback: SUCCESS - */ - @Test - public void testExecutionHookThreadTimeoutSuccessfulFallbackRunSuccess() { - assertHooksOnSuccess( - new Func0() { - @Override - public C call() { - return getCommand(ExecutionIsolationStrategy.THREAD, ExecutionResult.SUCCESS, 150, FallbackResult.SUCCESS, 50); - } - }, - new Action1() { - @Override - public void call(C command) { - TestableExecutionHook hook = command.getBuilder().executionHook; - assertTrue(hook.commandEmissionsMatch(1, 0, 1)); - assertTrue(hook.executionEventsMatch(0, 0, 0)); - assertTrue(hook.fallbackEventsMatch(1, 0, 1)); - assertEquals("onStart - onThreadStart - !onRunStart - onExecutionStart - onFallbackStart - onFallbackEmit - !onFallbackSuccess - !onComplete - onEmit - onFallbackSuccess - onSuccess - ", hook.executionSequence.toString()); - - try { - Thread.sleep(300); - } catch (Exception ex) { - throw new RuntimeException(ex); - } - - assertTrue(hook.commandEmissionsMatch(1, 0, 1)); - assertTrue(hook.executionEventsMatch(1, 0, 1)); - assertTrue(hook.fallbackEventsMatch(1, 0, 1)); - assertEquals("onStart - onThreadStart - !onRunStart - onExecutionStart - onFallbackStart - onFallbackEmit - !onFallbackSuccess - !onComplete - onEmit - onFallbackSuccess - onSuccess - onExecutionEmit - !onRunSuccess - onExecutionSuccess - onThreadComplete - ", hook.executionSequence.toString()); - - } - }); - } - - /** - * Short-circuit? : NO - * Thread/semaphore: THREAD - * Thread Pool full? : NO - * Thread Pool Queue full?: NO - * Timeout: YES - * Execution Result: SUCCESS (but timeout prior) - * Fallback: synchronous HystrixRuntimeException - */ - @Test - public void testExecutionHookThreadTimeoutUnsuccessfulFallbackRunSuccess() { - assertHooksOnFailure( - new Func0() { - @Override - public C call() { - return getCommand(ExecutionIsolationStrategy.THREAD, ExecutionResult.SUCCESS, 150, FallbackResult.FAILURE, 50); - } - }, - new Action1() { - @Override - public void call(C command) { - TestableExecutionHook hook = command.getBuilder().executionHook; - assertTrue(hook.commandEmissionsMatch(0, 1, 0)); - assertTrue(hook.executionEventsMatch(0, 0, 0)); - assertTrue(hook.fallbackEventsMatch(0, 1, 0)); - assertEquals(TimeoutException.class, hook.getCommandException().getClass()); - assertEquals(RuntimeException.class, hook.getFallbackException().getClass()); - assertEquals("onStart - onThreadStart - !onRunStart - onExecutionStart - onFallbackStart - onFallbackError - onError - ", hook.executionSequence.toString()); - - try { - Thread.sleep(300); - } catch (Exception ex) { - throw new RuntimeException(ex); - } - assertTrue(hook.commandEmissionsMatch(0, 1, 0)); - assertTrue(hook.executionEventsMatch(1, 0, 1)); - assertTrue(hook.fallbackEventsMatch(0, 1, 0)); - assertEquals("onStart - onThreadStart - !onRunStart - onExecutionStart - onFallbackStart - onFallbackError - onError - onExecutionEmit - !onRunSuccess - onExecutionSuccess - onThreadComplete - ", hook.executionSequence.toString()); - - } - }); - } - - /** - * Short-circuit? : NO - * Thread/semaphore: THREAD - * Thread Pool full? : NO - * Thread Pool Queue full?: NO - * Timeout: YES - * Execution Result: HystrixRuntimeException (but timeout prior) - * Fallback: UnsupportedOperationException - */ - @Test - public void testExecutionHookThreadTimeoutNoFallbackRunFailure() { - assertHooksOnFailure( - new Func0() { - @Override - public C call() { - return getCommand(ExecutionIsolationStrategy.THREAD, ExecutionResult.FAILURE, 150, FallbackResult.UNIMPLEMENTED, 50); - } - }, - new Action1() { - @Override - public void call(C command) { - TestableExecutionHook hook = command.getBuilder().executionHook; - assertTrue(hook.commandEmissionsMatch(0, 1, 0)); - assertTrue(hook.executionEventsMatch(0, 0, 0)); - assertTrue(hook.fallbackEventsMatch(0, 0, 0)); - assertEquals(TimeoutException.class, hook.getCommandException().getClass()); - assertNull(hook.getFallbackException()); - assertEquals("onStart - onThreadStart - !onRunStart - onExecutionStart - onError - ", hook.executionSequence.toString()); - - try { - Thread.sleep(300); - } catch (Exception ex) { - throw new RuntimeException(ex); - } - assertTrue(hook.commandEmissionsMatch(0, 1, 0)); - assertTrue(hook.executionEventsMatch(0, 1, 0)); - assertTrue(hook.fallbackEventsMatch(0, 0, 0)); - assertEquals(RuntimeException.class, hook.getExecutionException().getClass()); - assertEquals("onStart - onThreadStart - !onRunStart - onExecutionStart - onError - onExecutionError - !onRunError - onThreadComplete - ", hook.executionSequence.toString()); - - } - }); - } - - /** - * Short-circuit? : NO - * Thread/semaphore: THREAD - * Thread Pool full? : NO - * Thread Pool Queue full?: NO - * Timeout: YES - * Execution Result: HystrixRuntimeException (but timeout prior) - * Fallback: SUCCESS - */ - @Test - public void testExecutionHookThreadTimeoutSuccessfulFallbackRunFailure() { - assertHooksOnSuccess( - new Func0() { - @Override - public C call() { - return getCommand(ExecutionIsolationStrategy.THREAD, ExecutionResult.FAILURE, 150, FallbackResult.SUCCESS, 50); - } - }, - new Action1() { - @Override - public void call(C command) { - TestableExecutionHook hook = command.getBuilder().executionHook; - assertTrue(hook.commandEmissionsMatch(1, 0, 1)); - assertTrue(hook.executionEventsMatch(0, 0, 0)); - assertTrue(hook.fallbackEventsMatch(1, 0, 1)); - assertEquals("onStart - onThreadStart - !onRunStart - onExecutionStart - onFallbackStart - onFallbackEmit - !onFallbackSuccess - !onComplete - onEmit - onFallbackSuccess - onSuccess - ", hook.executionSequence.toString()); - - try { - Thread.sleep(300); - } catch (Exception ex) { - throw new RuntimeException(ex); - } - assertTrue(hook.commandEmissionsMatch(1, 0, 1)); - assertTrue(hook.executionEventsMatch(0, 1, 0)); - assertTrue(hook.fallbackEventsMatch(1, 0, 1)); - assertEquals(RuntimeException.class, hook.getExecutionException().getClass()); - assertEquals("onStart - onThreadStart - !onRunStart - onExecutionStart - onFallbackStart - onFallbackEmit - !onFallbackSuccess - !onComplete - onEmit - onFallbackSuccess - onSuccess - onExecutionError - !onRunError - onThreadComplete - ", hook.executionSequence.toString()); - - } - }); - } - - /** - * Short-circuit? : NO - * Thread/semaphore: THREAD - * Thread Pool full? : NO - * Thread Pool Queue full?: NO - * Timeout: YES - * Execution Result: HystrixRuntimeException (but timeout prior) - * Fallback: synchronous HystrixRuntimeException - */ - @Test - public void testExecutionHookThreadTimeoutUnsuccessfulFallbackRunFailure() { - assertHooksOnFailure( - new Func0() { - @Override - public C call() { - return getCommand(ExecutionIsolationStrategy.THREAD, ExecutionResult.FAILURE, 150, FallbackResult.FAILURE, 50); - } - }, - new Action1() { - @Override - public void call(C command) { - TestableExecutionHook hook = command.getBuilder().executionHook; - assertTrue(hook.commandEmissionsMatch(0, 1, 0)); - assertTrue(hook.executionEventsMatch(0, 0, 0)); - assertTrue(hook.fallbackEventsMatch(0, 1, 0)); - assertEquals(TimeoutException.class, hook.getCommandException().getClass()); - assertEquals(RuntimeException.class, hook.getFallbackException().getClass()); - assertEquals("onStart - onThreadStart - !onRunStart - onExecutionStart - onFallbackStart - onFallbackError - onError - ", hook.executionSequence.toString()); - - try { - Thread.sleep(300); - } catch (Exception ex) { - throw new RuntimeException(ex); - } - assertTrue(hook.commandEmissionsMatch(0, 1, 0)); - assertTrue(hook.executionEventsMatch(0, 1, 0)); - assertTrue(hook.fallbackEventsMatch(0, 1, 0)); - assertEquals(RuntimeException.class, hook.getExecutionException().getClass()); - assertEquals("onStart - onThreadStart - !onRunStart - onExecutionStart - onFallbackStart - onFallbackError - onError - onExecutionError - !onRunError - onThreadComplete - ", hook.executionSequence.toString()); - - } - }); - } - - /** - * Short-circuit? : NO - * Thread/semaphore: THREAD - * Thread Pool full? : YES - * Thread Pool Queue full?: YES - * Fallback: UnsupportedOperationException - */ - @Test - public void testExecutionHookThreadPoolQueueFullNoFallback() { - assertHooksOnFailFast( - new Func0() { - @Override - public C call() { - HystrixCircuitBreakerTest.TestCircuitBreaker circuitBreaker = new HystrixCircuitBreakerTest.TestCircuitBreaker(); - HystrixThreadPool pool = new SingleThreadedPoolWithQueue(1); - try { - // fill the pool - getLatentCommand(ExecutionIsolationStrategy.THREAD, ExecutionResult.SUCCESS, 500, FallbackResult.UNIMPLEMENTED, circuitBreaker, pool, 600).observe(); - // fill the queue - getLatentCommand(ExecutionIsolationStrategy.THREAD, ExecutionResult.SUCCESS, 500, FallbackResult.UNIMPLEMENTED, circuitBreaker, pool, 600).observe(); - } catch (Exception e) { - // ignore - } - return getLatentCommand(ExecutionIsolationStrategy.THREAD, ExecutionResult.SUCCESS, 500, FallbackResult.UNIMPLEMENTED, circuitBreaker, pool, 600); - } - }, - new Action1() { - @Override - public void call(C command) { - TestableExecutionHook hook = command.getBuilder().executionHook; - assertTrue(hook.commandEmissionsMatch(0, 1, 0)); - assertTrue(hook.executionEventsMatch(0, 0, 0)); - assertTrue(hook.fallbackEventsMatch(0, 0, 0)); - assertEquals(RejectedExecutionException.class, hook.getCommandException().getClass()); - assertNull(hook.getFallbackException()); - assertEquals("onStart - onError - ", hook.executionSequence.toString()); - } - }); - } - - /** - * Short-circuit? : NO - * Thread/semaphore: THREAD - * Thread Pool full? : YES - * Thread Pool Queue full?: YES - * Fallback: SUCCESS - */ - @Test - public void testExecutionHookThreadPoolQueueFullSuccessfulFallback() { - assertHooksOnSuccess( - new Func0() { - @Override - public C call() { - HystrixCircuitBreakerTest.TestCircuitBreaker circuitBreaker = new HystrixCircuitBreakerTest.TestCircuitBreaker(); - HystrixThreadPool pool = new SingleThreadedPoolWithQueue(1); - try { - // fill the pool - getLatentCommand(ExecutionIsolationStrategy.THREAD, ExecutionResult.SUCCESS, 500, FallbackResult.SUCCESS, circuitBreaker, pool, 600).observe(); - // fill the queue - getLatentCommand(ExecutionIsolationStrategy.THREAD, ExecutionResult.SUCCESS, 500, FallbackResult.SUCCESS, circuitBreaker, pool, 600).observe(); - } catch (Exception e) { - // ignore - } - - return getLatentCommand(ExecutionIsolationStrategy.THREAD, ExecutionResult.SUCCESS, 500, FallbackResult.SUCCESS, circuitBreaker, pool, 600); - } - }, - new Action1() { - @Override - public void call(C command) { - TestableExecutionHook hook = command.getBuilder().executionHook; - assertTrue(hook.commandEmissionsMatch(1, 0, 1)); - assertTrue(hook.executionEventsMatch(0, 0, 0)); - assertTrue(hook.fallbackEventsMatch(1, 0, 1)); - assertEquals("onStart - onFallbackStart - onFallbackEmit - !onFallbackSuccess - !onComplete - onEmit - onFallbackSuccess - onSuccess - ", hook.executionSequence.toString()); - } - }); - } - - /** - * Short-circuit? : NO - * Thread/semaphore: THREAD - * Thread Pool full? : YES - * Thread Pool Queue full?: YES - * Fallback: synchronous HystrixRuntimeException - */ - @Test - public void testExecutionHookThreadPoolQueueFullUnsuccessfulFallback() { - assertHooksOnFailFast( - new Func0() { - @Override - public C call() { - HystrixCircuitBreakerTest.TestCircuitBreaker circuitBreaker = new HystrixCircuitBreakerTest.TestCircuitBreaker(); - HystrixThreadPool pool = new SingleThreadedPoolWithQueue(1); - try { - // fill the pool - getLatentCommand(ExecutionIsolationStrategy.THREAD, ExecutionResult.SUCCESS, 500, FallbackResult.FAILURE, circuitBreaker, pool, 600).observe(); - // fill the queue - getLatentCommand(ExecutionIsolationStrategy.THREAD, ExecutionResult.SUCCESS, 500, FallbackResult.FAILURE, circuitBreaker, pool, 600).observe(); - } catch (Exception e) { - // ignore - } - - return getLatentCommand(ExecutionIsolationStrategy.THREAD, ExecutionResult.SUCCESS, 500, FallbackResult.FAILURE, circuitBreaker, pool, 600); - } - }, - new Action1() { - @Override - public void call(C command) { - TestableExecutionHook hook = command.getBuilder().executionHook; - assertTrue(hook.commandEmissionsMatch(0, 1, 0)); - assertTrue(hook.executionEventsMatch(0, 0, 0)); - assertTrue(hook.fallbackEventsMatch(0, 1, 0)); - assertEquals(RejectedExecutionException.class, hook.getCommandException().getClass()); - assertEquals(RuntimeException.class, hook.getFallbackException().getClass()); - assertEquals("onStart - onFallbackStart - onFallbackError - onError - ", hook.executionSequence.toString()); - } - }); - } - - /** - * Short-circuit? : NO - * Thread/semaphore: THREAD - * Thread Pool full? : YES - * Thread Pool Queue full?: N/A - * Fallback: UnsupportedOperationException - */ - @Test - public void testExecutionHookThreadPoolFullNoFallback() { - assertHooksOnFailFast( - new Func0() { - @Override - public C call() { - HystrixCircuitBreakerTest.TestCircuitBreaker circuitBreaker = new HystrixCircuitBreakerTest.TestCircuitBreaker(); - HystrixThreadPool pool = new SingleThreadedPoolWithNoQueue(); - try { - // fill the pool - getLatentCommand(ExecutionIsolationStrategy.THREAD, ExecutionResult.SUCCESS, 500, FallbackResult.UNIMPLEMENTED, circuitBreaker, pool, 600).observe(); - } catch (Exception e) { - // ignore - } - - return getLatentCommand(ExecutionIsolationStrategy.THREAD, ExecutionResult.SUCCESS, 500, FallbackResult.UNIMPLEMENTED, circuitBreaker, pool, 600); - } - }, - new Action1() { - @Override - public void call(C command) { - TestableExecutionHook hook = command.getBuilder().executionHook; - assertTrue(hook.commandEmissionsMatch(0, 1, 0)); - assertTrue(hook.executionEventsMatch(0, 0, 0)); - assertTrue(hook.fallbackEventsMatch(0, 0, 0)); - assertEquals(RejectedExecutionException.class, hook.getCommandException().getClass()); - assertNull(hook.getFallbackException()); - assertEquals("onStart - onError - ", hook.executionSequence.toString()); - } - }); - } - - /** - * Short-circuit? : NO - * Thread/semaphore: THREAD - * Thread Pool full? : YES - * Thread Pool Queue full?: N/A - * Fallback: SUCCESS - */ - @Test - public void testExecutionHookThreadPoolFullSuccessfulFallback() { - assertHooksOnSuccess( - new Func0() { - @Override - public C call() { - HystrixCircuitBreakerTest.TestCircuitBreaker circuitBreaker = new HystrixCircuitBreakerTest.TestCircuitBreaker(); - HystrixThreadPool pool = new SingleThreadedPoolWithNoQueue(); - try { - // fill the pool - getLatentCommand(ExecutionIsolationStrategy.THREAD, ExecutionResult.SUCCESS, 500, FallbackResult.SUCCESS, circuitBreaker, pool, 600).observe(); - } catch (Exception e) { - // ignore - } - - return getLatentCommand(ExecutionIsolationStrategy.THREAD, ExecutionResult.SUCCESS, 500, FallbackResult.SUCCESS, circuitBreaker, pool, 600); - } - }, - new Action1() { - @Override - public void call(C command) { - TestableExecutionHook hook = command.getBuilder().executionHook; - assertTrue(hook.commandEmissionsMatch(1, 0, 1)); - assertTrue(hook.executionEventsMatch(0, 0, 0)); - assertEquals("onStart - onFallbackStart - onFallbackEmit - !onFallbackSuccess - !onComplete - onEmit - onFallbackSuccess - onSuccess - ", hook.executionSequence.toString()); - } - }); - } - - /** - * Short-circuit? : NO - * Thread/semaphore: THREAD - * Thread Pool full? : YES - * Thread Pool Queue full?: N/A - * Fallback: synchronous HystrixRuntimeException - */ - @Test - public void testExecutionHookThreadPoolFullUnsuccessfulFallback() { - assertHooksOnFailFast( - new Func0() { - @Override - public C call() { - HystrixCircuitBreakerTest.TestCircuitBreaker circuitBreaker = new HystrixCircuitBreakerTest.TestCircuitBreaker(); - HystrixThreadPool pool = new SingleThreadedPoolWithNoQueue(); - try { - // fill the pool - getLatentCommand(ExecutionIsolationStrategy.THREAD, ExecutionResult.SUCCESS, 500, FallbackResult.FAILURE, circuitBreaker, pool, 600).observe(); - } catch (Exception e) { - // ignore - } - - return getLatentCommand(ExecutionIsolationStrategy.THREAD, ExecutionResult.SUCCESS, 500, FallbackResult.FAILURE, circuitBreaker, pool, 600); - } - }, - new Action1() { - @Override - public void call(C command) { - TestableExecutionHook hook = command.getBuilder().executionHook; - assertTrue(hook.commandEmissionsMatch(0, 1, 0)); - assertTrue(hook.executionEventsMatch(0, 0, 0)); - assertTrue(hook.fallbackEventsMatch(0, 1, 0)); - assertEquals(RejectedExecutionException.class, hook.getCommandException().getClass()); - assertEquals(RuntimeException.class, hook.getFallbackException().getClass()); - assertEquals("onStart - onFallbackStart - onFallbackError - onError - ", hook.executionSequence.toString()); - } - }); - } - - /** - * Short-circuit? : YES - * Thread/semaphore: THREAD - * Fallback: UnsupportedOperationException - */ - @Test - public void testExecutionHookThreadShortCircuitNoFallback() { - assertHooksOnFailFast( - new Func0() { - @Override - public C call() { - return getCircuitOpenCommand(ExecutionIsolationStrategy.THREAD, FallbackResult.UNIMPLEMENTED); - } - }, - new Action1() { - @Override - public void call(C command) { - TestableExecutionHook hook = command.getBuilder().executionHook; - assertTrue(hook.commandEmissionsMatch(0, 1, 0)); - assertTrue(hook.executionEventsMatch(0, 0, 0)); - assertTrue(hook.fallbackEventsMatch(0, 0, 0)); - assertEquals(RuntimeException.class, hook.getCommandException().getClass()); - assertNull(hook.getFallbackException()); - assertEquals("onStart - onError - ", hook.executionSequence.toString()); - } - }); - } - - /** - * Short-circuit? : YES - * Thread/semaphore: THREAD - * Fallback: SUCCESS - */ - @Test - public void testExecutionHookThreadShortCircuitSuccessfulFallback() { - assertHooksOnSuccess( - new Func0() { - @Override - public C call() { - return getCircuitOpenCommand(ExecutionIsolationStrategy.THREAD, FallbackResult.SUCCESS); - } - }, - new Action1() { - @Override - public void call(C command) { - TestableExecutionHook hook = command.getBuilder().executionHook; - assertTrue(hook.commandEmissionsMatch(1, 0, 1)); - assertTrue(hook.executionEventsMatch(0, 0, 0)); - assertTrue(hook.fallbackEventsMatch(1, 0, 1)); - assertEquals("onStart - onFallbackStart - onFallbackEmit - !onFallbackSuccess - !onComplete - onEmit - onFallbackSuccess - onSuccess - ", hook.executionSequence.toString()); - } - }); - } - - /** - * Short-circuit? : YES - * Thread/semaphore: THREAD - * Fallback: synchronous HystrixRuntimeException - */ - @Test - public void testExecutionHookThreadShortCircuitUnsuccessfulFallback() { - assertHooksOnFailFast( - new Func0() { - @Override - public C call() { - HystrixCircuitBreakerTest.TestCircuitBreaker circuitBreaker = new HystrixCircuitBreakerTest.TestCircuitBreaker().setForceShortCircuit(true); - return getCircuitOpenCommand(ExecutionIsolationStrategy.THREAD, FallbackResult.FAILURE); - } - }, - new Action1() { - @Override - public void call(C command) { - TestableExecutionHook hook = command.getBuilder().executionHook; - assertTrue(hook.commandEmissionsMatch(0, 1, 0)); - assertTrue(hook.executionEventsMatch(0, 0, 0)); - assertTrue(hook.fallbackEventsMatch(0, 1, 0)); - assertEquals(RuntimeException.class, hook.getCommandException().getClass()); - assertEquals(RuntimeException.class, hook.getFallbackException().getClass()); - assertEquals("onStart - onFallbackStart - onFallbackError - onError - ", hook.executionSequence.toString()); - } - }); - } - - /** - * Short-circuit? : NO - * Request-cache? : YES - */ - @Test - public void testExecutionHookResponseFromCache() { - final HystrixCommandKey key = HystrixCommandKey.Factory.asKey("Hook-Cache"); - getCommand(key, ExecutionIsolationStrategy.THREAD, ExecutionResult.SUCCESS, 0, FallbackResult.UNIMPLEMENTED, 0, new HystrixCircuitBreakerTest.TestCircuitBreaker(), null, 100, CacheEnabled.YES, 42, 10, 10).observe(); - - assertHooksOnSuccess( - new Func0() { - @Override - public C call() { - return getCommand(key, ExecutionIsolationStrategy.THREAD, ExecutionResult.SUCCESS, 0, FallbackResult.UNIMPLEMENTED, 0, new HystrixCircuitBreakerTest.TestCircuitBreaker(), null, 100, CacheEnabled.YES, 42, 10, 10); - } - }, - new Action1() { - @Override - public void call(C command) { - TestableExecutionHook hook = command.getBuilder().executionHook; - assertTrue(hook.commandEmissionsMatch(0, 0, 0)); - assertTrue(hook.executionEventsMatch(0, 0, 0)); - assertTrue(hook.fallbackEventsMatch(0, 0, 0)); - assertEquals("onCacheHit - ", hook.executionSequence.toString()); - } - }); - } - - - - /** - *********************** END THREAD-ISOLATED Execution Hook Tests ************************************** - */ /** ********************* SEMAPHORE-ISOLATED Execution Hook Tests *********************************** @@ -1515,7 +743,7 @@ C getCommand(ExecutionIsolationStrategy isolationStrategy, ExecutionResult execu return getCommand(isolationStrategy, executionResult, executionLatency, fallbackResult, fallbackLatency, circuitBreaker, threadPool, timeout, cacheEnabled, value, executionSemaphoreCount, fallbackSemaphoreCount, false); } - private C getCommand(HystrixCommandKey key, ExecutionIsolationStrategy isolationStrategy, ExecutionResult executionResult, int executionLatency, FallbackResult fallbackResult, int fallbackLatency, HystrixCircuitBreakerTest.TestCircuitBreaker circuitBreaker, HystrixThreadPool threadPool, int timeout, CacheEnabled cacheEnabled, Object value, int executionSemaphoreCount, int fallbackSemaphoreCount) { + C getCommand(HystrixCommandKey key, ExecutionIsolationStrategy isolationStrategy, ExecutionResult executionResult, int executionLatency, FallbackResult fallbackResult, int fallbackLatency, HystrixCircuitBreakerTest.TestCircuitBreaker circuitBreaker, HystrixThreadPool threadPool, int timeout, CacheEnabled cacheEnabled, Object value, int executionSemaphoreCount, int fallbackSemaphoreCount) { AbstractCommand.TryableSemaphoreActual executionSemaphore = new AbstractCommand.TryableSemaphoreActual(HystrixProperty.Factory.asProperty(executionSemaphoreCount)); AbstractCommand.TryableSemaphoreActual fallbackSemaphore = new AbstractCommand.TryableSemaphoreActual(HystrixProperty.Factory.asProperty(fallbackSemaphoreCount)); diff --git a/hystrix-core/src/test/java/com/netflix/hystrix/HystrixCommandTest.java b/hystrix-core/src/test/java/com/netflix/hystrix/HystrixCommandTest.java index bb9ec7af9..f8c5823bb 100644 --- a/hystrix-core/src/test/java/com/netflix/hystrix/HystrixCommandTest.java +++ b/hystrix-core/src/test/java/com/netflix/hystrix/HystrixCommandTest.java @@ -39,7 +39,9 @@ import rx.Subscription; import rx.functions.Action0; import rx.functions.Action1; +import rx.functions.Action2; import rx.functions.Func0; +import rx.observables.AsyncOnSubscribe; import rx.observers.TestSubscriber; import rx.schedulers.Schedulers; @@ -91,6 +93,7 @@ public void testExecutionSuccess() { assertCommandExecutionEvents(command, HystrixEventType.SUCCESS); assertEquals(0, command.getBuilder().metrics.getCurrentConcurrentExecutionCount()); + System.out.println("ReqLog : " + HystrixRequestLog.getCurrentRequest().getExecutedCommandsAsString()); assertSaneHystrixRequestLog(1); } @@ -1565,7 +1568,6 @@ public void testRequestCache1() throws Exception { assertFalse(command2.executed); assertTrue(command1.getExecutionTimeInMilliseconds() > -1); assertFalse(command1.isResponseFromCache()); - assertTrue(command2.getExecutionTimeInMilliseconds() == -1); assertTrue(command2.isResponseFromCache()); assertCommandExecutionEvents(command1, HystrixEventType.SUCCESS); assertCommandExecutionEvents(command2, HystrixEventType.SUCCESS, HystrixEventType.RESPONSE_FROM_CACHE); @@ -1628,12 +1630,12 @@ public void testRequestCache3() throws Exception { assertTrue(command2.executed); // but the 3rd should come from cache assertFalse(command3.executed); - assertTrue(command3.getExecutionTimeInMilliseconds() == -1); assertTrue(command3.isResponseFromCache()); assertCommandExecutionEvents(command1, HystrixEventType.SUCCESS); assertCommandExecutionEvents(command2, HystrixEventType.SUCCESS); assertCommandExecutionEvents(command3, HystrixEventType.SUCCESS, HystrixEventType.RESPONSE_FROM_CACHE); assertEquals(0, circuitBreaker.metrics.getCurrentConcurrentExecutionCount()); + System.out.println("ReqLog : " + HystrixRequestLog.getCurrentRequest().getExecutedCommandsAsString()); assertSaneHystrixRequestLog(3); } @@ -2928,6 +2930,8 @@ protected Integer run() throws Exception { assertFalse(onThreadStartInvoked.get()); assertFalse(onThreadCompleteInvoked.get()); assertFalse(executionAttempted.get()); + assertEquals(0, semaphoreCmd.metrics.getCurrentConcurrentExecutionCount()); + } @Test @@ -2982,10 +2986,14 @@ protected Integer run() throws Exception { assertTrue(onThreadStartInvoked.get()); assertTrue(onThreadCompleteInvoked.get()); assertFalse(executionAttempted.get()); + assertEquals(0, threadCmd.metrics.getCurrentConcurrentExecutionCount()); + } @Test - public void testEarlyUnsubscribeDuringExecution() { + public void testEarlyUnsubscribeDuringExecutionViaToObservable() { + final AtomicBoolean hystrixThreadStartedExecuting = new AtomicBoolean(false); + class AsyncCommand extends HystrixCommand { public AsyncCommand() { @@ -2994,6 +3002,7 @@ public AsyncCommand() { @Override protected Boolean run() { + hystrixThreadStartedExecuting.set(true); try { Thread.sleep(100); return true; @@ -3020,8 +3029,81 @@ public void call() { @Override public void onCompleted() { System.out.println("OnCompleted"); + } + + @Override + public void onError(Throwable e) { + System.out.println("OnError : " + e); + } + + @Override + public void onNext(Boolean b) { + System.out.println("OnNext : " + b); + } + }); + + try { + Thread.sleep(10); + s.unsubscribe(); + assertTrue(latch.await(200, TimeUnit.MILLISECONDS)); + assertEquals("Number of execution semaphores in use", 0, cmd.getExecutionSemaphore().getNumberOfPermitsUsed()); + assertEquals("Number of fallback semaphores in use", 0, cmd.getFallbackSemaphore().getNumberOfPermitsUsed()); + assertFalse(cmd.isExecutionComplete()); + assertEquals(hystrixThreadStartedExecuting.get(), cmd.isExecutedInThread()); + assertEquals(null, cmd.getFailedExecutionException()); + assertNull(cmd.getExecutionException()); + System.out.println("Execution time : " + cmd.getExecutionTimeInMilliseconds()); + assertTrue(cmd.getExecutionTimeInMilliseconds() > -1); + assertFalse(cmd.isSuccessfulExecution()); + assertCommandExecutionEvents(cmd, HystrixEventType.CANCELLED); + assertEquals(0, cmd.metrics.getCurrentConcurrentExecutionCount()); + System.out.println("ReqLog : " + HystrixRequestLog.getCurrentRequest().getExecutedCommandsAsString()); + assertSaneHystrixRequestLog(1); + } catch (InterruptedException ex) { + ex.printStackTrace(); + } + } + + @Test + public void testEarlyUnsubscribeDuringExecutionViaObserve() { + final AtomicBoolean hystrixThreadStartedExecuting = new AtomicBoolean(false); + + class AsyncCommand extends HystrixCommand { + + public AsyncCommand() { + super(Setter.withGroupKey(HystrixCommandGroupKey.Factory.asKey("ASYNC"))); + } + + @Override + protected Boolean run() { + try { + hystrixThreadStartedExecuting.set(true); + Thread.sleep(100); + return true; + } catch (InterruptedException ex) { + throw new RuntimeException(ex); + } + } + } + + HystrixCommand cmd = new AsyncCommand(); + + final CountDownLatch latch = new CountDownLatch(1); + + Observable o = cmd.observe(); + Subscription s = o. + doOnUnsubscribe(new Action0() { + @Override + public void call() { + System.out.println("OnUnsubscribe"); latch.countDown(); } + }). + subscribe(new Subscriber() { + @Override + public void onCompleted() { + System.out.println("OnCompleted"); + } @Override public void onError(Throwable e) { @@ -3035,15 +3117,21 @@ public void onNext(Boolean b) { }); try { + Thread.sleep(10); s.unsubscribe(); assertTrue(latch.await(200, TimeUnit.MILLISECONDS)); assertEquals("Number of execution semaphores in use", 0, cmd.getExecutionSemaphore().getNumberOfPermitsUsed()); assertEquals("Number of fallback semaphores in use", 0, cmd.getFallbackSemaphore().getNumberOfPermitsUsed()); - assertTrue(cmd.isExecutionComplete()); - assertTrue(cmd.isExecutedInThread()); - System.out.println("EventCounts : " + cmd.getEventCounts()); - System.out.println("Execution Time : " + cmd.getExecutionTimeInMilliseconds()); - System.out.println("Is Successful : " + cmd.isSuccessfulExecution()); + assertFalse(cmd.isExecutionComplete()); + assertEquals(hystrixThreadStartedExecuting.get(), cmd.isExecutedInThread()); + assertEquals(null, cmd.getFailedExecutionException()); + assertNull(cmd.getExecutionException()); + assertTrue(cmd.getExecutionTimeInMilliseconds() > -1); + assertFalse(cmd.isSuccessfulExecution()); + assertCommandExecutionEvents(cmd, HystrixEventType.CANCELLED); + assertEquals(0, cmd.metrics.getCurrentConcurrentExecutionCount()); + System.out.println("ReqLog : " + HystrixRequestLog.getCurrentRequest().getExecutedCommandsAsString()); + assertSaneHystrixRequestLog(1); } catch (InterruptedException ex) { ex.printStackTrace(); } @@ -3051,6 +3139,8 @@ public void onNext(Boolean b) { @Test public void testEarlyUnsubscribeDuringFallback() { + final AtomicBoolean hystrixThreadStartedExecuting = new AtomicBoolean(false); + class AsyncCommand extends HystrixCommand { public AsyncCommand() { @@ -3065,6 +3155,7 @@ protected Boolean run() { @Override protected Boolean getFallback() { try { + hystrixThreadStartedExecuting.set(true); Thread.sleep(100); return false; } catch (InterruptedException ex) { @@ -3110,176 +3201,1451 @@ public void onNext(Boolean b) { assertTrue(latch.await(200, TimeUnit.MILLISECONDS)); assertEquals("Number of execution semaphores in use", 0, cmd.getExecutionSemaphore().getNumberOfPermitsUsed()); assertEquals("Number of fallback semaphores in use", 0, cmd.getFallbackSemaphore().getNumberOfPermitsUsed()); - assertTrue(cmd.isExecutionComplete()); - assertTrue(cmd.isExecutedInThread()); + assertEquals(0, cmd.metrics.getCurrentConcurrentExecutionCount()); + assertFalse(cmd.isExecutionComplete()); + assertEquals(hystrixThreadStartedExecuting.get(), cmd.isExecutedInThread()); } catch (InterruptedException ex) { ex.printStackTrace(); } } - /* ******************************************************************************** */ - /* ******************************************************************************** */ - /* private HystrixCommand class implementations for unit testing */ - /* ******************************************************************************** */ - /* ******************************************************************************** */ + @Test + public void testRequestThenCacheHitAndCacheHitUnsubscribed() { + AsyncCacheableCommand original = new AsyncCacheableCommand("foo"); + AsyncCacheableCommand fromCache = new AsyncCacheableCommand("foo"); - static AtomicInteger uniqueNameCounter = new AtomicInteger(1); + final AtomicReference originalValue = new AtomicReference(null); + final AtomicReference fromCacheValue = new AtomicReference(null); - @Override - TestHystrixCommand getCommand(ExecutionIsolationStrategy isolationStrategy, AbstractTestHystrixCommand.ExecutionResult executionResult, int executionLatency, AbstractTestHystrixCommand.FallbackResult fallbackResult, int fallbackLatency, TestCircuitBreaker circuitBreaker, HystrixThreadPool threadPool, int timeout, AbstractTestHystrixCommand.CacheEnabled cacheEnabled, Object value, TryableSemaphore executionSemaphore, TryableSemaphore fallbackSemaphore, boolean circuitBreakerDisabled) { - HystrixCommandKey commandKey = HystrixCommandKey.Factory.asKey("Flexible-" + uniqueNameCounter.getAndIncrement()); - return FlexibleTestHystrixCommand.from(commandKey, isolationStrategy, executionResult, executionLatency, fallbackResult, fallbackLatency, circuitBreaker, threadPool, timeout, cacheEnabled, value, executionSemaphore, fallbackSemaphore, circuitBreakerDisabled); - } + final CountDownLatch originalLatch = new CountDownLatch(1); + final CountDownLatch fromCacheLatch = new CountDownLatch(1); - @Override - TestHystrixCommand getCommand(HystrixCommandKey commandKey, ExecutionIsolationStrategy isolationStrategy, AbstractTestHystrixCommand.ExecutionResult executionResult, int executionLatency, AbstractTestHystrixCommand.FallbackResult fallbackResult, int fallbackLatency, TestCircuitBreaker circuitBreaker, HystrixThreadPool threadPool, int timeout, AbstractTestHystrixCommand.CacheEnabled cacheEnabled, Object value, TryableSemaphore executionSemaphore, TryableSemaphore fallbackSemaphore, boolean circuitBreakerDisabled) { - return FlexibleTestHystrixCommand.from(commandKey, isolationStrategy, executionResult, executionLatency, fallbackResult, fallbackLatency, circuitBreaker, threadPool, timeout, cacheEnabled, value, executionSemaphore, fallbackSemaphore, circuitBreakerDisabled); - } + Observable originalObservable = original.toObservable(); + Observable fromCacheObservable = fromCache.toObservable(); - private static class FlexibleTestHystrixCommand { + Subscription originalSubscription = originalObservable.doOnUnsubscribe(new Action0() { + @Override + public void call() { + System.out.println(System.currentTimeMillis() + " : " + Thread.currentThread().getName() + " Test.Original Unsubscribe"); + originalLatch.countDown(); + } + }).subscribe(new Subscriber() { + @Override + public void onCompleted() { + System.out.println(System.currentTimeMillis() + " : " + Thread.currentThread().getName() + " Test.Original OnCompleted"); + originalLatch.countDown(); + } - public static Integer EXECUTE_VALUE = 1; - public static Integer FALLBACK_VALUE = 11; + @Override + public void onError(Throwable e) { + System.out.println(System.currentTimeMillis() + " : " + Thread.currentThread().getName() + " Test.Original OnError : " + e); + originalLatch.countDown(); + } - public static AbstractFlexibleTestHystrixCommand from(HystrixCommandKey commandKey, ExecutionIsolationStrategy isolationStrategy, AbstractTestHystrixCommand.ExecutionResult executionResult, int executionLatency, AbstractTestHystrixCommand.FallbackResult fallbackResult, int fallbackLatency, TestCircuitBreaker circuitBreaker, HystrixThreadPool threadPool, int timeout, AbstractTestHystrixCommand.CacheEnabled cacheEnabled, Object value, TryableSemaphore executionSemaphore, TryableSemaphore fallbackSemaphore, boolean circuitBreakerDisabled) { - if (fallbackResult.equals(AbstractTestHystrixCommand.FallbackResult.UNIMPLEMENTED)) { - return new FlexibleTestHystrixCommandNoFallback(commandKey, isolationStrategy, executionResult, executionLatency, circuitBreaker, threadPool, timeout, cacheEnabled, value, executionSemaphore, fallbackSemaphore, circuitBreakerDisabled); - } else { - return new FlexibleTestHystrixCommandWithFallback(commandKey, isolationStrategy, executionResult, executionLatency, fallbackResult, fallbackLatency, circuitBreaker, threadPool, timeout, cacheEnabled, value, executionSemaphore, fallbackSemaphore, circuitBreakerDisabled); + @Override + public void onNext(Boolean b) { + System.out.println(System.currentTimeMillis() + " : " + Thread.currentThread().getName() + " Test.Original OnNext : " + b); + originalValue.set(b); + } + }); + + Subscription fromCacheSubscription = fromCacheObservable.doOnUnsubscribe(new Action0() { + @Override + public void call() { + System.out.println(System.currentTimeMillis() + " : " + Thread.currentThread().getName() + " FromCache Unsubscribe"); + fromCacheLatch.countDown(); + } + }).subscribe(new Subscriber() { + @Override + public void onCompleted() { + System.out.println(System.currentTimeMillis() + " : " + Thread.currentThread().getName() + " FromCache OnCompleted"); + fromCacheLatch.countDown(); + } + + @Override + public void onError(Throwable e) { + System.out.println(System.currentTimeMillis() + " : " + Thread.currentThread().getName() + " FromCache OnError : " + e); + fromCacheLatch.countDown(); + } + + @Override + public void onNext(Boolean b) { + System.out.println(System.currentTimeMillis() + " : " + Thread.currentThread().getName() + " FromCache OnNext : " + b); + fromCacheValue.set(b); } + }); + + try { + fromCacheSubscription.unsubscribe(); + assertTrue(fromCacheLatch.await(200, TimeUnit.MILLISECONDS)); + assertTrue(originalLatch.await(200, TimeUnit.MILLISECONDS)); + assertEquals("Number of execution semaphores in use (original)", 0, original.getExecutionSemaphore().getNumberOfPermitsUsed()); + assertEquals("Number of fallback semaphores in use (original)", 0, original.getFallbackSemaphore().getNumberOfPermitsUsed()); + assertTrue(original.isExecutionComplete()); + assertTrue(original.isExecutedInThread()); + assertEquals(null, original.getFailedExecutionException()); + assertNull(original.getExecutionException()); + assertTrue(original.getExecutionTimeInMilliseconds() > -1); + assertTrue(original.isSuccessfulExecution()); + assertCommandExecutionEvents(original, HystrixEventType.SUCCESS); + assertTrue(originalValue.get()); + assertEquals(0, original.metrics.getCurrentConcurrentExecutionCount()); + + + assertEquals("Number of execution semaphores in use (fromCache)", 0, fromCache.getExecutionSemaphore().getNumberOfPermitsUsed()); + assertEquals("Number of fallback semaphores in use (fromCache)", 0, fromCache.getFallbackSemaphore().getNumberOfPermitsUsed()); + assertFalse(fromCache.isExecutionComplete()); + assertFalse(fromCache.isExecutedInThread()); + assertEquals(null, fromCache.getFailedExecutionException()); + assertNull(fromCache.getExecutionException()); + assertCommandExecutionEvents(fromCache, HystrixEventType.RESPONSE_FROM_CACHE, HystrixEventType.CANCELLED); + assertTrue(fromCache.getExecutionTimeInMilliseconds() == -1); + assertFalse(fromCache.isSuccessfulExecution()); + assertEquals(0, fromCache.metrics.getCurrentConcurrentExecutionCount()); + + assertFalse(original.isCancelled()); //underlying work + System.out.println("ReqLog : " + HystrixRequestLog.getCurrentRequest().getExecutedCommandsAsString()); + assertSaneHystrixRequestLog(2); + } catch (InterruptedException ex) { + ex.printStackTrace(); } } - private static class AbstractFlexibleTestHystrixCommand extends TestHystrixCommand { - protected final AbstractTestHystrixCommand.ExecutionResult executionResult; - protected final int executionLatency; - - protected final CacheEnabled cacheEnabled; - protected final Object value; + @Test + public void testRequestThenCacheHitAndOriginalUnsubscribed() { + AsyncCacheableCommand original = new AsyncCacheableCommand("foo"); + AsyncCacheableCommand fromCache = new AsyncCacheableCommand("foo"); + final AtomicReference originalValue = new AtomicReference(null); + final AtomicReference fromCacheValue = new AtomicReference(null); - AbstractFlexibleTestHystrixCommand(HystrixCommandKey commandKey, ExecutionIsolationStrategy isolationStrategy, AbstractTestHystrixCommand.ExecutionResult executionResult, int executionLatency, TestCircuitBreaker circuitBreaker, HystrixThreadPool threadPool, int timeout, CacheEnabled cacheEnabled, Object value, TryableSemaphore executionSemaphore, TryableSemaphore fallbackSemaphore, boolean circuitBreakerDisabled) { - super(testPropsBuilder(circuitBreaker) - .setCommandKey(commandKey) - .setCircuitBreaker(circuitBreaker) - .setMetrics(circuitBreaker.metrics) - .setThreadPool(threadPool) - .setCommandPropertiesDefaults(HystrixCommandPropertiesTest.getUnitTestPropertiesSetter() - .withExecutionIsolationStrategy(isolationStrategy) - .withExecutionTimeoutInMilliseconds(timeout) - .withCircuitBreakerEnabled(!circuitBreakerDisabled)) - .setExecutionSemaphore(executionSemaphore) - .setFallbackSemaphore(fallbackSemaphore)); - this.executionResult = executionResult; - this.executionLatency = executionLatency; + final CountDownLatch originalLatch = new CountDownLatch(1); + final CountDownLatch fromCacheLatch = new CountDownLatch(1); - this.cacheEnabled = cacheEnabled; - this.value = value; - } + Observable originalObservable = original.toObservable(); + Observable fromCacheObservable = fromCache.toObservable(); - @Override - protected Integer run() throws Exception { - System.out.println(System.currentTimeMillis() + " : " + Thread.currentThread().getName() + " starting the run() method"); - addLatency(executionLatency); - if (executionResult == AbstractTestHystrixCommand.ExecutionResult.SUCCESS) { - return FlexibleTestHystrixCommand.EXECUTE_VALUE; - } else if (executionResult == AbstractTestHystrixCommand.ExecutionResult.FAILURE) { - throw new RuntimeException("Execution Failure for TestHystrixCommand"); - } else if (executionResult == AbstractTestHystrixCommand.ExecutionResult.HYSTRIX_FAILURE) { - throw new HystrixRuntimeException(HystrixRuntimeException.FailureType.COMMAND_EXCEPTION, AbstractFlexibleTestHystrixCommand.class, "Execution Hystrix Failure for TestHystrixCommand", new RuntimeException("Execution Failure for TestHystrixCommand"), new RuntimeException("Fallback Failure for TestHystrixCommand")); - } else if (executionResult == AbstractTestHystrixCommand.ExecutionResult.RECOVERABLE_ERROR) { - throw new java.lang.Error("Execution ERROR for TestHystrixCommand"); - } else if (executionResult == AbstractTestHystrixCommand.ExecutionResult.UNRECOVERABLE_ERROR) { - throw new StackOverflowError("Unrecoverable Error for TestHystrixCommand"); - } else if (executionResult == AbstractTestHystrixCommand.ExecutionResult.BAD_REQUEST) { - throw new HystrixBadRequestException("Execution BadRequestException for TestHystrixCommand"); - } else { - throw new RuntimeException("You passed in a executionResult enum that can't be represented in HystrixCommand: " + executionResult); + Subscription originalSubscription = originalObservable.doOnUnsubscribe(new Action0() { + @Override + public void call() { + System.out.println(System.currentTimeMillis() + " : " + Thread.currentThread().getName() + " Test.Original Unsubscribe"); + originalLatch.countDown(); + } + }).subscribe(new Subscriber() { + @Override + public void onCompleted() { + System.out.println(System.currentTimeMillis() + " : " + Thread.currentThread().getName() + " Test.Original OnCompleted"); + originalLatch.countDown(); } - } - - @Override - public String getCacheKey() { - if (cacheEnabled == CacheEnabled.YES) - return value.toString(); - else - return null; - } - protected void addLatency(int latency) { - if (latency > 0) { - try { - System.out.println(System.currentTimeMillis() + " : " + Thread.currentThread().getName() + " About to sleep for : " + latency); - Thread.sleep(latency); - System.out.println(System.currentTimeMillis() + " : " + Thread.currentThread().getName() + " Woke up from sleep!"); - } catch (InterruptedException e) { - e.printStackTrace(); - // ignore and sleep some more to simulate a dependency that doesn't obey interrupts - try { - Thread.sleep(latency); - } catch (Exception e2) { - // ignore - } - System.out.println("after interruption with extra sleep"); - } + @Override + public void onError(Throwable e) { + System.out.println(System.currentTimeMillis() + " : " + Thread.currentThread().getName() + " Test.Original OnError : " + e); + originalLatch.countDown(); } - } - } + @Override + public void onNext(Boolean b) { + System.out.println(System.currentTimeMillis() + " : " + Thread.currentThread().getName() + " Test.Original OnNext : " + b); + originalValue.set(b); + } + }); - private static class FlexibleTestHystrixCommandWithFallback extends AbstractFlexibleTestHystrixCommand { - protected final AbstractTestHystrixCommand.FallbackResult fallbackResult; - protected final int fallbackLatency; + Subscription fromCacheSubscription = fromCacheObservable.doOnUnsubscribe(new Action0() { + @Override + public void call() { + System.out.println(System.currentTimeMillis() + " : " + Thread.currentThread().getName() + " Test.FromCache Unsubscribe"); + fromCacheLatch.countDown(); + } + }).subscribe(new Subscriber() { + @Override + public void onCompleted() { + System.out.println(System.currentTimeMillis() + " : " + Thread.currentThread().getName() + " Test.FromCache OnCompleted"); + fromCacheLatch.countDown(); + } - FlexibleTestHystrixCommandWithFallback(HystrixCommandKey commandKey, ExecutionIsolationStrategy isolationStrategy, AbstractTestHystrixCommand.ExecutionResult executionResult, int executionLatency, FallbackResult fallbackResult, int fallbackLatency, TestCircuitBreaker circuitBreaker, HystrixThreadPool threadPool, int timeout, CacheEnabled cacheEnabled, Object value, TryableSemaphore executionSemaphore, TryableSemaphore fallbackSemaphore, boolean circuitBreakerDisabled) { - super(commandKey, isolationStrategy, executionResult, executionLatency, circuitBreaker, threadPool, timeout, cacheEnabled, value, executionSemaphore, fallbackSemaphore, circuitBreakerDisabled); - this.fallbackResult = fallbackResult; - this.fallbackLatency = fallbackLatency; - } + @Override + public void onError(Throwable e) { + System.out.println(System.currentTimeMillis() + " : " + Thread.currentThread().getName() + " Test.FromCache OnError : " + e); + fromCacheLatch.countDown(); + } - @Override - protected Integer getFallback() { - addLatency(fallbackLatency); - if (fallbackResult == AbstractTestHystrixCommand.FallbackResult.SUCCESS) { - return FlexibleTestHystrixCommand.FALLBACK_VALUE; - } else if (fallbackResult == AbstractTestHystrixCommand.FallbackResult.FAILURE) { - throw new RuntimeException("Fallback Failure for TestHystrixCommand"); - } else if (fallbackResult == FallbackResult.UNIMPLEMENTED) { - return super.getFallback(); - } else { - throw new RuntimeException("You passed in a fallbackResult enum that can't be represented in HystrixCommand: " + fallbackResult); + @Override + public void onNext(Boolean b) { + System.out.println(System.currentTimeMillis() + " : " + Thread.currentThread().getName() + " Test.FromCache OnNext : " + b); + fromCacheValue.set(b); } - } - } + }); - private static class FlexibleTestHystrixCommandNoFallback extends AbstractFlexibleTestHystrixCommand { - FlexibleTestHystrixCommandNoFallback(HystrixCommandKey commandKey, ExecutionIsolationStrategy isolationStrategy, AbstractTestHystrixCommand.ExecutionResult executionResult, int executionLatency, TestCircuitBreaker circuitBreaker, HystrixThreadPool threadPool, int timeout, CacheEnabled cacheEnabled, Object value, TryableSemaphore executionSemaphore, TryableSemaphore fallbackSemaphore, boolean circuitBreakerDisabled) { - super(commandKey, isolationStrategy, executionResult, executionLatency, circuitBreaker, threadPool, timeout, cacheEnabled, value, executionSemaphore, fallbackSemaphore, circuitBreakerDisabled); + try { + Thread.sleep(10); + originalSubscription.unsubscribe(); + assertTrue(originalLatch.await(200, TimeUnit.MILLISECONDS)); + assertTrue(fromCacheLatch.await(200, TimeUnit.MILLISECONDS)); + assertEquals("Number of execution semaphores in use (original)", 0, original.getExecutionSemaphore().getNumberOfPermitsUsed()); + assertEquals("Number of fallback semaphores in use (original)", 0, original.getFallbackSemaphore().getNumberOfPermitsUsed()); + assertFalse(original.isExecutionComplete()); + assertTrue(original.isExecutedInThread()); + assertEquals(null, original.getFailedExecutionException()); + assertNull(original.getExecutionException()); + assertTrue(original.getExecutionTimeInMilliseconds() > -1); + assertFalse(original.isSuccessfulExecution()); + assertCommandExecutionEvents(original, HystrixEventType.CANCELLED); + assertNull(originalValue.get()); + assertEquals(0, original.metrics.getCurrentConcurrentExecutionCount()); + + assertEquals("Number of execution semaphores in use (fromCache)", 0, fromCache.getExecutionSemaphore().getNumberOfPermitsUsed()); + assertEquals("Number of fallback semaphores in use (fromCache)", 0, fromCache.getFallbackSemaphore().getNumberOfPermitsUsed()); + assertTrue(fromCache.isExecutionComplete()); + assertFalse(fromCache.isExecutedInThread()); + assertEquals(null, fromCache.getFailedExecutionException()); + assertNull(fromCache.getExecutionException()); + assertCommandExecutionEvents(fromCache, HystrixEventType.SUCCESS, HystrixEventType.RESPONSE_FROM_CACHE); + assertTrue(fromCache.getExecutionTimeInMilliseconds() == -1); + assertTrue(fromCache.isSuccessfulExecution()); + assertEquals(0, fromCache.metrics.getCurrentConcurrentExecutionCount()); + + assertFalse(original.isCancelled()); //underlying work + System.out.println("ReqLog : " + HystrixRequestLog.getCurrentRequest().getExecutedCommandsAsString()); + assertSaneHystrixRequestLog(2); + } catch (InterruptedException ex) { + ex.printStackTrace(); } } - /** - * Successful execution - no fallback implementation. - */ - private static class SuccessfulTestCommand extends TestHystrixCommand { - - public SuccessfulTestCommand() { - this(HystrixCommandPropertiesTest.getUnitTestPropertiesSetter()); - } + @Test + public void testRequestThenTwoCacheHitsOriginalAndOneCacheHitUnsubscribed() { + AsyncCacheableCommand original = new AsyncCacheableCommand("foo"); + AsyncCacheableCommand fromCache1 = new AsyncCacheableCommand("foo"); + AsyncCacheableCommand fromCache2 = new AsyncCacheableCommand("foo"); - public SuccessfulTestCommand(HystrixCommandProperties.Setter properties) { - super(testPropsBuilder().setCommandPropertiesDefaults(properties)); - } + final AtomicReference originalValue = new AtomicReference(null); + final AtomicReference fromCache1Value = new AtomicReference(null); + final AtomicReference fromCache2Value = new AtomicReference(null); - @Override - protected Boolean run() { - return true; - } + final CountDownLatch originalLatch = new CountDownLatch(1); + final CountDownLatch fromCache1Latch = new CountDownLatch(1); + final CountDownLatch fromCache2Latch = new CountDownLatch(1); - } + Observable originalObservable = original.toObservable(); + Observable fromCache1Observable = fromCache1.toObservable(); + Observable fromCache2Observable = fromCache2.toObservable(); - /** - * Successful execution - no fallback implementation. + Subscription originalSubscription = originalObservable.doOnUnsubscribe(new Action0() { + @Override + public void call() { + System.out.println(System.currentTimeMillis() + " : " + Thread.currentThread().getName() + " Test.Original Unsubscribe"); + originalLatch.countDown(); + } + }).subscribe(new Subscriber() { + @Override + public void onCompleted() { + System.out.println(System.currentTimeMillis() + " : " + Thread.currentThread().getName() + " Test.Original OnCompleted"); + originalLatch.countDown(); + } + + @Override + public void onError(Throwable e) { + System.out.println(System.currentTimeMillis() + " : " + Thread.currentThread().getName() + " Test.Original OnError : " + e); + originalLatch.countDown(); + } + + @Override + public void onNext(Boolean b) { + System.out.println(System.currentTimeMillis() + " : " + Thread.currentThread().getName() + " Test.Original OnNext : " + b); + originalValue.set(b); + } + }); + + Subscription fromCache1Subscription = fromCache1Observable.doOnUnsubscribe(new Action0() { + @Override + public void call() { + System.out.println(System.currentTimeMillis() + " : " + Thread.currentThread().getName() + " Test.FromCache1 Unsubscribe"); + fromCache1Latch.countDown(); + } + }).subscribe(new Subscriber() { + @Override + public void onCompleted() { + System.out.println(System.currentTimeMillis() + " : " + Thread.currentThread().getName() + " Test.FromCache1 OnCompleted"); + fromCache1Latch.countDown(); + } + + @Override + public void onError(Throwable e) { + System.out.println(System.currentTimeMillis() + " : " + Thread.currentThread().getName() + " Test.FromCache1 OnError : " + e); + fromCache1Latch.countDown(); + } + + @Override + public void onNext(Boolean b) { + System.out.println(System.currentTimeMillis() + " : " + Thread.currentThread().getName() + " Test.FromCache1 OnNext : " + b); + fromCache1Value.set(b); + } + }); + + Subscription fromCache2Subscription = fromCache2Observable.doOnUnsubscribe(new Action0() { + @Override + public void call() { + System.out.println(System.currentTimeMillis() + " : " + Thread.currentThread().getName() + " Test.FromCache2 Unsubscribe"); + fromCache2Latch.countDown(); + } + }).subscribe(new Subscriber() { + @Override + public void onCompleted() { + System.out.println(System.currentTimeMillis() + " : " + Thread.currentThread().getName() + " Test.FromCache2 OnCompleted"); + fromCache2Latch.countDown(); + } + + @Override + public void onError(Throwable e) { + System.out.println(System.currentTimeMillis() + " : " + Thread.currentThread().getName() + " Test.FromCache2 OnError : " + e); + fromCache2Latch.countDown(); + } + + @Override + public void onNext(Boolean b) { + System.out.println(System.currentTimeMillis() + " : " + Thread.currentThread().getName() + " Test.FromCache2 OnNext : " + b); + fromCache2Value.set(b); + } + }); + + try { + Thread.sleep(10); + originalSubscription.unsubscribe(); + //fromCache1Subscription.unsubscribe(); + fromCache2Subscription.unsubscribe(); + assertTrue(originalLatch.await(200, TimeUnit.MILLISECONDS)); + assertTrue(fromCache1Latch.await(200, TimeUnit.MILLISECONDS)); + assertTrue(fromCache2Latch.await(200, TimeUnit.MILLISECONDS)); + assertEquals("Number of execution semaphores in use (original)", 0, original.getExecutionSemaphore().getNumberOfPermitsUsed()); + assertEquals("Number of fallback semaphores in use (original)", 0, original.getFallbackSemaphore().getNumberOfPermitsUsed()); + assertFalse(original.isExecutionComplete()); + assertTrue(original.isExecutedInThread()); + assertEquals(null, original.getFailedExecutionException()); + assertNull(original.getExecutionException()); + assertTrue(original.getExecutionTimeInMilliseconds() > -1); + assertFalse(original.isSuccessfulExecution()); + assertCommandExecutionEvents(original, HystrixEventType.CANCELLED); + assertNull(originalValue.get()); + assertFalse(original.isCancelled()); //underlying work + assertEquals(0, original.metrics.getCurrentConcurrentExecutionCount()); + + + assertEquals("Number of execution semaphores in use (fromCache1)", 0, fromCache1.getExecutionSemaphore().getNumberOfPermitsUsed()); + assertEquals("Number of fallback semaphores in use (fromCache1)", 0, fromCache1.getFallbackSemaphore().getNumberOfPermitsUsed()); + assertTrue(fromCache1.isExecutionComplete()); + assertFalse(fromCache1.isExecutedInThread()); + assertEquals(null, fromCache1.getFailedExecutionException()); + assertNull(fromCache1.getExecutionException()); + assertCommandExecutionEvents(fromCache1, HystrixEventType.SUCCESS, HystrixEventType.RESPONSE_FROM_CACHE); + assertTrue(fromCache1.getExecutionTimeInMilliseconds() == -1); + assertTrue(fromCache1.isSuccessfulExecution()); + assertTrue(fromCache1Value.get()); + assertEquals(0, fromCache1.metrics.getCurrentConcurrentExecutionCount()); + + + assertEquals("Number of execution semaphores in use (fromCache2)", 0, fromCache2.getExecutionSemaphore().getNumberOfPermitsUsed()); + assertEquals("Number of fallback semaphores in use (fromCache2)", 0, fromCache2.getFallbackSemaphore().getNumberOfPermitsUsed()); + assertFalse(fromCache2.isExecutionComplete()); + assertFalse(fromCache2.isExecutedInThread()); + assertEquals(null, fromCache2.getFailedExecutionException()); + assertNull(fromCache2.getExecutionException()); + assertCommandExecutionEvents(fromCache2, HystrixEventType.RESPONSE_FROM_CACHE, HystrixEventType.CANCELLED); + assertTrue(fromCache2.getExecutionTimeInMilliseconds() == -1); + assertFalse(fromCache2.isSuccessfulExecution()); + assertNull(fromCache2Value.get()); + assertEquals(0, fromCache2.metrics.getCurrentConcurrentExecutionCount()); + + System.out.println("ReqLog : " + HystrixRequestLog.getCurrentRequest().getExecutedCommandsAsString()); + assertSaneHystrixRequestLog(3); + } catch (InterruptedException ex) { + ex.printStackTrace(); + } + } + + @Test + public void testRequestThenTwoCacheHitsAllUnsubscribed() { + AsyncCacheableCommand original = new AsyncCacheableCommand("foo"); + AsyncCacheableCommand fromCache1 = new AsyncCacheableCommand("foo"); + AsyncCacheableCommand fromCache2 = new AsyncCacheableCommand("foo"); + + final CountDownLatch originalLatch = new CountDownLatch(1); + final CountDownLatch fromCache1Latch = new CountDownLatch(1); + final CountDownLatch fromCache2Latch = new CountDownLatch(1); + + Observable originalObservable = original.toObservable(); + Observable fromCache1Observable = fromCache1.toObservable(); + Observable fromCache2Observable = fromCache2.toObservable(); + + Subscription originalSubscription = originalObservable.doOnUnsubscribe(new Action0() { + @Override + public void call() { + System.out.println(System.currentTimeMillis() + " : " + Thread.currentThread().getName() + " Test.Original Unsubscribe"); + originalLatch.countDown(); + } + }).subscribe(new Subscriber() { + @Override + public void onCompleted() { + System.out.println(System.currentTimeMillis() + " : " + Thread.currentThread().getName() + " Test.Original OnCompleted"); + originalLatch.countDown(); + } + + @Override + public void onError(Throwable e) { + System.out.println(System.currentTimeMillis() + " : " + Thread.currentThread().getName() + " Test.Original OnError : " + e); + originalLatch.countDown(); + } + + @Override + public void onNext(Boolean b) { + System.out.println(System.currentTimeMillis() + " : " + Thread.currentThread().getName() + " Test.Original OnNext : " + b); + } + }); + + Subscription fromCache1Subscription = fromCache1Observable.doOnUnsubscribe(new Action0() { + @Override + public void call() { + System.out.println(System.currentTimeMillis() + " : " + Thread.currentThread().getName() + " Test.FromCache1 Unsubscribe"); + fromCache1Latch.countDown(); + } + }).subscribe(new Subscriber() { + @Override + public void onCompleted() { + System.out.println(System.currentTimeMillis() + " : " + Thread.currentThread().getName() + " Test.FromCache1 OnCompleted"); + fromCache1Latch.countDown(); + } + + @Override + public void onError(Throwable e) { + System.out.println(System.currentTimeMillis() + " : " + Thread.currentThread().getName() + " Test.FromCache1 OnError : " + e); + fromCache1Latch.countDown(); + } + + @Override + public void onNext(Boolean b) { + System.out.println(System.currentTimeMillis() + " : " + Thread.currentThread().getName() + " Test.FromCache1 OnNext : " + b); + } + }); + + Subscription fromCache2Subscription = fromCache2Observable.doOnUnsubscribe(new Action0() { + @Override + public void call() { + System.out.println(System.currentTimeMillis() + " : " + Thread.currentThread().getName() + " Test.FromCache2 Unsubscribe"); + fromCache2Latch.countDown(); + } + }).subscribe(new Subscriber() { + @Override + public void onCompleted() { + System.out.println(System.currentTimeMillis() + " : " + Thread.currentThread().getName() + " Test.FromCache2 OnCompleted"); + fromCache2Latch.countDown(); + } + + @Override + public void onError(Throwable e) { + System.out.println(System.currentTimeMillis() + " : " + Thread.currentThread().getName() + " Test.FromCache2 OnError : " + e); + fromCache2Latch.countDown(); + } + + @Override + public void onNext(Boolean b) { + System.out.println(System.currentTimeMillis() + " : " + Thread.currentThread().getName() + " Test.FromCache2 OnNext : " + b); + } + }); + + try { + Thread.sleep(10); + originalSubscription.unsubscribe(); + fromCache1Subscription.unsubscribe(); + fromCache2Subscription.unsubscribe(); + assertTrue(originalLatch.await(200, TimeUnit.MILLISECONDS)); + assertTrue(fromCache1Latch.await(200, TimeUnit.MILLISECONDS)); + assertTrue(fromCache2Latch.await(200, TimeUnit.MILLISECONDS)); + assertEquals("Number of execution semaphores in use (original)", 0, original.getExecutionSemaphore().getNumberOfPermitsUsed()); + assertEquals("Number of fallback semaphores in use (original)", 0, original.getFallbackSemaphore().getNumberOfPermitsUsed()); + assertFalse(original.isExecutionComplete()); + assertTrue(original.isExecutedInThread()); + System.out.println("FEE : " + original.getFailedExecutionException()); + if (original.getFailedExecutionException() != null) { + original.getFailedExecutionException().printStackTrace(); + } + assertNull(original.getFailedExecutionException()); + assertNull(original.getExecutionException()); + assertTrue(original.getExecutionTimeInMilliseconds() > -1); + assertFalse(original.isSuccessfulExecution()); + assertCommandExecutionEvents(original, HystrixEventType.CANCELLED); + //assertTrue(original.isCancelled()); //underlying work This doesn't work yet + assertEquals(0, original.metrics.getCurrentConcurrentExecutionCount()); + + + assertEquals("Number of execution semaphores in use (fromCache1)", 0, fromCache1.getExecutionSemaphore().getNumberOfPermitsUsed()); + assertEquals("Number of fallback semaphores in use (fromCache1)", 0, fromCache1.getFallbackSemaphore().getNumberOfPermitsUsed()); + assertFalse(fromCache1.isExecutionComplete()); + assertFalse(fromCache1.isExecutedInThread()); + assertEquals(null, fromCache1.getFailedExecutionException()); + assertNull(fromCache1.getExecutionException()); + assertCommandExecutionEvents(fromCache1, HystrixEventType.RESPONSE_FROM_CACHE, HystrixEventType.CANCELLED); + assertTrue(fromCache1.getExecutionTimeInMilliseconds() == -1); + assertFalse(fromCache1.isSuccessfulExecution()); + assertEquals(0, fromCache1.metrics.getCurrentConcurrentExecutionCount()); + + + assertEquals("Number of execution semaphores in use (fromCache2)", 0, fromCache2.getExecutionSemaphore().getNumberOfPermitsUsed()); + assertEquals("Number of fallback semaphores in use (fromCache2)", 0, fromCache2.getFallbackSemaphore().getNumberOfPermitsUsed()); + assertFalse(fromCache2.isExecutionComplete()); + assertFalse(fromCache2.isExecutedInThread()); + assertEquals(null, fromCache2.getFailedExecutionException()); + assertNull(fromCache2.getExecutionException()); + assertCommandExecutionEvents(fromCache2, HystrixEventType.RESPONSE_FROM_CACHE, HystrixEventType.CANCELLED); + assertTrue(fromCache2.getExecutionTimeInMilliseconds() == -1); + assertFalse(fromCache2.isSuccessfulExecution()); + assertEquals(0, fromCache2.metrics.getCurrentConcurrentExecutionCount()); + + System.out.println("ReqLog : " + HystrixRequestLog.getCurrentRequest().getExecutedCommandsAsString()); + assertSaneHystrixRequestLog(3); + } catch (InterruptedException ex) { + ex.printStackTrace(); + } + } + + /** + *********************** THREAD-ISOLATED Execution Hook Tests ************************************** + */ + + /** + * Short-circuit? : NO + * Thread/semaphore: THREAD + * Thread Pool full? : NO + * Thread Pool Queue full?: NO + * Timeout: NO + * Execution Result: SUCCESS + */ + @Test + public void testExecutionHookThreadSuccess() { + assertHooksOnSuccess( + new Func0>() { + @Override + public TestHystrixCommand call() { + return getCommand(ExecutionIsolationStrategy.THREAD, AbstractTestHystrixCommand.ExecutionResult.SUCCESS); + } + }, + new Action1>() { + @Override + public void call(TestHystrixCommand command) { + TestableExecutionHook hook = command.getBuilder().executionHook; + assertTrue(hook.commandEmissionsMatch(1, 0, 1)); + assertTrue(hook.executionEventsMatch(1, 0, 1)); + assertTrue(hook.fallbackEventsMatch(0, 0, 0)); + assertEquals("onStart - onThreadStart - !onRunStart - onExecutionStart - onExecutionEmit - !onRunSuccess - !onComplete - onEmit - onExecutionSuccess - onThreadComplete - onSuccess - ", hook.executionSequence.toString()); + } + }); + } + + /** + * Short-circuit? : NO + * Thread/semaphore: THREAD + * Thread Pool full? : NO + * Thread Pool Queue full?: NO + * Timeout: NO + * Execution Result: synchronous HystrixBadRequestException + */ + @Test + public void testExecutionHookThreadBadRequestException() { + assertHooksOnFailure( + new Func0>() { + @Override + public TestHystrixCommand call() { + return getCommand(ExecutionIsolationStrategy.THREAD, AbstractTestHystrixCommand.ExecutionResult.BAD_REQUEST); + } + }, + new Action1>() { + @Override + public void call(TestHystrixCommand command) { + TestableExecutionHook hook = command.getBuilder().executionHook; + assertTrue(hook.commandEmissionsMatch(0, 1, 0)); + assertTrue(hook.executionEventsMatch(0, 1, 0)); + assertTrue(hook.fallbackEventsMatch(0, 0, 0)); + assertEquals(HystrixBadRequestException.class, hook.getCommandException().getClass()); + assertEquals(HystrixBadRequestException.class, hook.getExecutionException().getClass()); + assertEquals("onStart - onThreadStart - !onRunStart - onExecutionStart - onExecutionError - !onRunError - onError - onThreadComplete - ", hook.executionSequence.toString()); + } + }); + } + + + + /** + * Short-circuit? : NO + * Thread/semaphore: THREAD + * Thread Pool full? : NO + * Thread Pool Queue full?: NO + * Timeout: NO + * Execution Result: synchronous HystrixRuntimeException + * Fallback: UnsupportedOperationException + */ + @Test + public void testExecutionHookThreadExceptionNoFallback() { + assertHooksOnFailure( + new Func0>() { + @Override + public TestHystrixCommand call() { + return getCommand(ExecutionIsolationStrategy.THREAD, AbstractTestHystrixCommand.ExecutionResult.FAILURE, 0, AbstractTestHystrixCommand.FallbackResult.UNIMPLEMENTED); + } + }, + new Action1>() { + @Override + public void call(TestHystrixCommand command) { + TestableExecutionHook hook = command.getBuilder().executionHook; + assertTrue(hook.commandEmissionsMatch(0, 1, 0)); + assertTrue(hook.executionEventsMatch(0, 1, 0)); + assertTrue(hook.fallbackEventsMatch(0, 0, 0)); + assertEquals(RuntimeException.class, hook.getCommandException().getClass()); + assertEquals(RuntimeException.class, hook.getExecutionException().getClass()); + assertNull(hook.getFallbackException()); + assertEquals("onStart - onThreadStart - !onRunStart - onExecutionStart - onExecutionError - !onRunError - onError - onThreadComplete - ", hook.executionSequence.toString()); + } + }); + } + + /** + * Short-circuit? : NO + * Thread/semaphore: THREAD + * Thread Pool full? : NO + * Thread Pool Queue full?: NO + * Timeout: NO + * Execution Result: synchronous HystrixRuntimeException + * Fallback: SUCCESS + */ + @Test + public void testExecutionHookThreadExceptionSuccessfulFallback() { + assertHooksOnSuccess( + new Func0>() { + @Override + public TestHystrixCommand call() { + return getCommand(ExecutionIsolationStrategy.THREAD, AbstractTestHystrixCommand.ExecutionResult.FAILURE, 0, AbstractTestHystrixCommand.FallbackResult.SUCCESS); + } + }, + new Action1>() { + @Override + public void call(TestHystrixCommand command) { + TestableExecutionHook hook = command.getBuilder().executionHook; + assertTrue(hook.commandEmissionsMatch(1, 0, 1)); + assertTrue(hook.executionEventsMatch(0, 1, 0)); + assertTrue(hook.fallbackEventsMatch(1, 0, 1)); + assertEquals(RuntimeException.class, hook.getExecutionException().getClass()); + assertEquals("onStart - onThreadStart - !onRunStart - onExecutionStart - onExecutionError - !onRunError - onFallbackStart - onFallbackEmit - !onFallbackSuccess - !onComplete - onEmit - onFallbackSuccess - onThreadComplete - onSuccess - ", hook.executionSequence.toString()); + } + }); + } + + /** + * Short-circuit? : NO + * Thread/semaphore: THREAD + * Thread Pool full? : NO + * Thread Pool Queue full?: NO + * Timeout: NO + * Execution Result: synchronous HystrixRuntimeException + * Fallback: HystrixRuntimeException + */ + @Test + public void testExecutionHookThreadExceptionUnsuccessfulFallback() { + assertHooksOnFailure( + new Func0>() { + @Override + public TestHystrixCommand call() { + return getCommand(ExecutionIsolationStrategy.THREAD, AbstractTestHystrixCommand.ExecutionResult.FAILURE, 0, AbstractTestHystrixCommand.FallbackResult.FAILURE); + } + }, + new Action1>() { + @Override + public void call(TestHystrixCommand command) { + TestableExecutionHook hook = command.getBuilder().executionHook; + assertTrue(hook.commandEmissionsMatch(0, 1, 0)); + assertTrue(hook.executionEventsMatch(0, 1, 0)); + assertTrue(hook.fallbackEventsMatch(0, 1, 0)); + assertEquals(RuntimeException.class, hook.getCommandException().getClass()); + assertEquals(RuntimeException.class, hook.getExecutionException().getClass()); + assertEquals(RuntimeException.class, hook.getFallbackException().getClass()); + assertEquals("onStart - onThreadStart - !onRunStart - onExecutionStart - onExecutionError - !onRunError - onFallbackStart - onFallbackError - onError - onThreadComplete - ", hook.executionSequence.toString()); + } + }); + } + + /** + * Short-circuit? : NO + * Thread/semaphore: THREAD + * Thread Pool full? : NO + * Thread Pool Queue full?: NO + * Timeout: YES + * Execution Result: SUCCESS (but timeout prior) + * Fallback: UnsupportedOperationException + */ + @Test + public void testExecutionHookThreadTimeoutNoFallbackRunSuccess() { + assertHooksOnFailure( + new Func0>() { + @Override + public TestHystrixCommand call() { + return getCommand(ExecutionIsolationStrategy.THREAD, AbstractTestHystrixCommand.ExecutionResult.SUCCESS, 150, AbstractTestHystrixCommand.FallbackResult.UNIMPLEMENTED, 50); + } + }, + new Action1>() { + @Override + public void call(TestHystrixCommand command) { + TestableExecutionHook hook = command.getBuilder().executionHook; + assertTrue(hook.commandEmissionsMatch(0, 1, 0)); + assertTrue(hook.executionEventsMatch(0, 0, 0)); + assertTrue(hook.fallbackEventsMatch(0, 0, 0)); + assertEquals(TimeoutException.class, hook.getCommandException().getClass()); + assertNull(hook.getFallbackException()); + assertEquals("onStart - onThreadStart - !onRunStart - onExecutionStart - onError - ", hook.executionSequence.toString()); + + try { + Thread.sleep(300); + } catch (Exception ex) { + throw new RuntimeException(ex); + } + + //should be the same as above, since Hystrix thread is unsubscribed from by timeout + assertTrue(hook.commandEmissionsMatch(0, 1, 0)); + assertTrue(hook.executionEventsMatch(0, 0, 0)); + assertTrue(hook.fallbackEventsMatch(0, 0, 0)); + assertEquals("onStart - onThreadStart - !onRunStart - onExecutionStart - onError - ", hook.executionSequence.toString()); + + } + }); + } + + /** + * Short-circuit? : NO + * Thread/semaphore: THREAD + * Thread Pool full? : NO + * Thread Pool Queue full?: NO + * Timeout: YES + * Execution Result: SUCCESS (but timeout prior) + * Fallback: SUCCESS + */ + @Test + public void testExecutionHookThreadTimeoutSuccessfulFallbackRunSuccess() { + assertHooksOnSuccess( + new Func0>() { + @Override + public TestHystrixCommand call() { + return getCommand(ExecutionIsolationStrategy.THREAD, AbstractTestHystrixCommand.ExecutionResult.SUCCESS, 150, AbstractTestHystrixCommand.FallbackResult.SUCCESS, 50); + } + }, + new Action1>() { + @Override + public void call(TestHystrixCommand command) { + TestableExecutionHook hook = command.getBuilder().executionHook; + assertTrue(hook.commandEmissionsMatch(1, 0, 1)); + assertTrue(hook.executionEventsMatch(0, 0, 0)); + assertTrue(hook.fallbackEventsMatch(1, 0, 1)); + assertEquals("onStart - onThreadStart - !onRunStart - onExecutionStart - onFallbackStart - onFallbackEmit - !onFallbackSuccess - !onComplete - onEmit - onFallbackSuccess - onSuccess - ", hook.executionSequence.toString()); + + try { + Thread.sleep(300); + } catch (Exception ex) { + throw new RuntimeException(ex); + } + + //should be the same as above, since Hystrix thread is unsubscribed from by timeout + assertTrue(hook.commandEmissionsMatch(1, 0, 1)); + assertTrue(hook.executionEventsMatch(0, 0, 0)); + assertTrue(hook.fallbackEventsMatch(1, 0, 1)); + assertEquals("onStart - onThreadStart - !onRunStart - onExecutionStart - onFallbackStart - onFallbackEmit - !onFallbackSuccess - !onComplete - onEmit - onFallbackSuccess - onSuccess - ", hook.executionSequence.toString()); + + } + }); + } + + /** + * Short-circuit? : NO + * Thread/semaphore: THREAD + * Thread Pool full? : NO + * Thread Pool Queue full?: NO + * Timeout: YES + * Execution Result: SUCCESS (but timeout prior) + * Fallback: synchronous HystrixRuntimeException + */ + @Test + public void testExecutionHookThreadTimeoutUnsuccessfulFallbackRunSuccess() { + assertHooksOnFailure( + new Func0>() { + @Override + public TestHystrixCommand call() { + return getCommand(ExecutionIsolationStrategy.THREAD, AbstractTestHystrixCommand.ExecutionResult.SUCCESS, 150, AbstractTestHystrixCommand.FallbackResult.FAILURE, 50); + } + }, + new Action1>() { + @Override + public void call(TestHystrixCommand command) { + TestableExecutionHook hook = command.getBuilder().executionHook; + assertTrue(hook.commandEmissionsMatch(0, 1, 0)); + assertTrue(hook.executionEventsMatch(0, 0, 0)); + assertTrue(hook.fallbackEventsMatch(0, 1, 0)); + assertEquals(TimeoutException.class, hook.getCommandException().getClass()); + assertEquals(RuntimeException.class, hook.getFallbackException().getClass()); + assertEquals("onStart - onThreadStart - !onRunStart - onExecutionStart - onFallbackStart - onFallbackError - onError - ", hook.executionSequence.toString()); + + try { + Thread.sleep(300); + } catch (Exception ex) { + throw new RuntimeException(ex); + } + + //should be the same as above, since Hystrix thread is unsubscribed from by timeout + assertTrue(hook.commandEmissionsMatch(0, 1, 0)); + assertTrue(hook.executionEventsMatch(0, 0, 0)); + assertTrue(hook.fallbackEventsMatch(0, 1, 0)); + assertEquals("onStart - onThreadStart - !onRunStart - onExecutionStart - onFallbackStart - onFallbackError - onError - ", hook.executionSequence.toString()); + + } + }); + } + + /** + * Short-circuit? : NO + * Thread/semaphore: THREAD + * Thread Pool full? : NO + * Thread Pool Queue full?: NO + * Timeout: YES + * Execution Result: HystrixRuntimeException (but timeout prior) + * Fallback: UnsupportedOperationException + */ + @Test + public void testExecutionHookThreadTimeoutNoFallbackRunFailure() { + assertHooksOnFailure( + new Func0>() { + @Override + public TestHystrixCommand call() { + return getCommand(ExecutionIsolationStrategy.THREAD, AbstractTestHystrixCommand.ExecutionResult.FAILURE, 150, AbstractTestHystrixCommand.FallbackResult.UNIMPLEMENTED, 50); + } + }, + new Action1>() { + @Override + public void call(TestHystrixCommand command) { + TestableExecutionHook hook = command.getBuilder().executionHook; + assertTrue(hook.commandEmissionsMatch(0, 1, 0)); + assertTrue(hook.executionEventsMatch(0, 0, 0)); + assertTrue(hook.fallbackEventsMatch(0, 0, 0)); + assertEquals(TimeoutException.class, hook.getCommandException().getClass()); + assertNull(hook.getFallbackException()); + assertEquals("onStart - onThreadStart - !onRunStart - onExecutionStart - onError - ", hook.executionSequence.toString()); + + try { + Thread.sleep(300); + } catch (Exception ex) { + throw new RuntimeException(ex); + } + assertTrue(hook.commandEmissionsMatch(0, 1, 0)); + assertTrue(hook.executionEventsMatch(0, 1, 0)); + assertTrue(hook.fallbackEventsMatch(0, 0, 0)); + assertEquals(RuntimeException.class, hook.getExecutionException().getClass()); + assertEquals("onStart - onThreadStart - !onRunStart - onExecutionStart - onError - onExecutionError - !onRunError - onThreadComplete - ", hook.executionSequence.toString()); + + } + }); + } + + /** + * Short-circuit? : NO + * Thread/semaphore: THREAD + * Thread Pool full? : NO + * Thread Pool Queue full?: NO + * Timeout: YES + * Execution Result: HystrixRuntimeException (but timeout prior) + * Fallback: SUCCESS + */ + @Test + public void testExecutionHookThreadTimeoutSuccessfulFallbackRunFailure() { + assertHooksOnSuccess( + new Func0>() { + @Override + public TestHystrixCommand call() { + return getCommand(ExecutionIsolationStrategy.THREAD, AbstractTestHystrixCommand.ExecutionResult.FAILURE, 150, AbstractTestHystrixCommand.FallbackResult.SUCCESS, 50); + } + }, + new Action1>() { + @Override + public void call(TestHystrixCommand command) { + TestableExecutionHook hook = command.getBuilder().executionHook; + assertTrue(hook.commandEmissionsMatch(1, 0, 1)); + assertTrue(hook.executionEventsMatch(0, 0, 0)); + assertTrue(hook.fallbackEventsMatch(1, 0, 1)); + assertEquals("onStart - onThreadStart - !onRunStart - onExecutionStart - onFallbackStart - onFallbackEmit - !onFallbackSuccess - !onComplete - onEmit - onFallbackSuccess - onSuccess - ", hook.executionSequence.toString()); + + try { + Thread.sleep(300); + } catch (Exception ex) { + throw new RuntimeException(ex); + } + assertTrue(hook.commandEmissionsMatch(1, 0, 1)); + assertTrue(hook.executionEventsMatch(0, 1, 0)); + assertTrue(hook.fallbackEventsMatch(1, 0, 1)); + assertEquals(RuntimeException.class, hook.getExecutionException().getClass()); + assertEquals("onStart - onThreadStart - !onRunStart - onExecutionStart - onFallbackStart - onFallbackEmit - !onFallbackSuccess - !onComplete - onEmit - onFallbackSuccess - onSuccess - onExecutionError - !onRunError - onThreadComplete - ", hook.executionSequence.toString()); + + } + }); + } + + /** + * Short-circuit? : NO + * Thread/semaphore: THREAD + * Thread Pool full? : NO + * Thread Pool Queue full?: NO + * Timeout: YES + * Execution Result: HystrixRuntimeException (but timeout prior) + * Fallback: synchronous HystrixRuntimeException + */ + @Test + public void testExecutionHookThreadTimeoutUnsuccessfulFallbackRunFailure() { + assertHooksOnFailure( + new Func0>() { + @Override + public TestHystrixCommand call() { + return getCommand(ExecutionIsolationStrategy.THREAD, AbstractTestHystrixCommand.ExecutionResult.FAILURE, 150, AbstractTestHystrixCommand.FallbackResult.FAILURE, 50); + } + }, + new Action1>() { + @Override + public void call(TestHystrixCommand command) { + TestableExecutionHook hook = command.getBuilder().executionHook; + assertTrue(hook.commandEmissionsMatch(0, 1, 0)); + assertTrue(hook.executionEventsMatch(0, 0, 0)); + assertTrue(hook.fallbackEventsMatch(0, 1, 0)); + assertEquals(TimeoutException.class, hook.getCommandException().getClass()); + assertEquals(RuntimeException.class, hook.getFallbackException().getClass()); + assertEquals("onStart - onThreadStart - !onRunStart - onExecutionStart - onFallbackStart - onFallbackError - onError - ", hook.executionSequence.toString()); + + try { + Thread.sleep(300); + } catch (Exception ex) { + throw new RuntimeException(ex); + } + assertTrue(hook.commandEmissionsMatch(0, 1, 0)); + assertTrue(hook.executionEventsMatch(0, 1, 0)); + assertTrue(hook.fallbackEventsMatch(0, 1, 0)); + assertEquals(RuntimeException.class, hook.getExecutionException().getClass()); + assertEquals("onStart - onThreadStart - !onRunStart - onExecutionStart - onFallbackStart - onFallbackError - onError - onExecutionError - !onRunError - onThreadComplete - ", hook.executionSequence.toString()); + + } + }); + } + + /** + * Short-circuit? : NO + * Thread/semaphore: THREAD + * Thread Pool full? : YES + * Thread Pool Queue full?: YES + * Fallback: UnsupportedOperationException + */ + @Test + public void testExecutionHookThreadPoolQueueFullNoFallback() { + assertHooksOnFailFast( + new Func0>() { + @Override + public TestHystrixCommand call() { + HystrixCircuitBreakerTest.TestCircuitBreaker circuitBreaker = new HystrixCircuitBreakerTest.TestCircuitBreaker(); + HystrixThreadPool pool = new SingleThreadedPoolWithQueue(1); + try { + // fill the pool + getLatentCommand(ExecutionIsolationStrategy.THREAD, AbstractTestHystrixCommand.ExecutionResult.SUCCESS, 500, AbstractTestHystrixCommand.FallbackResult.UNIMPLEMENTED, circuitBreaker, pool, 600).observe(); + // fill the queue + getLatentCommand(ExecutionIsolationStrategy.THREAD, AbstractTestHystrixCommand.ExecutionResult.SUCCESS, 500, AbstractTestHystrixCommand.FallbackResult.UNIMPLEMENTED, circuitBreaker, pool, 600).observe(); + } catch (Exception e) { + // ignore + } + return getLatentCommand(ExecutionIsolationStrategy.THREAD, AbstractTestHystrixCommand.ExecutionResult.SUCCESS, 500, AbstractTestHystrixCommand.FallbackResult.UNIMPLEMENTED, circuitBreaker, pool, 600); + } + }, + new Action1>() { + @Override + public void call(TestHystrixCommand command) { + TestableExecutionHook hook = command.getBuilder().executionHook; + assertTrue(hook.commandEmissionsMatch(0, 1, 0)); + assertTrue(hook.executionEventsMatch(0, 0, 0)); + assertTrue(hook.fallbackEventsMatch(0, 0, 0)); + assertEquals(RejectedExecutionException.class, hook.getCommandException().getClass()); + assertNull(hook.getFallbackException()); + assertEquals("onStart - onError - ", hook.executionSequence.toString()); + } + }); + } + + /** + * Short-circuit? : NO + * Thread/semaphore: THREAD + * Thread Pool full? : YES + * Thread Pool Queue full?: YES + * Fallback: SUCCESS + */ + @Test + public void testExecutionHookThreadPoolQueueFullSuccessfulFallback() { + assertHooksOnSuccess( + new Func0>() { + @Override + public TestHystrixCommand call() { + HystrixCircuitBreakerTest.TestCircuitBreaker circuitBreaker = new HystrixCircuitBreakerTest.TestCircuitBreaker(); + HystrixThreadPool pool = new SingleThreadedPoolWithQueue(1); + try { + // fill the pool + getLatentCommand(ExecutionIsolationStrategy.THREAD, AbstractTestHystrixCommand.ExecutionResult.SUCCESS, 500, AbstractTestHystrixCommand.FallbackResult.SUCCESS, circuitBreaker, pool, 600).observe(); + // fill the queue + getLatentCommand(ExecutionIsolationStrategy.THREAD, AbstractTestHystrixCommand.ExecutionResult.SUCCESS, 500, AbstractTestHystrixCommand.FallbackResult.SUCCESS, circuitBreaker, pool, 600).observe(); + } catch (Exception e) { + // ignore + } + + return getLatentCommand(ExecutionIsolationStrategy.THREAD, AbstractTestHystrixCommand.ExecutionResult.SUCCESS, 500, AbstractTestHystrixCommand.FallbackResult.SUCCESS, circuitBreaker, pool, 600); + } + }, + new Action1>() { + @Override + public void call(TestHystrixCommand command) { + TestableExecutionHook hook = command.getBuilder().executionHook; + assertTrue(hook.commandEmissionsMatch(1, 0, 1)); + assertTrue(hook.executionEventsMatch(0, 0, 0)); + assertTrue(hook.fallbackEventsMatch(1, 0, 1)); + assertEquals("onStart - onFallbackStart - onFallbackEmit - !onFallbackSuccess - !onComplete - onEmit - onFallbackSuccess - onSuccess - ", hook.executionSequence.toString()); + } + }); + } + + /** + * Short-circuit? : NO + * Thread/semaphore: THREAD + * Thread Pool full? : YES + * Thread Pool Queue full?: YES + * Fallback: synchronous HystrixRuntimeException + */ + @Test + public void testExecutionHookThreadPoolQueueFullUnsuccessfulFallback() { + assertHooksOnFailFast( + new Func0>() { + @Override + public TestHystrixCommand call() { + HystrixCircuitBreakerTest.TestCircuitBreaker circuitBreaker = new HystrixCircuitBreakerTest.TestCircuitBreaker(); + HystrixThreadPool pool = new SingleThreadedPoolWithQueue(1); + try { + // fill the pool + getLatentCommand(ExecutionIsolationStrategy.THREAD, AbstractTestHystrixCommand.ExecutionResult.SUCCESS, 500, AbstractTestHystrixCommand.FallbackResult.FAILURE, circuitBreaker, pool, 600).observe(); + // fill the queue + getLatentCommand(ExecutionIsolationStrategy.THREAD, AbstractTestHystrixCommand.ExecutionResult.SUCCESS, 500, AbstractTestHystrixCommand.FallbackResult.FAILURE, circuitBreaker, pool, 600).observe(); + } catch (Exception e) { + // ignore + } + + return getLatentCommand(ExecutionIsolationStrategy.THREAD, AbstractTestHystrixCommand.ExecutionResult.SUCCESS, 500, AbstractTestHystrixCommand.FallbackResult.FAILURE, circuitBreaker, pool, 600); + } + }, + new Action1>() { + @Override + public void call(TestHystrixCommand command) { + TestableExecutionHook hook = command.getBuilder().executionHook; + assertTrue(hook.commandEmissionsMatch(0, 1, 0)); + assertTrue(hook.executionEventsMatch(0, 0, 0)); + assertTrue(hook.fallbackEventsMatch(0, 1, 0)); + assertEquals(RejectedExecutionException.class, hook.getCommandException().getClass()); + assertEquals(RuntimeException.class, hook.getFallbackException().getClass()); + assertEquals("onStart - onFallbackStart - onFallbackError - onError - ", hook.executionSequence.toString()); + } + }); + } + + /** + * Short-circuit? : NO + * Thread/semaphore: THREAD + * Thread Pool full? : YES + * Thread Pool Queue full?: N/A + * Fallback: UnsupportedOperationException + */ + @Test + public void testExecutionHookThreadPoolFullNoFallback() { + assertHooksOnFailFast( + new Func0>() { + @Override + public TestHystrixCommand call() { + HystrixCircuitBreakerTest.TestCircuitBreaker circuitBreaker = new HystrixCircuitBreakerTest.TestCircuitBreaker(); + HystrixThreadPool pool = new SingleThreadedPoolWithNoQueue(); + try { + // fill the pool + getLatentCommand(ExecutionIsolationStrategy.THREAD, AbstractTestHystrixCommand.ExecutionResult.SUCCESS, 500, AbstractTestHystrixCommand.FallbackResult.UNIMPLEMENTED, circuitBreaker, pool, 600).observe(); + } catch (Exception e) { + // ignore + } + + return getLatentCommand(ExecutionIsolationStrategy.THREAD, AbstractTestHystrixCommand.ExecutionResult.SUCCESS, 500, AbstractTestHystrixCommand.FallbackResult.UNIMPLEMENTED, circuitBreaker, pool, 600); + } + }, + new Action1>() { + @Override + public void call(TestHystrixCommand command) { + TestableExecutionHook hook = command.getBuilder().executionHook; + assertTrue(hook.commandEmissionsMatch(0, 1, 0)); + assertTrue(hook.executionEventsMatch(0, 0, 0)); + assertTrue(hook.fallbackEventsMatch(0, 0, 0)); + assertEquals(RejectedExecutionException.class, hook.getCommandException().getClass()); + assertNull(hook.getFallbackException()); + assertEquals("onStart - onError - ", hook.executionSequence.toString()); + } + }); + } + + /** + * Short-circuit? : NO + * Thread/semaphore: THREAD + * Thread Pool full? : YES + * Thread Pool Queue full?: N/A + * Fallback: SUCCESS + */ + @Test + public void testExecutionHookThreadPoolFullSuccessfulFallback() { + assertHooksOnSuccess( + new Func0>() { + @Override + public TestHystrixCommand call() { + HystrixCircuitBreakerTest.TestCircuitBreaker circuitBreaker = new HystrixCircuitBreakerTest.TestCircuitBreaker(); + HystrixThreadPool pool = new SingleThreadedPoolWithNoQueue(); + try { + // fill the pool + getLatentCommand(ExecutionIsolationStrategy.THREAD, AbstractTestHystrixCommand.ExecutionResult.SUCCESS, 500, AbstractTestHystrixCommand.FallbackResult.SUCCESS, circuitBreaker, pool, 600).observe(); + } catch (Exception e) { + // ignore + } + + return getLatentCommand(ExecutionIsolationStrategy.THREAD, AbstractTestHystrixCommand.ExecutionResult.SUCCESS, 500, AbstractTestHystrixCommand.FallbackResult.SUCCESS, circuitBreaker, pool, 600); + } + }, + new Action1>() { + @Override + public void call(TestHystrixCommand command) { + TestableExecutionHook hook = command.getBuilder().executionHook; + assertTrue(hook.commandEmissionsMatch(1, 0, 1)); + assertTrue(hook.executionEventsMatch(0, 0, 0)); + assertEquals("onStart - onFallbackStart - onFallbackEmit - !onFallbackSuccess - !onComplete - onEmit - onFallbackSuccess - onSuccess - ", hook.executionSequence.toString()); + } + }); + } + + /** + * Short-circuit? : NO + * Thread/semaphore: THREAD + * Thread Pool full? : YES + * Thread Pool Queue full?: N/A + * Fallback: synchronous HystrixRuntimeException + */ + @Test + public void testExecutionHookThreadPoolFullUnsuccessfulFallback() { + assertHooksOnFailFast( + new Func0>() { + @Override + public TestHystrixCommand call() { + HystrixCircuitBreakerTest.TestCircuitBreaker circuitBreaker = new HystrixCircuitBreakerTest.TestCircuitBreaker(); + HystrixThreadPool pool = new SingleThreadedPoolWithNoQueue(); + try { + // fill the pool + getLatentCommand(ExecutionIsolationStrategy.THREAD, AbstractTestHystrixCommand.ExecutionResult.SUCCESS, 500, AbstractTestHystrixCommand.FallbackResult.FAILURE, circuitBreaker, pool, 600).observe(); + } catch (Exception e) { + // ignore + } + + return getLatentCommand(ExecutionIsolationStrategy.THREAD, AbstractTestHystrixCommand.ExecutionResult.SUCCESS, 500, AbstractTestHystrixCommand.FallbackResult.FAILURE, circuitBreaker, pool, 600); + } + }, + new Action1>() { + @Override + public void call(TestHystrixCommand command) { + TestableExecutionHook hook = command.getBuilder().executionHook; + assertTrue(hook.commandEmissionsMatch(0, 1, 0)); + assertTrue(hook.executionEventsMatch(0, 0, 0)); + assertTrue(hook.fallbackEventsMatch(0, 1, 0)); + assertEquals(RejectedExecutionException.class, hook.getCommandException().getClass()); + assertEquals(RuntimeException.class, hook.getFallbackException().getClass()); + assertEquals("onStart - onFallbackStart - onFallbackError - onError - ", hook.executionSequence.toString()); + } + }); + } + + /** + * Short-circuit? : YES + * Thread/semaphore: THREAD + * Fallback: UnsupportedOperationException + */ + @Test + public void testExecutionHookThreadShortCircuitNoFallback() { + assertHooksOnFailFast( + new Func0>() { + @Override + public TestHystrixCommand call() { + return getCircuitOpenCommand(ExecutionIsolationStrategy.THREAD, AbstractTestHystrixCommand.FallbackResult.UNIMPLEMENTED); + } + }, + new Action1>() { + @Override + public void call(TestHystrixCommand command) { + TestableExecutionHook hook = command.getBuilder().executionHook; + assertTrue(hook.commandEmissionsMatch(0, 1, 0)); + assertTrue(hook.executionEventsMatch(0, 0, 0)); + assertTrue(hook.fallbackEventsMatch(0, 0, 0)); + assertEquals(RuntimeException.class, hook.getCommandException().getClass()); + assertNull(hook.getFallbackException()); + assertEquals("onStart - onError - ", hook.executionSequence.toString()); + } + }); + } + + /** + * Short-circuit? : YES + * Thread/semaphore: THREAD + * Fallback: SUCCESS + */ + @Test + public void testExecutionHookThreadShortCircuitSuccessfulFallback() { + assertHooksOnSuccess( + new Func0>() { + @Override + public TestHystrixCommand call() { + return getCircuitOpenCommand(ExecutionIsolationStrategy.THREAD, AbstractTestHystrixCommand.FallbackResult.SUCCESS); + } + }, + new Action1>() { + @Override + public void call(TestHystrixCommand command) { + TestableExecutionHook hook = command.getBuilder().executionHook; + assertTrue(hook.commandEmissionsMatch(1, 0, 1)); + assertTrue(hook.executionEventsMatch(0, 0, 0)); + assertTrue(hook.fallbackEventsMatch(1, 0, 1)); + assertEquals("onStart - onFallbackStart - onFallbackEmit - !onFallbackSuccess - !onComplete - onEmit - onFallbackSuccess - onSuccess - ", hook.executionSequence.toString()); + } + }); + } + + /** + * Short-circuit? : YES + * Thread/semaphore: THREAD + * Fallback: synchronous HystrixRuntimeException + */ + @Test + public void testExecutionHookThreadShortCircuitUnsuccessfulFallback() { + assertHooksOnFailFast( + new Func0>() { + @Override + public TestHystrixCommand call() { + HystrixCircuitBreakerTest.TestCircuitBreaker circuitBreaker = new HystrixCircuitBreakerTest.TestCircuitBreaker().setForceShortCircuit(true); + return getCircuitOpenCommand(ExecutionIsolationStrategy.THREAD, AbstractTestHystrixCommand.FallbackResult.FAILURE); + } + }, + new Action1>() { + @Override + public void call(TestHystrixCommand command) { + TestableExecutionHook hook = command.getBuilder().executionHook; + assertTrue(hook.commandEmissionsMatch(0, 1, 0)); + assertTrue(hook.executionEventsMatch(0, 0, 0)); + assertTrue(hook.fallbackEventsMatch(0, 1, 0)); + assertEquals(RuntimeException.class, hook.getCommandException().getClass()); + assertEquals(RuntimeException.class, hook.getFallbackException().getClass()); + assertEquals("onStart - onFallbackStart - onFallbackError - onError - ", hook.executionSequence.toString()); + } + }); + } + + /** + * Short-circuit? : NO + * Request-cache? : YES + */ + @Test + public void testExecutionHookResponseFromCache() { + final HystrixCommandKey key = HystrixCommandKey.Factory.asKey("Hook-Cache"); + getCommand(key, ExecutionIsolationStrategy.THREAD, AbstractTestHystrixCommand.ExecutionResult.SUCCESS, 0, AbstractTestHystrixCommand.FallbackResult.UNIMPLEMENTED, 0, new HystrixCircuitBreakerTest.TestCircuitBreaker(), null, 100, AbstractTestHystrixCommand.CacheEnabled.YES, 42, 10, 10).observe(); + + assertHooksOnSuccess( + new Func0>() { + @Override + public TestHystrixCommand call() { + return getCommand(key, ExecutionIsolationStrategy.THREAD, AbstractTestHystrixCommand.ExecutionResult.SUCCESS, 0, AbstractTestHystrixCommand.FallbackResult.UNIMPLEMENTED, 0, new HystrixCircuitBreakerTest.TestCircuitBreaker(), null, 100, AbstractTestHystrixCommand.CacheEnabled.YES, 42, 10, 10); + } + }, + new Action1>() { + @Override + public void call(TestHystrixCommand command) { + TestableExecutionHook hook = command.getBuilder().executionHook; + assertTrue(hook.commandEmissionsMatch(0, 0, 0)); + assertTrue(hook.executionEventsMatch(0, 0, 0)); + assertTrue(hook.fallbackEventsMatch(0, 0, 0)); + assertEquals("onCacheHit - ", hook.executionSequence.toString()); + } + }); + } + + /** + *********************** END THREAD-ISOLATED Execution Hook Tests ************************************** + */ + + + /* ******************************************************************************** */ + /* ******************************************************************************** */ + /* private HystrixCommand class implementations for unit testing */ + /* ******************************************************************************** */ + /* ******************************************************************************** */ + + static AtomicInteger uniqueNameCounter = new AtomicInteger(1); + + @Override + TestHystrixCommand getCommand(ExecutionIsolationStrategy isolationStrategy, AbstractTestHystrixCommand.ExecutionResult executionResult, int executionLatency, AbstractTestHystrixCommand.FallbackResult fallbackResult, int fallbackLatency, TestCircuitBreaker circuitBreaker, HystrixThreadPool threadPool, int timeout, AbstractTestHystrixCommand.CacheEnabled cacheEnabled, Object value, TryableSemaphore executionSemaphore, TryableSemaphore fallbackSemaphore, boolean circuitBreakerDisabled) { + HystrixCommandKey commandKey = HystrixCommandKey.Factory.asKey("Flexible-" + uniqueNameCounter.getAndIncrement()); + return FlexibleTestHystrixCommand.from(commandKey, isolationStrategy, executionResult, executionLatency, fallbackResult, fallbackLatency, circuitBreaker, threadPool, timeout, cacheEnabled, value, executionSemaphore, fallbackSemaphore, circuitBreakerDisabled); + } + + @Override + TestHystrixCommand getCommand(HystrixCommandKey commandKey, ExecutionIsolationStrategy isolationStrategy, AbstractTestHystrixCommand.ExecutionResult executionResult, int executionLatency, AbstractTestHystrixCommand.FallbackResult fallbackResult, int fallbackLatency, TestCircuitBreaker circuitBreaker, HystrixThreadPool threadPool, int timeout, AbstractTestHystrixCommand.CacheEnabled cacheEnabled, Object value, TryableSemaphore executionSemaphore, TryableSemaphore fallbackSemaphore, boolean circuitBreakerDisabled) { + return FlexibleTestHystrixCommand.from(commandKey, isolationStrategy, executionResult, executionLatency, fallbackResult, fallbackLatency, circuitBreaker, threadPool, timeout, cacheEnabled, value, executionSemaphore, fallbackSemaphore, circuitBreakerDisabled); + } + + private static class FlexibleTestHystrixCommand { + + public static Integer EXECUTE_VALUE = 1; + public static Integer FALLBACK_VALUE = 11; + + public static AbstractFlexibleTestHystrixCommand from(HystrixCommandKey commandKey, ExecutionIsolationStrategy isolationStrategy, AbstractTestHystrixCommand.ExecutionResult executionResult, int executionLatency, AbstractTestHystrixCommand.FallbackResult fallbackResult, int fallbackLatency, TestCircuitBreaker circuitBreaker, HystrixThreadPool threadPool, int timeout, AbstractTestHystrixCommand.CacheEnabled cacheEnabled, Object value, TryableSemaphore executionSemaphore, TryableSemaphore fallbackSemaphore, boolean circuitBreakerDisabled) { + if (fallbackResult.equals(AbstractTestHystrixCommand.FallbackResult.UNIMPLEMENTED)) { + return new FlexibleTestHystrixCommandNoFallback(commandKey, isolationStrategy, executionResult, executionLatency, circuitBreaker, threadPool, timeout, cacheEnabled, value, executionSemaphore, fallbackSemaphore, circuitBreakerDisabled); + } else { + return new FlexibleTestHystrixCommandWithFallback(commandKey, isolationStrategy, executionResult, executionLatency, fallbackResult, fallbackLatency, circuitBreaker, threadPool, timeout, cacheEnabled, value, executionSemaphore, fallbackSemaphore, circuitBreakerDisabled); + } + } + } + + private static class AbstractFlexibleTestHystrixCommand extends TestHystrixCommand { + protected final AbstractTestHystrixCommand.ExecutionResult executionResult; + protected final int executionLatency; + + protected final CacheEnabled cacheEnabled; + protected final Object value; + + + AbstractFlexibleTestHystrixCommand(HystrixCommandKey commandKey, ExecutionIsolationStrategy isolationStrategy, AbstractTestHystrixCommand.ExecutionResult executionResult, int executionLatency, TestCircuitBreaker circuitBreaker, HystrixThreadPool threadPool, int timeout, CacheEnabled cacheEnabled, Object value, TryableSemaphore executionSemaphore, TryableSemaphore fallbackSemaphore, boolean circuitBreakerDisabled) { + super(testPropsBuilder(circuitBreaker) + .setCommandKey(commandKey) + .setCircuitBreaker(circuitBreaker) + .setMetrics(circuitBreaker.metrics) + .setThreadPool(threadPool) + .setCommandPropertiesDefaults(HystrixCommandPropertiesTest.getUnitTestPropertiesSetter() + .withExecutionIsolationStrategy(isolationStrategy) + .withExecutionTimeoutInMilliseconds(timeout) + .withCircuitBreakerEnabled(!circuitBreakerDisabled)) + .setExecutionSemaphore(executionSemaphore) + .setFallbackSemaphore(fallbackSemaphore)); + this.executionResult = executionResult; + this.executionLatency = executionLatency; + + this.cacheEnabled = cacheEnabled; + this.value = value; + } + + @Override + protected Integer run() throws Exception { + System.out.println(System.currentTimeMillis() + " : " + Thread.currentThread().getName() + " starting the run() method"); + addLatency(executionLatency); + if (executionResult == AbstractTestHystrixCommand.ExecutionResult.SUCCESS) { + return FlexibleTestHystrixCommand.EXECUTE_VALUE; + } else if (executionResult == AbstractTestHystrixCommand.ExecutionResult.FAILURE) { + throw new RuntimeException("Execution Failure for TestHystrixCommand"); + } else if (executionResult == AbstractTestHystrixCommand.ExecutionResult.HYSTRIX_FAILURE) { + throw new HystrixRuntimeException(HystrixRuntimeException.FailureType.COMMAND_EXCEPTION, AbstractFlexibleTestHystrixCommand.class, "Execution Hystrix Failure for TestHystrixCommand", new RuntimeException("Execution Failure for TestHystrixCommand"), new RuntimeException("Fallback Failure for TestHystrixCommand")); + } else if (executionResult == AbstractTestHystrixCommand.ExecutionResult.RECOVERABLE_ERROR) { + throw new java.lang.Error("Execution ERROR for TestHystrixCommand"); + } else if (executionResult == AbstractTestHystrixCommand.ExecutionResult.UNRECOVERABLE_ERROR) { + throw new StackOverflowError("Unrecoverable Error for TestHystrixCommand"); + } else if (executionResult == AbstractTestHystrixCommand.ExecutionResult.BAD_REQUEST) { + throw new HystrixBadRequestException("Execution BadRequestException for TestHystrixCommand"); + } else { + throw new RuntimeException("You passed in a executionResult enum that can't be represented in HystrixCommand: " + executionResult); + } + } + + @Override + public String getCacheKey() { + if (cacheEnabled == CacheEnabled.YES) + return value.toString(); + else + return null; + } + + protected void addLatency(int latency) { + if (latency > 0) { + try { + System.out.println(System.currentTimeMillis() + " : " + Thread.currentThread().getName() + " About to sleep for : " + latency); + Thread.sleep(latency); + System.out.println(System.currentTimeMillis() + " : " + Thread.currentThread().getName() + " Woke up from sleep!"); + } catch (InterruptedException e) { + e.printStackTrace(); + // ignore and sleep some more to simulate a dependency that doesn't obey interrupts + try { + Thread.sleep(latency); + } catch (Exception e2) { + // ignore + } + System.out.println("after interruption with extra sleep"); + } + } + } + + } + + private static class FlexibleTestHystrixCommandWithFallback extends AbstractFlexibleTestHystrixCommand { + protected final AbstractTestHystrixCommand.FallbackResult fallbackResult; + protected final int fallbackLatency; + + FlexibleTestHystrixCommandWithFallback(HystrixCommandKey commandKey, ExecutionIsolationStrategy isolationStrategy, AbstractTestHystrixCommand.ExecutionResult executionResult, int executionLatency, FallbackResult fallbackResult, int fallbackLatency, TestCircuitBreaker circuitBreaker, HystrixThreadPool threadPool, int timeout, CacheEnabled cacheEnabled, Object value, TryableSemaphore executionSemaphore, TryableSemaphore fallbackSemaphore, boolean circuitBreakerDisabled) { + super(commandKey, isolationStrategy, executionResult, executionLatency, circuitBreaker, threadPool, timeout, cacheEnabled, value, executionSemaphore, fallbackSemaphore, circuitBreakerDisabled); + this.fallbackResult = fallbackResult; + this.fallbackLatency = fallbackLatency; + } + + @Override + protected Integer getFallback() { + addLatency(fallbackLatency); + if (fallbackResult == AbstractTestHystrixCommand.FallbackResult.SUCCESS) { + return FlexibleTestHystrixCommand.FALLBACK_VALUE; + } else if (fallbackResult == AbstractTestHystrixCommand.FallbackResult.FAILURE) { + throw new RuntimeException("Fallback Failure for TestHystrixCommand"); + } else if (fallbackResult == FallbackResult.UNIMPLEMENTED) { + return super.getFallback(); + } else { + throw new RuntimeException("You passed in a fallbackResult enum that can't be represented in HystrixCommand: " + fallbackResult); + } + } + } + + private static class FlexibleTestHystrixCommandNoFallback extends AbstractFlexibleTestHystrixCommand { + FlexibleTestHystrixCommandNoFallback(HystrixCommandKey commandKey, ExecutionIsolationStrategy isolationStrategy, AbstractTestHystrixCommand.ExecutionResult executionResult, int executionLatency, TestCircuitBreaker circuitBreaker, HystrixThreadPool threadPool, int timeout, CacheEnabled cacheEnabled, Object value, TryableSemaphore executionSemaphore, TryableSemaphore fallbackSemaphore, boolean circuitBreakerDisabled) { + super(commandKey, isolationStrategy, executionResult, executionLatency, circuitBreaker, threadPool, timeout, cacheEnabled, value, executionSemaphore, fallbackSemaphore, circuitBreakerDisabled); + } + } + + /** + * Successful execution - no fallback implementation. + */ + private static class SuccessfulTestCommand extends TestHystrixCommand { + + public SuccessfulTestCommand() { + this(HystrixCommandPropertiesTest.getUnitTestPropertiesSetter()); + } + + public SuccessfulTestCommand(HystrixCommandProperties.Setter properties) { + super(testPropsBuilder().setCommandPropertiesDefaults(properties)); + } + + @Override + protected Boolean run() { + return true; + } + + } + + /** + * Successful execution - no fallback implementation. */ private static class DynamicOwnerTestCommand extends TestHystrixCommand { @@ -3893,6 +5259,36 @@ protected String getCacheKey() { } + private static class AsyncCacheableCommand extends HystrixCommand { + private final String arg; + private final AtomicBoolean cancelled = new AtomicBoolean(false); + + public AsyncCacheableCommand(String arg) { + super(Setter.withGroupKey(HystrixCommandGroupKey.Factory.asKey("ASYNC"))); + this.arg = arg; + } + + @Override + protected Boolean run() { + try { + Thread.sleep(100); + return true; + } catch (InterruptedException ex) { + cancelled.set(true); + throw new RuntimeException(ex); + } + } + + @Override + protected String getCacheKey() { + return arg; + } + + public boolean isCancelled() { + return cancelled.get(); + } + } + private static class BusinessException extends Exception { public BusinessException(String msg) { super(msg); diff --git a/hystrix-core/src/test/java/com/netflix/hystrix/HystrixObservableCommandTest.java b/hystrix-core/src/test/java/com/netflix/hystrix/HystrixObservableCommandTest.java index f122af4d1..5ad8beb94 100644 --- a/hystrix-core/src/test/java/com/netflix/hystrix/HystrixObservableCommandTest.java +++ b/hystrix-core/src/test/java/com/netflix/hystrix/HystrixObservableCommandTest.java @@ -4496,7 +4496,7 @@ public void testExecutionPartialSuccessWithFallback() { } @Test - public void testEarlyUnsubscribeDuringExecution() { + public void testEarlyUnsubscribeDuringExecutionViaToObservable() { class AsyncCommand extends HystrixObservableCommand { public AsyncCommand() { @@ -4555,7 +4555,7 @@ public void onNext(Boolean b) { assertTrue(latch.await(200, TimeUnit.MILLISECONDS)); assertEquals("Number of execution semaphores in use", 0, cmd.getExecutionSemaphore().getNumberOfPermitsUsed()); assertEquals("Number of fallback semaphores in use", 0, cmd.getFallbackSemaphore().getNumberOfPermitsUsed()); - assertTrue(cmd.isExecutionComplete()); + assertFalse(cmd.isExecutionComplete()); assertFalse(cmd.isExecutedInThread()); System.out.println("EventCounts : " + cmd.getEventCounts()); System.out.println("Execution Time : " + cmd.getExecutionTimeInMilliseconds()); @@ -4565,6 +4565,77 @@ public void onNext(Boolean b) { } } + @Test + public void testEarlyUnsubscribeDuringExecutionViaObserve() { + class AsyncCommand extends HystrixObservableCommand { + + public AsyncCommand() { + super(Setter.withGroupKey(HystrixCommandGroupKey.Factory.asKey("ASYNC"))); + } + + @Override + protected Observable construct() { + return Observable.defer(new Func0>() { + @Override + public Observable call() { + try { + Thread.sleep(100); + return Observable.just(true); + } catch (InterruptedException ex) { + return Observable.error(ex); + } + } + }).subscribeOn(Schedulers.io()); + } + } + + HystrixObservableCommand cmd = new AsyncCommand(); + + final CountDownLatch latch = new CountDownLatch(1); + + Observable o = cmd.observe(); + Subscription s = o. + doOnUnsubscribe(new Action0() { + @Override + public void call() { + System.out.println("OnUnsubscribe"); + latch.countDown(); + } + }). + subscribe(new Subscriber() { + @Override + public void onCompleted() { + System.out.println("OnCompleted"); + latch.countDown(); + } + + @Override + public void onError(Throwable e) { + System.out.println("OnError : " + e); + } + + @Override + public void onNext(Boolean b) { + System.out.println("OnNext : " + b); + } + }); + + try { + s.unsubscribe(); + assertTrue(latch.await(200, TimeUnit.MILLISECONDS)); + assertEquals("Number of execution semaphores in use", 0, cmd.getExecutionSemaphore().getNumberOfPermitsUsed()); + assertEquals("Number of fallback semaphores in use", 0, cmd.getFallbackSemaphore().getNumberOfPermitsUsed()); + assertFalse(cmd.isExecutionComplete()); + assertFalse(cmd.isExecutedInThread()); + System.out.println("EventCounts : " + cmd.getEventCounts()); + System.out.println("Execution Time : " + cmd.getExecutionTimeInMilliseconds()); + System.out.println("Is Successful : " + cmd.isSuccessfulExecution()); + } catch (InterruptedException ex) { + ex.printStackTrace(); + } + } + + @Test public void testEarlyUnsubscribeDuringFallback() { class AsyncCommand extends HystrixObservableCommand { @@ -4631,7 +4702,7 @@ public void onNext(Boolean b) { assertTrue(latch.await(200, TimeUnit.MILLISECONDS)); assertEquals("Number of execution semaphores in use", 0, cmd.getExecutionSemaphore().getNumberOfPermitsUsed()); assertEquals("Number of fallback semaphores in use", 0, cmd.getFallbackSemaphore().getNumberOfPermitsUsed()); - assertTrue(cmd.isExecutionComplete()); + assertFalse(cmd.isExecutionComplete()); assertFalse(cmd.isExecutedInThread()); } catch (InterruptedException ex) { ex.printStackTrace(); diff --git a/hystrix-core/src/test/java/com/netflix/hystrix/HystrixRequestCacheTest.java b/hystrix-core/src/test/java/com/netflix/hystrix/HystrixRequestCacheTest.java index 353ebc2cd..eb4d6da47 100644 --- a/hystrix-core/src/test/java/com/netflix/hystrix/HystrixRequestCacheTest.java +++ b/hystrix-core/src/test/java/com/netflix/hystrix/HystrixRequestCacheTest.java @@ -43,10 +43,10 @@ public void testCache() { HystrixRequestCache cache2 = HystrixRequestCache.getInstance(HystrixCommandKey.Factory.asKey("command2"), strategy); cache2.putIfAbsent("valueA", new TestObservable("a3")); - assertEquals("a1", cache1.get("valueA").toBlocking().last()); - assertEquals("b1", cache1.get("valueB").toBlocking().last()); + assertEquals("a1", cache1.get("valueA").toObservable().toBlocking().last()); + assertEquals("b1", cache1.get("valueB").toObservable().toBlocking().last()); - assertEquals("a3", cache2.get("valueA").toBlocking().last()); + assertEquals("a3", cache2.get("valueA").toObservable().toBlocking().last()); assertNull(cache2.get("valueB")); } catch (Exception e) { fail("Exception: " + e.getMessage()); @@ -81,7 +81,7 @@ public void testClearCache() { try { HystrixRequestCache cache1 = HystrixRequestCache.getInstance(HystrixCommandKey.Factory.asKey("command1"), strategy); cache1.putIfAbsent("valueA", new TestObservable("a1")); - assertEquals("a1", cache1.get("valueA").toBlocking().last()); + assertEquals("a1", cache1.get("valueA").toObservable().toBlocking().last()); cache1.clear("valueA"); assertNull(cache1.get("valueA")); } catch (Exception e) { @@ -107,17 +107,9 @@ public void testCacheWithoutRequestContext() { } } - private static class TestObservable extends Observable { - public TestObservable(final String value) { - super(new OnSubscribe() { - - @Override - public void call(Subscriber observer) { - observer.onNext(value); - observer.onCompleted(); - } - - }); + private static class TestObservable extends HystrixCachedObservable { + public TestObservable(String arg) { + super(Observable.just(arg), null); } } diff --git a/hystrix-core/src/test/java/com/netflix/hystrix/TestableExecutionHook.java b/hystrix-core/src/test/java/com/netflix/hystrix/TestableExecutionHook.java index c72d427b6..60340b739 100644 --- a/hystrix-core/src/test/java/com/netflix/hystrix/TestableExecutionHook.java +++ b/hystrix-core/src/test/java/com/netflix/hystrix/TestableExecutionHook.java @@ -53,6 +53,11 @@ private boolean eventsMatch(List> l, int numOnNext, int numOnErr int actualOnError = 0; int actualOnCompleted = 0; + + if (l.size() != numOnNext + numOnError + numOnCompleted) { + System.out.println("Actual : " + l + ", Expected : " + numOnNext + " OnNexts, " + numOnError + " OnErrors, " + numOnCompleted + " OnCompleted"); + return false; + } for (int n = 0; n < numOnNext; n++) { Notification current = l.get(n); if (!current.isOnNext()) { diff --git a/hystrix-core/src/test/java/com/netflix/hystrix/metric/CommandStreamTest.java b/hystrix-core/src/test/java/com/netflix/hystrix/metric/CommandStreamTest.java index d91506a87..0c269bad0 100644 --- a/hystrix-core/src/test/java/com/netflix/hystrix/metric/CommandStreamTest.java +++ b/hystrix-core/src/test/java/com/netflix/hystrix/metric/CommandStreamTest.java @@ -140,12 +140,21 @@ public static List getCommandsWithResponseFromCache(HystrixCommandGroup @Override protected Integer run() throws Exception { - Thread.sleep(executionLatency); - switch (executionResult) { - case SUCCESS: return 1; - case FAILURE: throw new RuntimeException("induced failure"); - case BAD_REQUEST: throw new HystrixBadRequestException("induced bad request"); - default: throw new RuntimeException("unhandled HystrixEventType : " + executionResult); + try { + Thread.sleep(executionLatency); + switch (executionResult) { + case SUCCESS: + return 1; + case FAILURE: + throw new RuntimeException("induced failure"); + case BAD_REQUEST: + throw new HystrixBadRequestException("induced bad request"); + default: + throw new RuntimeException("unhandled HystrixEventType : " + executionResult); + } + } catch (InterruptedException ex) { + System.out.println("Received InterruptedException : " + ex); + throw ex; } } diff --git a/hystrix-core/src/test/java/com/netflix/hystrix/metric/consumer/CumulativeCommandEventCounterStreamTest.java b/hystrix-core/src/test/java/com/netflix/hystrix/metric/consumer/CumulativeCommandEventCounterStreamTest.java index 4e32916ce..22581ec27 100644 --- a/hystrix-core/src/test/java/com/netflix/hystrix/metric/consumer/CumulativeCommandEventCounterStreamTest.java +++ b/hystrix-core/src/test/java/com/netflix/hystrix/metric/consumer/CumulativeCommandEventCounterStreamTest.java @@ -28,6 +28,8 @@ import org.junit.Before; import org.junit.Test; import rx.Subscriber; +import rx.Subscription; +import rx.functions.Action0; import java.util.ArrayList; import java.util.List; @@ -480,6 +482,56 @@ public void testFallbackRejection() { assertArrayEquals(expected, stream.getLatest()); } + @Test + public void testCancelled() { + HystrixCommandKey key = HystrixCommandKey.Factory.asKey("CMD-CumulativeCounter-M"); + stream = CumulativeCommandEventCounterStream.getInstance(key, 10, 100); + stream.startCachingStreamValuesIfUnstarted(); + + final CountDownLatch latch = new CountDownLatch(1); + stream.observe().take(10).subscribe(getSubscriber(latch)); + + Command toCancel = Command.from(groupKey, key, HystrixEventType.SUCCESS, 500); + + System.out.println(System.currentTimeMillis() + " : " + Thread.currentThread().getName() + " : about to observe and subscribe"); + Subscription s = toCancel.observe(). + doOnUnsubscribe(new Action0() { + @Override + public void call() { + System.out.println(System.currentTimeMillis() + " : " + Thread.currentThread().getName() + " : UnSubscribe from command.observe()"); + } + }). + subscribe(new Subscriber() { + @Override + public void onCompleted() { + System.out.println("Command OnCompleted"); + } + + @Override + public void onError(Throwable e) { + System.out.println("Command OnError : " + e); + } + + @Override + public void onNext(Integer i) { + System.out.println("Command OnNext : " + i); + } + }); + + System.out.println(System.currentTimeMillis() + " : " + Thread.currentThread().getName() + " : about to unsubscribe"); + s.unsubscribe(); + + try { + assertTrue(latch.await(10000, TimeUnit.MILLISECONDS)); + } catch (InterruptedException ex) { + fail("Interrupted ex"); + } + assertEquals(HystrixEventType.values().length, stream.getLatest().length); + long[] expected = new long[HystrixEventType.values().length]; + expected[HystrixEventType.CANCELLED.ordinal()] = 1; + assertArrayEquals(expected, stream.getLatest()); + } + @Test public void testCollapsed() { HystrixCommandKey key = HystrixCommandKey.Factory.asKey("BatchCommand"); @@ -508,7 +560,7 @@ public void testCollapsed() { @Test public void testMultipleEventsOverTimeGetStoredAndNeverAgeOut() { - HystrixCommandKey key = HystrixCommandKey.Factory.asKey("CMD-CumulativeCounter-M"); + HystrixCommandKey key = HystrixCommandKey.Factory.asKey("CMD-CumulativeCounter-N"); stream = CumulativeCommandEventCounterStream.getInstance(key, 10, 100); stream.startCachingStreamValuesIfUnstarted();