Skip to content

Commit

Permalink
feat(grpc): refine error message on deadline exceeded
Browse files Browse the repository at this point in the history
  • Loading branch information
rexledesma committed Dec 21, 2022
1 parent a28f5d5 commit 97e3945
Show file tree
Hide file tree
Showing 2 changed files with 13 additions and 1 deletion.
12 changes: 12 additions & 0 deletions python_modules/dagster/dagster/_grpc/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,18 @@ def _query(
):
try:
return self._get_response(method, request=request_type(**kwargs), timeout=timeout)
except grpc.RpcError as rpc_error:
if rpc_error.code() == grpc.StatusCode.DEADLINE_EXCEEDED:
raise DagsterUserCodeUnreachableError(
f"Your code location server took longer to respond than the {timeout} second timeout."
" One reason this might happen is if you are running a schedule or sensor"
" that is taking a long time to evaluate its tick. \n\n"
"You can fix this error by optimizing your evaluation function to be faster"
f"than the {timeout} second timeout, or by setting the environment variable"
f" `DAGSTER_GRPC_TIMEOUT_SECONDS` to be a value larger than {timeout}."
) from rpc_error

raise
except Exception as e:
raise DagsterUserCodeUnreachableError("Could not reach user code server") from e

Expand Down
2 changes: 1 addition & 1 deletion python_modules/dagster/dagster/_grpc/server.py
Original file line number Diff line number Diff line change
Expand Up @@ -1006,7 +1006,7 @@ def wait_for_grpc_server(server_process, client, subprocess_args, timeout=60):
try:
client.ping("")
return
except DagsterUserCodeUnreachableError:
except grpc._channel._InactiveRpcError:
last_error = serializable_error_info_from_exc_info(sys.exc_info())

if timeout > 0 and (time.time() - start_time > timeout):
Expand Down

0 comments on commit 97e3945

Please sign in to comment.