Skip to content

Commit

Permalink
retry after timeout
Browse files Browse the repository at this point in the history
  • Loading branch information
Davies Liu committed Apr 2, 2015
1 parent 191524e commit b838f35
Show file tree
Hide file tree
Showing 2 changed files with 11 additions and 1 deletion.
11 changes: 10 additions & 1 deletion core/src/main/scala/org/apache/spark/api/python/PythonRDD.scala
Original file line number Diff line number Diff line change
Expand Up @@ -613,7 +613,16 @@ private[spark] object PythonRDD extends Logging {
setDaemon(true)
override def run() {
try {
val sock = serverSocket.accept()
var sock: Socket = null
try {
sock = serverSocket.accept()
} catch {
case e: SocketTimeoutException =>
// there is a small chance that the client had connected, so retry
logWarning("Timed out after 4 seconds, retry once")
serverSocket.setSoTimeout(10)
sock = serverSocket.accept()
}
val out = new DataOutputStream(new BufferedOutputStream(sock.getOutputStream))
try {
writeIteratorToStream(items, out)
Expand Down
1 change: 1 addition & 0 deletions python/pyspark/rdd.py
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,7 @@ def _parse_memory(s):

def _load_from_socket(port, serializer):
sock = socket.socket()
sock.settimeout(5)
try:
sock.connect(("localhost", port))
rf = sock.makefile("rb", 65536)
Expand Down

0 comments on commit b838f35

Please sign in to comment.