Skip to content

Commit

Permalink
Update rdd pipe method for checkCode
Browse files Browse the repository at this point in the history
use boolean checkCode rather than more complicated mode optional argument.

Also add param to docstring
  • Loading branch information
megatron-me-uk committed Jul 7, 2015
1 parent ab9a2e1 commit 0c1e762
Showing 1 changed file with 4 additions and 13 deletions.
17 changes: 4 additions & 13 deletions python/pyspark/rdd.py
Original file line number Diff line number Diff line change
Expand Up @@ -687,24 +687,15 @@ def groupBy(self, f, numPartitions=None):
return self.map(lambda x: (f(x), x)).groupByKey(numPartitions)

@ignore_unicode_prefix
def pipe(self, command, env={}, mode='permissive'):
def pipe(self, command, env={}, checkCode=False):
"""
Return an RDD created by piping elements to a forked external process.
>>> sc.parallelize(['1', '2', '', '3']).pipe('cat').collect()
[u'1', u'2', u'', u'3']
:param checkCode: whether or not to check the return value of the shell command.
"""
if mode == 'permissive':
def fail_condition(x):
return False
elif mode == 'strict':
def fail_condition(x):
return x != 0
elif mode == 'grep':
def fail_condition(x):
return x != 0 and x != 1
else:
raise ValueError("mode must be one of 'permissive', 'strict' or 'grep'.")

def func(iterator):
pipe = Popen(
Expand All @@ -719,7 +710,7 @@ def pipe_objs(out):

def check_return_code():
pipe.wait()
if fail_condition(pipe.returncode):
if checkCode and pipe.returncode:
raise Exception("Pipe function `%s' exited "
"with error code %d" % (command, pipe.returncode))
else:
Expand Down

0 comments on commit 0c1e762

Please sign in to comment.