Skip to content

Commit

Permalink
Remove unused pair UTF8 class. Add comments to msgpack deserializer
Browse files Browse the repository at this point in the history
  • Loading branch information
MLnick committed Dec 19, 2013
1 parent 4294cbb commit 0f5cd84
Showing 1 changed file with 2 additions and 22 deletions.
24 changes: 2 additions & 22 deletions python/pyspark/serializers.py
Original file line number Diff line number Diff line change
Expand Up @@ -261,6 +261,7 @@ class MarshalSerializer(FramedSerializer):
dumps = marshal.dumps
loads = marshal.loads


class MUTF8Deserializer(Serializer):
"""
Deserializes streams written by Java's DataOutputStream.writeUTF().
Expand All @@ -280,29 +281,9 @@ def load_stream(self, stream):
return


class PairMUTF8Deserializer(Serializer):
"""
Deserializes streams of tuples written by Java's DataOutputStream.writeUTF().
"""
def loads(self, stream):
l1 = struct.unpack('>H', stream.read(2))[0]
a = stream.read(l1).decode('utf8')
l2 = struct.unpack('>H', stream.read(2))[0]
b = stream.read(l2).decode('utf8')
return (a, b)

def load_stream(self, stream):
while True:
try:
yield self.loads(stream)
except struct.error:
return
except EOFError:
return

class MsgPackDeserializer(FramedSerializer):
"""
Deserializes streams written by Java MsgPack
"""
def loads(self, obj):
return msgpack.loads(obj, use_list=0)
Expand All @@ -311,7 +292,6 @@ def dumps(self, obj):
return msgpack.dumps(obj)



def read_long(stream):
length = stream.read(8)
if length == "":
Expand Down

0 comments on commit 0f5cd84

Please sign in to comment.