Skip to content

Commit

Permalink
Convert numpy types to their native python types
Browse files Browse the repository at this point in the history
I don't know exactly why but for some reason the json encoder in python3 can't encode some of the numpy datetypes. Now I simply converted the numpy data types (http://docs.scipy.org/doc/numpy/user/basics.types.html) to the respective python types.
  • Loading branch information
LilithWittmann committed Sep 18, 2015
1 parent 1fb8f5b commit 24ee7f7
Showing 1 changed file with 13 additions and 1 deletion.
14 changes: 13 additions & 1 deletion airflow/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
from builtins import str, input, object
from past.builtins import basestring
from copy import copy
from datetime import datetime, timedelta
from datetime import datetime, date, timedelta
from dateutil.relativedelta import relativedelta # for doctest
from email.mime.text import MIMEText
from email.mime.multipart import MIMEMultipart
Expand All @@ -29,6 +29,8 @@
from sqlalchemy import event, exc
from sqlalchemy.pool import Pool

import numpy

from airflow import settings
from airflow.configuration import conf

Expand Down Expand Up @@ -619,9 +621,19 @@ def chain(*tasks):

class AirflowJsonEncoder(json.JSONEncoder):
def default(self, obj):
# convert dates and numpy objects in a json serializable format
if isinstance(obj, datetime):
return obj.strftime('%Y-%m-%dT%H:%M:%SZ')
elif isinstance(obj, date):
return obj.strftime('%Y-%m-%d')
elif type(obj) in [numpy.int_, numpy.intc, numpy.intp, numpy.int8,
numpy.int16, numpy.int32, numpy.int64, numpy.uint8, numpy.uint16, numpy.uint32, numpy.uint64]:
return int(obj)
elif type(obj) in [numpy.bool_]:
return bool(obj)
elif type(obj) in [numpy.float_, numpy.float16, numpy.float32, numpy.float64,
numpy.complex_, numpy.complex64, numpy.complex128,:
return float(obj)

# Let the base class default method raise the TypeError
return json.JSONEncoder.default(self, obj)

0 comments on commit 24ee7f7

Please sign in to comment.