From 24ee7f7ce51e1285ebc6cb34a9b4fb0361db9d6d Mon Sep 17 00:00:00 2001 From: Lilith Wittmann Date: Fri, 18 Sep 2015 22:27:38 +0200 Subject: [PATCH] Convert numpy types to their native python types I don't know exactly why but for some reason the json encoder in python3 can't encode some of the numpy datetypes. Now I simply converted the numpy data types (http://docs.scipy.org/doc/numpy/user/basics.types.html) to the respective python types. --- airflow/utils.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/airflow/utils.py b/airflow/utils.py index f07ce5f53c16f..6fc7071ebe279 100644 --- a/airflow/utils.py +++ b/airflow/utils.py @@ -2,7 +2,7 @@ from builtins import str, input, object from past.builtins import basestring from copy import copy -from datetime import datetime, timedelta +from datetime import datetime, date, timedelta from dateutil.relativedelta import relativedelta # for doctest from email.mime.text import MIMEText from email.mime.multipart import MIMEMultipart @@ -29,6 +29,8 @@ from sqlalchemy import event, exc from sqlalchemy.pool import Pool +import numpy + from airflow import settings from airflow.configuration import conf @@ -619,9 +621,19 @@ def chain(*tasks): class AirflowJsonEncoder(json.JSONEncoder): def default(self, obj): + # convert dates and numpy objects in a json serializable format if isinstance(obj, datetime): return obj.strftime('%Y-%m-%dT%H:%M:%SZ') elif isinstance(obj, date): return obj.strftime('%Y-%m-%d') + elif type(obj) in [numpy.int_, numpy.intc, numpy.intp, numpy.int8, + numpy.int16, numpy.int32, numpy.int64, numpy.uint8, numpy.uint16, numpy.uint32, numpy.uint64]: + return int(obj) + elif type(obj) in [numpy.bool_]: + return bool(obj) + elif type(obj) in [numpy.float_, numpy.float16, numpy.float32, numpy.float64, + numpy.complex_, numpy.complex64, numpy.complex128,: + return float(obj) + # Let the base class default method raise the TypeError return json.JSONEncoder.default(self, obj)