Skip to content

Commit

Permalink
Update createDataFrame and toDF
Browse files Browse the repository at this point in the history
Refactored to use the new `structType` and `structField` functions.
  • Loading branch information
cafreeman authored and Davies Liu committed Apr 14, 2015
1 parent be5d5c1 commit 836c4bf
Showing 1 changed file with 8 additions and 8 deletions.
16 changes: 8 additions & 8 deletions R/pkg/R/SQLContext.R
Original file line number Diff line number Diff line change
Expand Up @@ -54,9 +54,9 @@ infer_type <- function(x) {
# StructType
types <- lapply(x, infer_type)
fields <- lapply(1:length(x), function(i) {
field(names[[i]], types[[i]], TRUE)
structField(names[[i]], types[[i]], TRUE)
})
do.call(buildSchema, fields)
do.call(structType, fields)
}
} else if (length(x) > 1) {
list(type = "array", elementType = type, containsNull = TRUE)
Expand Down Expand Up @@ -110,7 +110,7 @@ createDataFrame <- function(sqlCtx, data, schema = NULL, samplingRatio = 1.0) {
stop(paste("unexpected type:", class(data)))
}

if (is.null(schema) || (!inherits(schema, "struct") && is.null(names(schema)))) {
if (is.null(schema) || (!inherits(schema, "structType") && is.null(names(schema)))) {
row <- first(rdd)
names <- if (is.null(schema)) {
names(row)
Expand All @@ -135,18 +135,18 @@ createDataFrame <- function(sqlCtx, data, schema = NULL, samplingRatio = 1.0) {

types <- lapply(row, infer_type)
fields <- lapply(1:length(row), function(i) {
field(names[[i]], types[[i]], TRUE)
structField(names[[i]], types[[i]], TRUE)
})
schema <- do.call(buildSchema, fields)
schema <- do.call(structType, fields)
}

stopifnot(class(schema) == "struct")
schemaString <- tojson(schema)
stopifnot(class(schema) == "structType")
# schemaString <- tojson(schema)

jrdd <- getJRDD(lapply(rdd, function(x) x), "row")
srdd <- callJMethod(jrdd, "rdd")
sdf <- callJStatic("org.apache.spark.sql.api.r.SQLUtils", "createDF",
srdd, schemaString, sqlCtx)
srdd, schema$jobj, sqlCtx)
dataFrame(sdf)
}

Expand Down

0 comments on commit 836c4bf

Please sign in to comment.