Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Converting to date, Cell conversion exception #187

Merged
merged 2 commits into from
Nov 23, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,10 @@ public fun AnyCol.convertTo(newType: KType): AnyCol {
public fun <T : Any> DataColumn<T>.convertToLocalDateTime(): DataColumn<LocalDateTime> = convertTo()
public fun <T : Any> DataColumn<T?>.convertToLocalDateTime(): DataColumn<LocalDateTime?> = convertTo()

@JvmName("convertToLocalDateFromT")
public fun <T : Any> DataColumn<T>.convertToLocalDate(): DataColumn<LocalDate> = convertTo()
public fun <T : Any> DataColumn<T?>.convertToLocalDate(): DataColumn<LocalDate?> = convertTo()

@JvmName("convertToLocalTimeFromT")
public fun <T : Any> DataColumn<T>.convertToLocalTime(): DataColumn<LocalTime> = convertTo()
public fun <T : Any> DataColumn<T?>.convertToLocalTime(): DataColumn<LocalTime?> = convertTo()
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
package org.jetbrains.kotlinx.dataframe.exceptions

import kotlin.reflect.KType

public class CellConversionException(
value: Any?,
from: KType,
to: KType,
public val column: String,
public val row: Int?,
override val cause: Throwable?
) : TypeConversionException(value, from, to) {
override val message: String
get() = "${super.message} in column $column, row $row"
}
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
package org.jetbrains.kotlinx.dataframe.exceptions

import kotlin.reflect.*
import kotlin.reflect.KType

public class TypeConversionException(public val value: Any?, public val from: KType, public val to: KType) : RuntimeException() {
public open class TypeConversionException(public val value: Any?, public val from: KType, public val to: KType) : RuntimeException() {

override val message: String
get() = "Failed to convert '$value' from $from to $to"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ import org.jetbrains.kotlinx.dataframe.api.to
import org.jetbrains.kotlinx.dataframe.columns.values
import org.jetbrains.kotlinx.dataframe.dataTypes.IFRAME
import org.jetbrains.kotlinx.dataframe.dataTypes.IMG
import org.jetbrains.kotlinx.dataframe.exceptions.CellConversionException
import org.jetbrains.kotlinx.dataframe.exceptions.TypeConversionException
import org.jetbrains.kotlinx.dataframe.exceptions.TypeConverterNotFoundException
import org.jetbrains.kotlinx.dataframe.impl.columns.DataColumnInternal
Expand Down Expand Up @@ -79,31 +80,50 @@ internal fun AnyCol.convertToTypeImpl(to: KType): AnyCol {
else -> throw TypeConversionException(null, from, to)
}

return when {
from == to -> this
from.isSubtypeOf(to) -> (this as DataColumnInternal<*>).changeType(to.withNullability(hasNulls()))
else -> when (val converter = getConverter(from, to, ParserOptions(locale = Locale.getDefault()))) {
null -> when (from.classifier) {
Any::class, Number::class, java.io.Serializable::class -> {
fun applyConverter(converter: TypeConverter): AnyCol {
var currentRow = 0
try {
val values = values.mapIndexed { row, value ->
currentRow = row
value?.let { converter(value) }.checkNulls()
}
return DataColumn.createValueColumn(name, values, to.withNullability(nullsFound))
} catch (e: TypeConversionException) {
throw CellConversionException(e.value, e.from, e.to, this.name(), currentRow, e)
}
}

fun convertPerCell(): AnyCol {
var currentRow = 0
try {
return when (from.classifier) {
Any::class, Comparable::class, Number::class, java.io.Serializable::class -> {
// find converter for every value
val values = values.map {
it?.let {
val values = values.mapIndexed { row, value ->
currentRow = row
value?.let {
val clazz = it.javaClass.kotlin
val type = clazz.createStarProjectedType(false)
val converter = getConverter(type, to, ParserOptions(locale = Locale.getDefault())) ?: throw TypeConverterNotFoundException(from, to)
val converter = getConverter(type, to, ParserOptions(locale = Locale.getDefault()))
?: throw TypeConverterNotFoundException(from, to)
converter(it)
}.checkNulls()
}
DataColumn.createValueColumn(name, values, to.withNullability(nullsFound))
}
else -> throw TypeConverterNotFoundException(from, to)
}
else -> {
val values = values.map {
it?.let { converter(it) }.checkNulls()
}
DataColumn.createValueColumn(name, values, to.withNullability(nullsFound))
}
} catch (e: TypeConversionException) {
throw CellConversionException(e.value, e.from, e.to, this.name(), currentRow, e)
}
}

return when {
from == to -> this
from.isSubtypeOf(to) -> (this as DataColumnInternal<*>).changeType(to.withNullability(hasNulls()))
else -> when (val converter = getConverter(from, to, ParserOptions(locale = Locale.getDefault()))) {
null -> convertPerCell()
else -> applyConverter(converter)
}
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import kotlinx.datetime.Instant
import org.jetbrains.kotlinx.dataframe.DataColumn
import org.jetbrains.kotlinx.dataframe.DataFrame
import org.jetbrains.kotlinx.dataframe.annotations.DataSchema
import org.jetbrains.kotlinx.dataframe.exceptions.CellConversionException
import org.jetbrains.kotlinx.dataframe.exceptions.TypeConversionException
import org.jetbrains.kotlinx.dataframe.exceptions.TypeConverterNotFoundException
import org.jetbrains.kotlinx.dataframe.hasNulls
Expand Down Expand Up @@ -80,6 +81,10 @@ class ConvertTests {
columnOf("a").convertTo<IntClass>()
}

shouldThrow<CellConversionException> {
columnOf("1", "10", "a").convertTo<IntClass>()
}.row shouldBe 2

shouldThrow<TypeConverterNotFoundException> {
columnOf(EnumClass.A).convertTo<IntClass>()
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,18 +3,27 @@ package org.jetbrains.kotlinx.dataframe.io
import io.kotest.assertions.throwables.shouldThrow
import io.kotest.matchers.shouldBe
import kotlinx.datetime.LocalDateTime
import kotlinx.datetime.TimeZone
import kotlinx.datetime.toKotlinLocalDate
import kotlinx.datetime.toKotlinLocalDateTime
import org.jetbrains.kotlinx.dataframe.DataColumn
import org.jetbrains.kotlinx.dataframe.DataFrame
import org.jetbrains.kotlinx.dataframe.api.cast
import org.jetbrains.kotlinx.dataframe.api.columnOf
import org.jetbrains.kotlinx.dataframe.api.convertTo
import org.jetbrains.kotlinx.dataframe.api.convertToDouble
import org.jetbrains.kotlinx.dataframe.api.convertToLocalDate
import org.jetbrains.kotlinx.dataframe.api.convertToLocalDateTime
import org.jetbrains.kotlinx.dataframe.api.convertToLocalTime
import org.jetbrains.kotlinx.dataframe.api.parse
import org.jetbrains.kotlinx.dataframe.api.parser
import org.jetbrains.kotlinx.dataframe.api.plus
import org.jetbrains.kotlinx.dataframe.api.times
import org.jetbrains.kotlinx.dataframe.api.tryParse
import org.jetbrains.kotlinx.dataframe.exceptions.TypeConversionException
import org.junit.Test
import java.math.BigDecimal
import java.time.LocalTime
import java.util.Locale
import kotlin.reflect.typeOf

Expand Down Expand Up @@ -75,6 +84,50 @@ class ParserTests {
)
}

@Test
fun `convert to date and time`() {
val daysToStandardMillis = 24 * 60 * 60 * 1000L
val longCol = columnOf(1L, 60L, 3600L).times(1000L).plus(daysToStandardMillis * 366)
val datetimeCol = longCol.convertToLocalDateTime(TimeZone.UTC)

datetimeCol.shouldBe(
columnOf(
java.time.LocalDateTime.of(1971, 1, 2, 0, 0, 1).toKotlinLocalDateTime(),
java.time.LocalDateTime.of(1971, 1, 2, 0, 1, 0).toKotlinLocalDateTime(),
java.time.LocalDateTime.of(1971, 1, 2, 1, 0, 0).toKotlinLocalDateTime()
)
)
longCol.convertToLocalDate(TimeZone.UTC).shouldBe(
columnOf(
java.time.LocalDate.of(1971, 1, 2).toKotlinLocalDate(),
java.time.LocalDate.of(1971, 1, 2).toKotlinLocalDate(),
java.time.LocalDate.of(1971, 1, 2).toKotlinLocalDate()
)
)
longCol.convertToLocalTime(TimeZone.UTC).shouldBe(
columnOf(
LocalTime.of(0, 0, 1),
LocalTime.of(0, 1, 0),
LocalTime.of(1, 0, 0)
)
)

datetimeCol.convertToLocalDate().shouldBe(
columnOf(
java.time.LocalDate.of(1971, 1, 2).toKotlinLocalDate(),
java.time.LocalDate.of(1971, 1, 2).toKotlinLocalDate(),
java.time.LocalDate.of(1971, 1, 2).toKotlinLocalDate()
)
)
datetimeCol.convertToLocalTime().shouldBe(
columnOf(
LocalTime.of(0, 0, 1),
LocalTime.of(0, 1, 0),
LocalTime.of(1, 0, 0)
)
)
}

@Test
fun `converting String to Double in different locales`() {
val currentLocale = Locale.getDefault()
Expand Down