From b46d74b13bbfc0290cf9c6ce13b309d50d10954d Mon Sep 17 00:00:00 2001 From: Cristine Guadelupe Date: Sun, 3 Dec 2023 19:39:45 -0300 Subject: [PATCH] Simplifies floats and integers internally (#120) --- lib/kino_explorer/data_transform_cell.ex | 171 +++-------------------- 1 file changed, 21 insertions(+), 150 deletions(-) diff --git a/lib/kino_explorer/data_transform_cell.ex b/lib/kino_explorer/data_transform_cell.ex index c6c5190..eb03072 100644 --- a/lib/kino_explorer/data_transform_cell.ex +++ b/lib/kino_explorer/data_transform_cell.ex @@ -17,19 +17,9 @@ defmodule KinoExplorer.DataTransformCell do "datetime[μs]", "datetime[ns]", "float", - "f[32]", - "f[64]", "integer", "string", - "s[8]", - "s[16]", - "s[32]", - "s[64]", - "time", - "u[8]", - "u[16]", - "u[32]", - "u[64]" + "time" ] @filter_options %{ "binary" => ["equal", "contains", "not contains", "not equal"], @@ -40,19 +30,9 @@ defmodule KinoExplorer.DataTransformCell do "datetime[μs]" => ["less", "less equal", "equal", "not equal", "greater equal", "greater"], "datetime[ns]" => ["less", "less equal", "equal", "not equal", "greater equal", "greater"], "float" => ["less", "less equal", "equal", "not equal", "greater equal", "greater"], - "f[32]" => ["less", "less equal", "equal", "not equal", "greater equal", "greater"], - "f[64]" => ["less", "less equal", "equal", "not equal", "greater equal", "greater"], "integer" => ["less", "less equal", "equal", "not equal", "greater equal", "greater"], "string" => ["equal", "contains", "not contains", "not equal"], - "s8" => ["less", "less equal", "equal", "not equal", "greater equal", "greater"], - "s16" => ["less", "less equal", "equal", "not equal", "greater equal", "greater"], - "s32" => ["less", "less equal", "equal", "not equal", "greater equal", "greater"], - "s64" => ["less", "less equal", "equal", "not equal", "greater equal", "greater"], - "time" => ["less", "less equal", "equal", "not equal", "greater equal", "greater"], - "u8" => ["less", "less equal", "equal", "not equal", "greater equal", "greater"], - "u16" => ["less", "less equal", "equal", "not equal", "greater equal", "greater"], - "u32" => ["less", "less equal", "equal", "not equal", "greater equal", "greater"], - "u64" => ["less", "less equal", "equal", "not equal", "greater equal", "greater"] + "time" => ["less", "less equal", "equal", "not equal", "greater equal", "greater"] } @fill_missing_options %{ "binary" => ["forward", "backward", "max", "min", "scalar"], @@ -63,19 +43,9 @@ defmodule KinoExplorer.DataTransformCell do "datetime[μs]" => ["forward", "backward", "max", "min", "mean", "scalar"], "datetime[ns]" => ["forward", "backward", "max", "min", "mean", "scalar"], "float" => ["forward", "backward", "max", "min", "mean", "scalar", "nan"], - "f[32]" => ["forward", "backward", "max", "min", "mean", "scalar", "nan"], - "f[64]" => ["forward", "backward", "max", "min", "mean", "scalar", "nan"], "integer" => ["forward", "backward", "max", "min", "mean", "scalar"], "string" => ["forward", "backward", "max", "min", "scalar"], - "s8" => ["forward", "backward", "max", "min", "mean", "scalar"], - "s16" => ["forward", "backward", "max", "min", "mean", "scalar"], - "s32" => ["forward", "backward", "max", "min", "mean", "scalar"], - "s64" => ["forward", "backward", "max", "min", "mean", "scalar"], - "time" => ["forward", "backward", "max", "min", "mean", "scalar"], - "u8" => ["forward", "backward", "max", "min", "mean", "scalar"], - "u16" => ["forward", "backward", "max", "min", "mean", "scalar"], - "u32" => ["forward", "backward", "max", "min", "mean", "scalar"], - "u64" => ["forward", "backward", "max", "min", "mean", "scalar"] + "time" => ["forward", "backward", "max", "min", "mean", "scalar"] } @summarise_options %{ count: @column_types, @@ -85,111 +55,25 @@ defmodule KinoExplorer.DataTransformCell do "datetime[μs]", "datetime[ns]", "float", - "f[32]", - "f[64]", "integer", - "s8", - "s16", - "s32", - "s64", - "time", - "u8", - "u16", - "u32", - "u64" - ], - mean: [ - "float", - "f[32]", - "f[64]", - "integer", - "s8", - "s16", - "s32", - "s64", - "u8", - "u16", - "u32", - "u64" - ], - median: [ - "float", - "f[32]", - "f[64]", - "integer", - "s8", - "s16", - "s32", - "s64", - "u8", - "u16", - "u32", - "u64" + "time" ], + mean: ["float", "integer"], + median: ["float", "integer"], min: [ "date", "datetime[ms]", "datetime[μs]", "datetime[ns]", "float", - "f[32]", - "f[64]", "integer", - "s8", - "s16", - "s32", - "s64", - "time", - "u8", - "u16", - "u32", - "u64" + "time" ], n_distinct: @column_types, nil_count: @column_types, - standard_deviation: [ - "float", - "f[32]", - "f[64]", - "integer", - "s8", - "s16", - "s32", - "s64", - "u8", - "u16", - "u32", - "u64" - ], - sum: [ - "boolean", - "float", - "f[32]", - "f[64]", - "integer", - "s8", - "s16", - "s32", - "s64", - "u8", - "u16", - "u32", - "u64" - ], - variance: [ - "float", - "f[32]", - "f[64]", - "integer", - "s8", - "s16", - "s32", - "s64", - "u8", - "u16", - "u32", - "u64" - ] + standard_deviation: ["float", "integer"], + sum: ["boolean", "float", "integer"], + variance: ["float", "integer"] } @pivot_wider_types %{ names_from: @column_types, @@ -207,20 +91,7 @@ defmodule KinoExplorer.DataTransformCell do "quantile(80)", "quantile(90)" ] - @queried_filter_types [ - "float", - "f[32]", - "f[64]", - "integer", - "s8", - "s16", - "s32", - "s64", - "u8", - "u16", - "u32", - "u64" - ] + @queried_filter_types ["float", "integer"] @grouped_fields_operations ["filters", "fill_missing", "summarise"] @validation_by_type [:filters, :fill_missing] @@ -1083,16 +954,16 @@ defmodule KinoExplorer.DataTransformCell do {k, {:datetime, :millisecond}} -> {k, "datetime[ms]"} {k, {:datetime, :microsecond}} -> {k, "datetime[μs]"} {k, {:datetime, :nanosecond}} -> {k, "datetime[ns]"} - {k, {:f, 32}} -> {k, "f[32]"} - {k, {:f, 64}} -> {k, "f[64]"} - {k, {:s, 8}} -> {k, "s[8]"} - {k, {:s, 16}} -> {k, "s[16]"} - {k, {:s, 32}} -> {k, "s[32]"} - {k, {:s, 64}} -> {k, "s[64]"} - {k, {:u, 8}} -> {k, "u[8]"} - {k, {:u, 16}} -> {k, "u[16]"} - {k, {:u, 32}} -> {k, "u[32]"} - {k, {:u, 64}} -> {k, "u[64]"} + {k, {:f, 32}} -> {k, "float"} + {k, {:f, 64}} -> {k, "float"} + {k, {:s, 8}} -> {k, "integer"} + {k, {:s, 16}} -> {k, "integer"} + {k, {:s, 32}} -> {k, "integer"} + {k, {:s, 64}} -> {k, "integer"} + {k, {:u, 8}} -> {k, "integer"} + {k, {:u, 16}} -> {k, "integer"} + {k, {:u, 32}} -> {k, "integer"} + {k, {:u, 64}} -> {k, "integer"} {k, v} -> {k, Atom.to_string(v)} end) |> Enum.into(%{})