Skip to content

Commit

Permalink
Merge pull request #568 from Altinity/issue-486
Browse files Browse the repository at this point in the history 
Fix display of complex ( object, map ) types
  • Loading branch information
@Slach
Slach authored Jun 10, 2024
2 parents 27ba5d3 + 92ff735 commit 74dbeeb
Show file tree
Hide file tree
Showing 4 changed files with 317 additions and 1 deletion.
17 changes: 17 additions & 0 deletions docker/clickhouse/init_schema.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -195,4 +195,21 @@ ORDER BY EventTime;

INSERT INTO requests SELECT now()-INTERVAL 3 HOUR+INTERVAL number SECOND, 'os' || rand() % 9 AS OS, 'user' || rand() % 1000 AS UserName, randUniform(10,10000) AS req_count FROM numbers(10000);

CREATE DATABASE test;

CREATE TABLE test.map_table
(
`time` DateTime,
`id` String,
`attributes` Map(String, String)
)
ENGINE = MergeTree
PARTITION BY toYYYYMM(time)
PRIMARY KEY (toDate(time), id)
ORDER BY (toDate(time), id)
TTL toStartOfMonth(time) + toIntervalMonth(12 * 5)
SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1;

INSERT INTO test.map_table values (now(), 'id1', {'key1': 'value1', 'key2':'value2'})
INSERT INTO test.map_table values (now(), 'id2', {'key1': 'value1', 'key2':'value2'})
INSERT INTO test.map_table values (now(), 'id2', {'key1': 'value1', 'key2':'value2'})
260 changes: 260 additions & 0 deletions docker/grafana/dashboards/json_data_issues_486_189.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,260 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"description": "JSON data display issue #486 #189",
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"id": 40,
"links": [],
"panels": [
{
"datasource": {
"type": "vertamedia-clickhouse-datasource",
"uid": "P7E099F39B84EA795"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": [
{
"__systemRef": "hideSeriesFrom",
"matcher": {
"id": "byNames",
"options": {
"mode": "exclude",
"names": [
"id1, {\"key1\":\"value1\",\"key2\":\"value2\"}"
],
"prefix": "All except:",
"readOnly": true
}
},
"properties": [
{
"id": "custom.hideFrom",
"value": {
"legend": false,
"tooltip": false,
"viz": true
}
}
]
}
]
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 0
},
"id": 1,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"maxHeight": 600,
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"add_metadata": true,
"database": "test",
"datasource": {
"type": "vertamedia-clickhouse-datasource",
"uid": "P7E099F39B84EA795"
},
"dateTimeColDataType": "time",
"dateTimeType": "DATETIME",
"editorMode": "builder",
"extrapolate": true,
"format": "time_series",
"formattedQuery": "SELECT $timeSeries as t, count() FROM $table WHERE $timeFilter GROUP BY t ORDER BY t",
"interval": "",
"intervalFactor": 1,
"query": "SELECT\n $timeSeries as t,\n count(),\n id,\n attributes\nFROM $table\nWHERE $timeFilter\nGROUP BY\n t,\n id,\n attributes\nORDER BY t",
"rawQuery": "/* grafana dashboard=New dashboard, user=0 */\nSELECT\n (intDiv(toUInt32(time), 20) * 20) * 1000 as t,\n count(),\n id,\n attributes\nFROM test.map_table\nWHERE time >= toDateTime(1718003664) AND time <= toDateTime(1718025264)\nGROUP BY\n t,\n id,\n attributes\nORDER BY t",
"refId": "A",
"round": "0s",
"skip_comments": true,
"table": "map_table"
}
],
"title": "Issue #486 JSON data display",
"type": "timeseries"
},
{
"datasource": {
"type": "vertamedia-clickhouse-datasource",
"uid": "P7E099F39B84EA795"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"align": "auto",
"cellOptions": {
"type": "json-view"
},
"inspect": true
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 0
},
"id": 2,
"options": {
"cellHeight": "sm",
"footer": {
"countRows": false,
"fields": "",
"reducer": [
"sum"
],
"show": false
},
"showHeader": true,
"sortBy": [
{
"desc": true,
"displayName": "attributes"
}
]
},
"pluginVersion": "11.0.0",
"targets": [
{
"add_metadata": true,
"database": "test",
"datasource": {
"type": "vertamedia-clickhouse-datasource",
"uid": "P7E099F39B84EA795"
},
"dateTimeColDataType": "time",
"dateTimeType": "DATETIME",
"editorMode": "builder",
"extrapolate": true,
"format": "table",
"formattedQuery": "SELECT $timeSeries as t, count() FROM $table WHERE $timeFilter GROUP BY t ORDER BY t",
"interval": "",
"intervalFactor": 1,
"query": "SELECT\n $timeSeries as t,\n count(),\n id,\n attributes\nFROM $table\nWHERE $timeFilter\nGROUP BY\n t,\n id,\n attributes\nORDER BY t",
"rawQuery": "/* grafana dashboard=New dashboard, user=0 */\nSELECT\n (intDiv(toUInt32(time), 30) * 30) * 1000 as t,\n count(),\n id,\n attributes\nFROM test.map_table\nWHERE time >= toDateTime(1718003713) AND time <= toDateTime(1718025313)\nGROUP BY\n t,\n id,\n attributes\nORDER BY t",
"refId": "A",
"round": "0s",
"skip_comments": true,
"table": "map_table"
}
],
"title": "Issue #189 JSON data display",
"type": "table"
}
],
"schemaVersion": 39,
"tags": [],
"templating": {
"list": []
},
"time": {
"from": "now-6h",
"to": "now"
},
"timeRangeUpdatedDuringEditOrView": false,
"timepicker": {},
"timezone": "browser",
"title": "JSON data display issue #486 #189",
"uid": "ddod1z3iogbuof",
"version": 1,
"weekStart": ""
}
22 changes: 22 additions & 0 deletions pkg/parser.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -172,6 +172,26 @@ func parseStringValue(value interface{}, isNullable bool) Value {
}
}

// parseMapValue parses a map value to JSON.
func parseMapValue(value interface{}, isNullable bool) Value {
// Check if the value is a map
switch m := value.(type) {
case map[string]interface{}: // Check if it's a map with string keys and any type of value
jsonValue, err := json.Marshal(m)
if err != nil {
return nil
}
// Return the JSON bytes
return string(jsonValue)
default:
if isNullable {
return nil
} else {
return ""
}
}
}

func parseUInt64Value(value interface{}, isNullable bool) Value {
if value != nil {
ui64v, err := strconv.ParseUint(fmt.Sprintf("%v", value), 10, 64)
Expand Down Expand Up @@ -259,6 +279,8 @@ func ParseValue(fieldName string, fieldType string, tz *time.Location, value int
return ParseValue(fieldName, strings.TrimSuffix(strings.TrimPrefix(fieldType, "Nullable("), ")"), tz, value, true)
} else if strings.HasPrefix(fieldType, "LowCardinality") {
return ParseValue(fieldName, strings.TrimSuffix(strings.TrimPrefix(fieldType, "LowCardinality("), ")"), tz, value, isNullable)
} else if strings.HasPrefix(fieldType, "Map(") && strings.HasSuffix(fieldType, ")") {
return parseMapValue(value, isNullable)
} else {
switch fieldType {
case "String", "UUID", "IPv4", "IPv6":
Expand Down
19 changes: 18 additions & 1 deletion src/datasource/sql_series.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -289,9 +289,18 @@ export default class SqlSeries {
let t = SqlSeries._formatValue(row[timeCol.name]);
/* Build composite key (categories) from GROUP BY */
let metricKey: any = null;

if (keyColumns.length > 0) {
metricKey = keyColumns.map((name: string) => row[name]).join(', ');
metricKey = keyColumns.map((name: string) => {
const value = row[name];
if (typeof value === 'object') {
return JSON.stringify(value);
} else {
return String(value);
}
}).join(', ');
}

/* Make sure all series end with a value or nil for current timestamp
* to render discontinuous timeseries properly. */
if (lastTimeStamp < t) {
Expand Down Expand Up @@ -483,6 +492,10 @@ export default class SqlSeries {
return value;
}

if (typeof value === 'object') {
return JSON.stringify(value);
}

let numeric = Number(value);
if (isNaN(numeric)) {
return value;
Expand All @@ -496,6 +509,10 @@ export default class SqlSeries {
return value;
}

if (typeof value === 'object') {
return JSON.stringify(value);
}

let numeric = Number(value);
if (isNaN(numeric) || t !== 'number') {
return value;
Expand Down

0 comments on commit 74dbeeb

Please sign in to comment.