From 1126d75cd0bf92a0e2dac7d9a8670ae144b2790e Mon Sep 17 00:00:00 2001 From: Dale Mcdiarmid Date: Fri, 21 Oct 2022 11:03:02 +0100 Subject: [PATCH 1/3] date Filter --- README.md | 3 ++- pkg/macros/macros.go | 13 +++++++++++++ pkg/macros/macros_test.go | 14 ++++++++++++++ pkg/plugin/driver.go | 1 + 4 files changed, 30 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 73f2ec6c..ebc9d9cf 100644 --- a/README.md +++ b/README.md @@ -119,8 +119,9 @@ WHERE $__timeFilter(date_time) ``` | Macro | Description | Output example | -| -------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------- | +|----------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------| | *$__timeFilter(columnName)* | Replaced by a conditional that filters the data (using the provided column) based on the time range of the panel in seconds | `time >= '1480001790' AND time <= '1482576232' )` | +| *$__dateFilter(columnName)* | Replaced by a conditional that filters the data (using the provided column) based on the date range of the panel | `date >= '2022-10-21' AND date <= '2022-10-23' )` | | *$__timeFilter_ms(columnName)* | Replaced by a conditional that filters the data (using the provided column) based on the time range of the panel in milliseconds | `time >= '1480001790671' AND time <= '1482576232479' )` | | *$__fromTime* | Replaced by the starting time of the range of the panel casted to DateTime | `toDateTime(intDiv(1415792726371,1000))` | | *$__toTime* | Replaced by the ending time of the range of the panel casted to DateTime | `toDateTime(intDiv(1415792726371,1000))` | diff --git a/pkg/macros/macros.go b/pkg/macros/macros.go index 909b407d..c24ca1e5 100644 --- a/pkg/macros/macros.go +++ b/pkg/macros/macros.go @@ -57,6 +57,19 @@ func TimeFilter(query *sqlds.Query, args []string) (string, error) { return fmt.Sprintf("%s >= '%d' AND %s <= '%d'", column, from, column, to), nil } +func DateFilter(query *sqlds.Query, args []string) (string, error) { + if len(args) != 1 { + return "", fmt.Errorf("%w: expected 1 argument, received %d", sqlds.ErrorBadArgumentCount, len(args)) + } + var ( + column = args[0] + from = query.TimeRange.From.Format("2006-01-02") + to = query.TimeRange.To.Format("2006-01-02") + ) + + return fmt.Sprintf("%s >= '%s' AND %s <= '%s'", column, from, column, to), nil +} + func TimeFilterMs(query *sqlds.Query, args []string) (string, error) { if len(args) != 1 { return "", fmt.Errorf("%w: expected 1 argument, received %d", sqlds.ErrorBadArgumentCount, len(args)) diff --git a/pkg/macros/macros_test.go b/pkg/macros/macros_test.go index d75c278b..e7a42011 100644 --- a/pkg/macros/macros_test.go +++ b/pkg/macros/macros_test.go @@ -74,6 +74,20 @@ func TestMacroToTimeFilter(t *testing.T) { } } +func TestMacroDateFilter(t *testing.T) { + from, _ := time.Parse("2006-01-02T15:04:05.000Z", "2014-11-12T11:45:26.371Z") + to, _ := time.Parse("2006-01-02T15:04:05.000Z", "2015-11-12T11:45:26.371Z") + query := sqlds.Query{ + TimeRange: backend.TimeRange{ + From: from, + To: to, + }, + } + got, err := macros.DateFilter(&query, []string{"dateCol"}) + assert.Nil(t, err) + assert.Equal(t, "dateCol >= '2014-11-12' AND dateCol <= '2015-11-12'", got) +} + func TestMacroTimeInterval(t *testing.T) { query := sqlds.Query{ RawSQL: "select $__timeInterval(col) from foo", diff --git a/pkg/plugin/driver.go b/pkg/plugin/driver.go index 8cfe2f18..bf262eeb 100644 --- a/pkg/plugin/driver.go +++ b/pkg/plugin/driver.go @@ -161,6 +161,7 @@ func (h *Clickhouse) Macros() sqlds.Macros { "toTime": macros.ToTimeFilter, "timeFilter_ms": macros.TimeFilterMs, "timeFilter": macros.TimeFilter, + "dateFilter": macros.DateFilter, "timeInterval": macros.TimeInterval, "interval_s": macros.IntervalSeconds, } From f29435d06bd151a99bf81c17ff6184ca7751b578 Mon Sep 17 00:00:00 2001 From: Dale Mcdiarmid Date: Tue, 21 Mar 2023 14:39:15 +0000 Subject: [PATCH 2/3] Support format option + traces --- pkg/converters/converters.go | 26 ++++++------ pkg/converters/converters_test.go | 28 +++++++++---- pkg/plugin/driver.go | 2 +- pkg/plugin/driver_test.go | 16 ++++++-- src/__mocks__/datasource.ts | 8 +++- src/components/FormatSelect.test.tsx | 11 ++++++ src/components/FormatSelect.tsx | 48 +++++++++++++++++++++++ src/components/QueryTypeSwitcher.test.tsx | 2 +- src/components/QueryTypeSwitcher.tsx | 5 ++- src/components/SQLEditor.test.tsx | 4 +- src/components/SQLEditor.tsx | 2 +- src/components/editor.ts | 29 ++++++++------ src/selectors.ts | 11 ++++++ src/styles.ts | 5 +++ src/types.ts | 6 +++ src/views/CHQueryEditor.test.tsx | 2 +- src/views/CHQueryEditor.tsx | 38 ++++++++++++++++-- 17 files changed, 192 insertions(+), 51 deletions(-) create mode 100644 src/components/FormatSelect.test.tsx create mode 100644 src/components/FormatSelect.tsx diff --git a/pkg/converters/converters.go b/pkg/converters/converters.go index f31a08e6..b01772e6 100644 --- a/pkg/converters/converters.go +++ b/pkg/converters/converters.go @@ -201,26 +201,26 @@ var Converters = map[string]Converter{ convert: decimalNullConvert, }, "Tuple()": { - fieldType: data.FieldTypeNullableString, + fieldType: data.FieldTypeNullableJSON, scanType: reflect.TypeOf((*interface{})(nil)).Elem(), matchRegex: tupleMatch, convert: jsonConverter, }, // NestedConverter currently only supports flatten_nested=0 only which can be marshalled into []map[string]interface{} "Nested()": { - fieldType: data.FieldTypeNullableString, + fieldType: data.FieldTypeNullableJSON, scanType: reflect.TypeOf([]map[string]interface{}{}), matchRegex: nestedMatch, convert: jsonConverter, }, "Array()": { - fieldType: data.FieldTypeNullableString, + fieldType: data.FieldTypeNullableJSON, scanType: reflect.TypeOf((*interface{})(nil)).Elem(), matchRegex: complexArrayMatch, convert: jsonConverter, }, "Map()": { - fieldType: data.FieldTypeNullableString, + fieldType: data.FieldTypeNullableJSON, scanType: reflect.TypeOf((*interface{})(nil)).Elem(), matchRegex: mapMatch, convert: jsonConverter, @@ -288,24 +288,20 @@ func createConverter(name string, converter Converter) sqlutil.Converter { } } -// MarshalJSON marshals the enum as a quoted json string -func marshalJSON(in interface{}) (string, error) { - jBytes, err := json.Marshal(in) - if err != nil { - return "", err - } - return string(jBytes), nil -} - func jsonConverter(in interface{}) (interface{}, error) { if in == nil { return (*string)(nil), nil } - json, err := marshalJSON(in) + jBytes, err := json.Marshal(in) + if err != nil { + return nil, err + } + var rawJSON json.RawMessage + err = json.Unmarshal(jBytes, &rawJSON) if err != nil { return nil, err } - return &json, nil + return &rawJSON, nil } func defaultConvert(in interface{}) (interface{}, error) { diff --git a/pkg/converters/converters_test.go b/pkg/converters/converters_test.go index 6db9659e..87062959 100644 --- a/pkg/converters/converters_test.go +++ b/pkg/converters/converters_test.go @@ -2,6 +2,7 @@ package converters_test import ( "encoding/json" + "errors" "github.com/grafana/clickhouse-datasource/pkg/converters" "github.com/shopspring/decimal" "github.com/stretchr/testify/assert" @@ -443,12 +444,17 @@ func TestNullableUInt256ShouldBeNil(t *testing.T) { assert.Equal(t, (*float64)(nil), actual) } -func toJson(obj interface{}) string { +func toJson(obj interface{}) (json.RawMessage, error) { bytes, err := json.Marshal(obj) if err != nil { - return "unable to marshal" + return nil, errors.New("unable to marshal") } - return string(bytes) + var rawJSON json.RawMessage + err = json.Unmarshal(bytes, &rawJSON) + if err != nil { + return nil, errors.New("unable to unmarshal") + } + return rawJSON, nil } func TestTuple(t *testing.T) { @@ -461,7 +467,9 @@ func TestTuple(t *testing.T) { sut := converters.GetConverter("Tuple(name String, id Uint16)") v, err := sut.FrameConverter.ConverterFunc(&value) assert.Nil(t, err) - assert.JSONEq(t, toJson(value), *v.(*string)) + msg, err := toJson(value) + assert.Nil(t, err) + assert.Equal(t, msg, *v.(*json.RawMessage)) } func TestNested(t *testing.T) { @@ -476,7 +484,9 @@ func TestNested(t *testing.T) { sut := converters.GetConverter("Nested(name String, id Uint16)") v, err := sut.FrameConverter.ConverterFunc(&value) assert.Nil(t, err) - assert.JSONEq(t, toJson(value), *v.(*string)) + msg, err := toJson(value) + assert.Nil(t, err) + assert.Equal(t, msg, *v.(*json.RawMessage)) } func TestMap(t *testing.T) { @@ -489,7 +499,9 @@ func TestMap(t *testing.T) { sut := converters.GetConverter("Map(String, Uint16)") v, err := sut.FrameConverter.ConverterFunc(&value) assert.Nil(t, err) - assert.JSONEq(t, toJson(value), *v.(*string)) + msg, err := toJson(value) + assert.Nil(t, err) + assert.Equal(t, msg, *v.(*json.RawMessage)) } func TestNullableFixedString(t *testing.T) { @@ -505,7 +517,9 @@ func TestArray(t *testing.T) { ipConverter := converters.GetConverter("Array(String)") v, err := ipConverter.FrameConverter.ConverterFunc(&value) assert.Nil(t, err) - assert.JSONEq(t, toJson(value), *v.(*string)) + msg, err := toJson(value) + assert.Nil(t, err) + assert.Equal(t, msg, *v.(*json.RawMessage)) } func TestIPv4(t *testing.T) { diff --git a/pkg/plugin/driver.go b/pkg/plugin/driver.go index c57b8d4e..4cce3dc9 100644 --- a/pkg/plugin/driver.go +++ b/pkg/plugin/driver.go @@ -216,6 +216,7 @@ func (h *Clickhouse) MutateQuery(ctx context.Context, req backend.DataQuery) (co Meta struct { TimeZone string `json:"timezone"` } `json:"meta"` + Format int `json:"format"` } if err := json.Unmarshal(req.JSON, &dataQuery); err != nil { @@ -227,6 +228,5 @@ func (h *Clickhouse) MutateQuery(ctx context.Context, req backend.DataQuery) (co } loc, _ := time.LoadLocation(dataQuery.Meta.TimeZone) - return clickhouse.Context(ctx, clickhouse.WithUserLocation(loc)), req } diff --git a/pkg/plugin/driver_test.go b/pkg/plugin/driver_test.go index 790f04c0..e6035369 100644 --- a/pkg/plugin/driver_test.go +++ b/pkg/plugin/driver_test.go @@ -5,6 +5,7 @@ import ( "crypto/tls" "database/sql" "encoding/json" + "errors" "fmt" "math/big" "net" @@ -209,12 +210,17 @@ func insertData(t *testing.T, conn *sql.DB, data ...interface{}) { require.NoError(t, scope.Commit()) } -func toJson(obj interface{}) string { +func toJson(obj interface{}) (json.RawMessage, error) { bytes, err := json.Marshal(obj) if err != nil { - return "unable to marshal" + return nil, errors.New("unable to marshal") } - return string(bytes) + var rawJSON json.RawMessage + json.Unmarshal(bytes, &rawJSON) + if err != nil { + return nil, errors.New("unable to unmarshal") + } + return rawJSON, nil } func checkFieldValue(t *testing.T, field *data.Field, expected ...interface{}) { @@ -230,7 +236,9 @@ func checkFieldValue(t *testing.T, field *data.Field, expected ...interface{}) { default: switch reflect.ValueOf(eVal).Kind() { case reflect.Map, reflect.Slice: - assert.JSONEq(t, toJson(tVal), *val.(*string)) + jsonRaw, err := toJson(tVal) + assert.Nil(t, err) + assert.Equal(t, jsonRaw, *val.(*json.RawMessage)) return } assert.Equal(t, eVal, val) diff --git a/src/__mocks__/datasource.ts b/src/__mocks__/datasource.ts index 309cdb65..a2da55c8 100644 --- a/src/__mocks__/datasource.ts +++ b/src/__mocks__/datasource.ts @@ -38,4 +38,10 @@ export const mockDatasource = new Datasource({ }, }); mockDatasource.adHocFiltersStatus = 1; // most tests should skip checking the CH version. We will set ad hoc filters to enabled to avoid running the CH version check -export const mockQuery: CHQuery = { rawSql: 'select * from foo', refId: '', format: 1, queryType: QueryType.SQL }; +export const mockQuery: CHQuery = { + rawSql: 'select * from foo', + refId: '', + format: 1, + queryType: QueryType.SQL, + selectedFormat: 4, +}; diff --git a/src/components/FormatSelect.test.tsx b/src/components/FormatSelect.test.tsx new file mode 100644 index 00000000..b12c23f9 --- /dev/null +++ b/src/components/FormatSelect.test.tsx @@ -0,0 +1,11 @@ +import React from 'react'; +import { render } from '@testing-library/react'; +import { FormatSelect } from './FormatSelect'; +import { Format } from '../types'; + +describe('FormatSelect', () => { + it('renders a format', () => { + const result = render( {}} />); + expect(result.container.firstChild).not.toBeNull(); + }); +}); diff --git a/src/components/FormatSelect.tsx b/src/components/FormatSelect.tsx new file mode 100644 index 00000000..c9e65ff1 --- /dev/null +++ b/src/components/FormatSelect.tsx @@ -0,0 +1,48 @@ +import React from 'react'; +import { InlineFormLabel, Select } from '@grafana/ui'; +import { selectors } from './../selectors'; +import { Format } from '../types'; +import { styles } from '../styles'; + +export type Props = { format: Format; value?: string; onChange: (format: Format) => void }; + +export const FormatSelect = (props: Props) => { + const { onChange, format } = props; + const { label, tooltip, options: formatLabels } = selectors.components.QueryEditor.Format; + return ( +
+ + {label} + + + className={`width-8 ${styles.Common.inlineSelect}`} + onChange={(e) => onChange(e.value!)} + options={[ + { + label: formatLabels.AUTO, + value: Format.AUTO, + }, + { + label: formatLabels.TABLE, + value: Format.TABLE, + }, + { + label: formatLabels.TIME_SERIES, + value: Format.TIMESERIES, + }, + { + label: formatLabels.LOGS, + value: Format.LOGS, + }, + { + label: formatLabels.TRACE, + value: Format.TRACE, + }, + ]} + value={format} + menuPlacement={'bottom'} + allowCustomValue={false} + /> +
+ ); +}; diff --git a/src/components/QueryTypeSwitcher.test.tsx b/src/components/QueryTypeSwitcher.test.tsx index 5cda0f6a..9c14d5f0 100644 --- a/src/components/QueryTypeSwitcher.test.tsx +++ b/src/components/QueryTypeSwitcher.test.tsx @@ -30,7 +30,7 @@ describe('QueryTypeSwitcher', () => { it('renders correctly SQL editor', () => { const result = render( {}} onRunQuery={() => {}} /> diff --git a/src/components/QueryTypeSwitcher.tsx b/src/components/QueryTypeSwitcher.tsx index a0f08abc..79640f84 100644 --- a/src/components/QueryTypeSwitcher.tsx +++ b/src/components/QueryTypeSwitcher.tsx @@ -3,7 +3,7 @@ import { SelectableValue } from '@grafana/data'; import { RadioButtonGroup, ConfirmModal, InlineFormLabel } from '@grafana/ui'; import { getQueryOptionsFromSql, getSQLFromQueryOptions } from './queryBuilder/utils'; import { selectors } from './../selectors'; -import { CHQuery, QueryType, defaultCHBuilderQuery, SqlBuilderOptions, CHSQLQuery, Format } from 'types'; +import { CHQuery, QueryType, defaultCHBuilderQuery, SqlBuilderOptions, CHSQLQuery } from 'types'; import { isString } from 'lodash'; interface QueryTypeSwitcherProps { @@ -56,7 +56,8 @@ export const QueryTypeSwitcher = (props: QueryTypeSwitcherProps) => { queryType, rawSql: getSQLFromQueryOptions(builderOptions), meta: { builderOptions }, - format: Format.TABLE, + format: query.format, + selectedFormat: query.selectedFormat, }); } else if (queryType === QueryType.Builder) { onChange({ ...query, queryType, rawSql: getSQLFromQueryOptions(builderOptions), builderOptions }); diff --git a/src/components/SQLEditor.test.tsx b/src/components/SQLEditor.test.tsx index de70ea3a..68b92b85 100644 --- a/src/components/SQLEditor.test.tsx +++ b/src/components/SQLEditor.test.tsx @@ -27,7 +27,7 @@ describe('SQL Editor', () => { const rawSql = 'foo'; render( { await act(async () => { render( { }); const onSqlChange = (sql: string) => { - const format = getFormat(sql); + const format = getFormat(sql, query.selectedFormat); onChange({ ...query, rawSql: sql, format, queryType: QueryType.SQL }); onRunQuery(); }; diff --git a/src/components/editor.ts b/src/components/editor.ts index 12154c5d..54642590 100644 --- a/src/components/editor.ts +++ b/src/components/editor.ts @@ -2,19 +2,22 @@ import { getFields } from 'data/ast'; import { Format } from 'types'; import { isString } from 'lodash'; -export const getFormat = (sql: string): Format => { - // convention to format as time series - // first field as "time" alias and requires at least 2 fields (time and metric) - const selectList = getFields(sql); - // if there are more than 2 fields, index 1 will be a ',' - if (selectList.length > 2 && isString(selectList[0])) { - const firstProjection = selectList[0].trim().toLowerCase(); - if (firstProjection.endsWith('as time')) { - return Format.TIMESERIES; - } - if (firstProjection.endsWith('as log_time')) { - return Format.LOGS; +export const getFormat = (sql: string, selectedFormat: Format): Format => { + if (selectedFormat === Format.AUTO) { + // convention to format as time series + // first field as "time" alias and requires at least 2 fields (time and metric) + const selectList = getFields(sql); + // if there are more than 2 fields, index 1 will be a ',' + if (selectList.length > 2 && isString(selectList[0])) { + const firstProjection = selectList[0].trim().toLowerCase(); + if (firstProjection.endsWith('as time')) { + return Format.TIMESERIES; + } + if (firstProjection.endsWith('as log_time')) { + return Format.LOGS; + } } + return Format.TABLE; } - return Format.TABLE; + return selectedFormat; }; diff --git a/src/selectors.ts b/src/selectors.ts index 1d7ec336..475060bb 100644 --- a/src/selectors.ts +++ b/src/selectors.ts @@ -79,6 +79,17 @@ export const Components = { container: 'data-testid-code-editor-container', Expand: 'data-testid-code-editor-expand-button', }, + Format: { + label: 'Format', + tooltip: 'Query Type', + options: { + AUTO: 'Auto', + TABLE: 'Table', + TIME_SERIES: 'Time Series', + LOGS: 'Logs', + TRACE: 'Trace', + }, + }, Types: { label: 'Query Type', tooltip: 'Query Type', diff --git a/src/styles.ts b/src/styles.ts index 4b0c75d9..032eb7ce 100644 --- a/src/styles.ts +++ b/src/styles.ts @@ -56,5 +56,10 @@ export const styles = { } `, }, + FormatSelector: { + formatSelector: css` + display: flex; + `, + }, VariablesEditor: {}, }; diff --git a/src/types.ts b/src/types.ts index edb1c520..80db2744 100644 --- a/src/types.ts +++ b/src/types.ts @@ -33,6 +33,8 @@ export enum Format { TIMESERIES = 0, TABLE = 1, LOGS = 2, + TRACE = 3, + AUTO = 4, } //#region Query @@ -50,6 +52,7 @@ export interface CHSQLQuery extends DataQuery { builderOptions?: SqlBuilderOptions; }; format: Format; + selectedFormat: Format; expand?: boolean; } @@ -58,6 +61,7 @@ export interface CHBuilderQuery extends DataQuery { rawSql: string; builderOptions: SqlBuilderOptions; format: Format; + selectedFormat: Format; meta?: { timezone?: string; }; @@ -240,11 +244,13 @@ export const defaultCHBuilderQuery: Omit = { limit: 100, }, format: Format.TABLE, + selectedFormat: Format.AUTO, }; export const defaultCHSQLQuery: Omit = { queryType: QueryType.SQL, rawSql: '', format: Format.TABLE, + selectedFormat: Format.AUTO, expand: false, }; //#endregion diff --git a/src/views/CHQueryEditor.test.tsx b/src/views/CHQueryEditor.test.tsx index 2520c646..1b08b69e 100644 --- a/src/views/CHQueryEditor.test.tsx +++ b/src/views/CHQueryEditor.test.tsx @@ -25,7 +25,7 @@ describe('Query Editor', () => { const rawSql = 'foo'; render( { const { query, onChange } = props; const onBuilderOptionsChange = (builderOptions: SqlBuilderOptions) => { const sql = getSQLFromQueryOptions(builderOptions); - const format = builderOptions.mode === BuilderMode.Trend ? Format.TIMESERIES : Format.TABLE; + const format = + query.selectedFormat === Format.AUTO + ? builderOptions.mode === BuilderMode.Trend + ? Format.TIMESERIES + : Format.TABLE + : query.selectedFormat; onChange({ ...query, queryType: QueryType.Builder, rawSql: sql, builderOptions, format }); }; @@ -60,7 +75,7 @@ export const CHQueryEditor = (props: CHQueryEditorProps) => { const runQuery = () => { if (query.queryType === QueryType.SQL) { - const format = getFormat(query.rawSql); + const format = getFormat(query.rawSql, query.selectedFormat); if (format !== query.format) { onChange({ ...query, format }); } @@ -68,6 +83,22 @@ export const CHQueryEditor = (props: CHQueryEditorProps) => { onRunQuery(); }; + const onFormatChange = (selectedFormat: Format) => { + switch (query.queryType) { + case QueryType.SQL: + onChange({ ...query, format: getFormat(query.rawSql, selectedFormat), selectedFormat }); + case QueryType.Builder: + default: + if (selectedFormat === Format.AUTO) { + let builderOptions = (query as CHBuilderQuery).builderOptions; + const format = builderOptions && builderOptions.mode === BuilderMode.Trend ? Format.TIMESERIES : Format.TABLE; + onChange({ ...query, format, selectedFormat }); + } else { + onChange({ ...query, format: selectedFormat, selectedFormat }); + } + } + }; + return ( <>
@@ -76,6 +107,7 @@ export const CHQueryEditor = (props: CHQueryEditorProps) => {
+ ); From a845fba7898d784b994da5191864f283f538592b Mon Sep 17 00:00:00 2001 From: Dale Mcdiarmid Date: Tue, 21 Mar 2023 16:06:37 +0000 Subject: [PATCH 3/3] Fix go lint --- pkg/plugin/driver_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/plugin/driver_test.go b/pkg/plugin/driver_test.go index e6035369..c3d7f763 100644 --- a/pkg/plugin/driver_test.go +++ b/pkg/plugin/driver_test.go @@ -216,7 +216,7 @@ func toJson(obj interface{}) (json.RawMessage, error) { return nil, errors.New("unable to marshal") } var rawJSON json.RawMessage - json.Unmarshal(bytes, &rawJSON) + err = json.Unmarshal(bytes, &rawJSON) if err != nil { return nil, errors.New("unable to unmarshal") }