diff --git a/posthog/api/test/__snapshots__/test_query.ambr b/posthog/api/test/__snapshots__/test_query.ambr index 67f26e430fcc5..614472b87ab07 100644 --- a/posthog/api/test/__snapshots__/test_query.ambr +++ b/posthog/api/test/__snapshots__/test_query.ambr @@ -10,7 +10,8 @@ WHERE and(equals(events.team_id, 66), less(events.timestamp, '2020-01-10 12:14:05.000000'), greater(events.timestamp, '2020-01-09 12:00:00.000000')) ORDER BY events.event ASC LIMIT 101 - OFFSET 0 + OFFSET 0 SETTINGS readonly=1, + max_execution_time=60 ' --- # name: TestQuery.test_event_property_filter.1 @@ -25,7 +26,8 @@ WHERE and(equals(events.team_id, 66), equals(replaceRegexpAll(JSONExtractRaw(events.properties, 'key'), '^"|"$', ''), 'test_val3'), less(events.timestamp, '2020-01-10 12:14:05.000000'), greater(events.timestamp, '2020-01-09 12:00:00.000000')) ORDER BY events.event ASC LIMIT 101 - OFFSET 0 + OFFSET 0 SETTINGS readonly=1, + max_execution_time=60 ' --- # name: TestQuery.test_event_property_filter.2 @@ -40,7 +42,8 @@ WHERE and(equals(events.team_id, 66), ilike(replaceRegexpAll(JSONExtractRaw(events.properties, 'path'), '^"|"$', ''), '%/%'), less(events.timestamp, '2020-01-10 12:14:05.000000'), greater(events.timestamp, '2020-01-09 12:00:00.000000')) ORDER BY events.event ASC LIMIT 101 - OFFSET 0 + OFFSET 0 SETTINGS readonly=1, + max_execution_time=60 ' --- # name: TestQuery.test_event_property_filter_materialized @@ -55,7 +58,8 @@ WHERE and(equals(events.team_id, 67), less(events.timestamp, '2020-01-10 12:14:05.000000'), greater(events.timestamp, '2020-01-09 12:00:00.000000')) ORDER BY events.event ASC LIMIT 101 - OFFSET 0 + OFFSET 0 SETTINGS readonly=1, + max_execution_time=60 ' --- # name: TestQuery.test_event_property_filter_materialized.1 @@ -70,7 +74,8 @@ WHERE and(equals(events.team_id, 67), equals(events.mat_key, 'test_val3'), less(events.timestamp, '2020-01-10 12:14:05.000000'), greater(events.timestamp, '2020-01-09 12:00:00.000000')) ORDER BY events.event ASC LIMIT 101 - OFFSET 0 + OFFSET 0 SETTINGS readonly=1, + max_execution_time=60 ' --- # name: TestQuery.test_event_property_filter_materialized.2 @@ -85,7 +90,8 @@ WHERE and(equals(events.team_id, 67), ilike(events.mat_path, '%/%'), less(events.timestamp, '2020-01-10 12:14:05.000000'), greater(events.timestamp, '2020-01-09 12:00:00.000000')) ORDER BY events.event ASC LIMIT 101 - OFFSET 0 + OFFSET 0 SETTINGS readonly=1, + max_execution_time=60 ' --- # name: TestQuery.test_full_hogql_query @@ -97,7 +103,8 @@ FROM events WHERE equals(events.team_id, 68) ORDER BY events.timestamp ASC - LIMIT 100 + LIMIT 100 SETTINGS readonly=1, + max_execution_time=60 ' --- # name: TestQuery.test_full_hogql_query_materialized @@ -109,7 +116,8 @@ FROM events WHERE equals(events.team_id, 69) ORDER BY events.timestamp ASC - LIMIT 100 + LIMIT 100 SETTINGS readonly=1, + max_execution_time=60 ' --- # name: TestQuery.test_hogql_property_filter @@ -124,7 +132,8 @@ WHERE and(equals(events.team_id, 70), less(events.timestamp, '2020-01-10 12:14:05.000000'), greater(events.timestamp, '2020-01-09 12:00:00.000000')) ORDER BY events.event ASC LIMIT 101 - OFFSET 0 + OFFSET 0 SETTINGS readonly=1, + max_execution_time=60 ' --- # name: TestQuery.test_hogql_property_filter.1 @@ -139,7 +148,8 @@ WHERE and(equals(events.team_id, 70), equals('a%sd', 'foo'), less(events.timestamp, '2020-01-10 12:14:05.000000'), greater(events.timestamp, '2020-01-09 12:00:00.000000')) ORDER BY events.event ASC LIMIT 101 - OFFSET 0 + OFFSET 0 SETTINGS readonly=1, + max_execution_time=60 ' --- # name: TestQuery.test_hogql_property_filter.2 @@ -154,7 +164,8 @@ WHERE and(equals(events.team_id, 70), equals('a%sd', 'a%sd'), less(events.timestamp, '2020-01-10 12:14:05.000000'), greater(events.timestamp, '2020-01-09 12:00:00.000000')) ORDER BY events.event ASC LIMIT 101 - OFFSET 0 + OFFSET 0 SETTINGS readonly=1, + max_execution_time=60 ' --- # name: TestQuery.test_hogql_property_filter.3 @@ -169,7 +180,8 @@ WHERE and(equals(events.team_id, 70), equals(replaceRegexpAll(JSONExtractRaw(events.properties, 'key'), '^"|"$', ''), 'test_val2'), less(events.timestamp, '2020-01-10 12:14:05.000000'), greater(events.timestamp, '2020-01-09 12:00:00.000000')) ORDER BY events.event ASC LIMIT 101 - OFFSET 0 + OFFSET 0 SETTINGS readonly=1, + max_execution_time=60 ' --- # name: TestQuery.test_hogql_property_filter_materialized @@ -184,7 +196,8 @@ WHERE and(equals(events.team_id, 71), less(events.timestamp, '2020-01-10 12:14:05.000000'), greater(events.timestamp, '2020-01-09 12:00:00.000000')) ORDER BY events.event ASC LIMIT 101 - OFFSET 0 + OFFSET 0 SETTINGS readonly=1, + max_execution_time=60 ' --- # name: TestQuery.test_hogql_property_filter_materialized.1 @@ -199,7 +212,8 @@ WHERE and(equals(events.team_id, 71), equals('a%sd', 'foo'), less(events.timestamp, '2020-01-10 12:14:05.000000'), greater(events.timestamp, '2020-01-09 12:00:00.000000')) ORDER BY events.event ASC LIMIT 101 - OFFSET 0 + OFFSET 0 SETTINGS readonly=1, + max_execution_time=60 ' --- # name: TestQuery.test_hogql_property_filter_materialized.2 @@ -214,7 +228,8 @@ WHERE and(equals(events.team_id, 71), equals('a%sd', 'a%sd'), less(events.timestamp, '2020-01-10 12:14:05.000000'), greater(events.timestamp, '2020-01-09 12:00:00.000000')) ORDER BY events.event ASC LIMIT 101 - OFFSET 0 + OFFSET 0 SETTINGS readonly=1, + max_execution_time=60 ' --- # name: TestQuery.test_hogql_property_filter_materialized.3 @@ -229,7 +244,8 @@ WHERE and(equals(events.team_id, 71), equals(events.mat_key, 'test_val2'), less(events.timestamp, '2020-01-10 12:14:05.000000'), greater(events.timestamp, '2020-01-09 12:00:00.000000')) ORDER BY events.event ASC LIMIT 101 - OFFSET 0 + OFFSET 0 SETTINGS readonly=1, + max_execution_time=60 ' --- # name: TestQuery.test_person_property_filter @@ -258,7 +274,8 @@ WHERE and(equals(events.team_id, 75), equals(events__pdi__person.properties___email, 'tom@posthog.com'), less(events.timestamp, '2020-01-10 12:14:05.000000'), greater(events.timestamp, '2020-01-09 12:00:00.000000')) ORDER BY events.event ASC LIMIT 101 - OFFSET 0 + OFFSET 0 SETTINGS readonly=1, + max_execution_time=60 ' --- # name: TestQuery.test_person_property_filter_materialized @@ -287,7 +304,8 @@ WHERE and(equals(events.team_id, 76), equals(events__pdi__person.properties___email, 'tom@posthog.com'), less(events.timestamp, '2020-01-10 12:14:05.000000'), greater(events.timestamp, '2020-01-09 12:00:00.000000')) ORDER BY events.event ASC LIMIT 101 - OFFSET 0 + OFFSET 0 SETTINGS readonly=1, + max_execution_time=60 ' --- # name: TestQuery.test_property_filter_aggregations @@ -300,7 +318,8 @@ GROUP BY replaceRegexpAll(JSONExtractRaw(events.properties, 'key'), '^"|"$', '') ORDER BY count() DESC LIMIT 101 - OFFSET 0 + OFFSET 0 SETTINGS readonly=1, + max_execution_time=60 ' --- # name: TestQuery.test_property_filter_aggregations.1 @@ -314,7 +333,8 @@ HAVING and(greater(count(), 1)) ORDER BY count() DESC LIMIT 101 - OFFSET 0 + OFFSET 0 SETTINGS readonly=1, + max_execution_time=60 ' --- # name: TestQuery.test_property_filter_aggregations_materialized @@ -327,7 +347,8 @@ GROUP BY events.mat_key ORDER BY count() DESC LIMIT 101 - OFFSET 0 + OFFSET 0 SETTINGS readonly=1, + max_execution_time=60 ' --- # name: TestQuery.test_property_filter_aggregations_materialized.1 @@ -341,7 +362,8 @@ HAVING and(greater(count(), 1)) ORDER BY count() DESC LIMIT 101 - OFFSET 0 + OFFSET 0 SETTINGS readonly=1, + max_execution_time=60 ' --- # name: TestQuery.test_select_event_person @@ -354,7 +376,8 @@ WHERE and(equals(events.team_id, 79), less(events.timestamp, '2020-01-10 12:14:05.000000'), greater(events.timestamp, '2020-01-09 12:00:00.000000')) ORDER BY events.event ASC LIMIT 101 - OFFSET 0 + OFFSET 0 SETTINGS readonly=1, + max_execution_time=60 ' --- # name: TestQuery.test_select_hogql_expressions @@ -368,7 +391,8 @@ WHERE and(equals(events.team_id, 80), less(events.timestamp, '2020-01-10 12:14:05.000000'), greater(events.timestamp, '2020-01-09 12:00:00.000000')) ORDER BY replaceRegexpAll(JSONExtractRaw(events.properties, 'key'), '^"|"$', '') ASC LIMIT 101 - OFFSET 0 + OFFSET 0 SETTINGS readonly=1, + max_execution_time=60 ' --- # name: TestQuery.test_select_hogql_expressions.1 @@ -380,7 +404,8 @@ WHERE and(equals(events.team_id, 80), less(events.timestamp, '2020-01-10 12:14:05.000000'), greater(events.timestamp, '2020-01-09 12:00:00.000000')) ORDER BY tuple(events.uuid, events.event, events.properties, events.timestamp, events.team_id, events.distinct_id, events.elements_chain, events.created_at) ASC LIMIT 101 - OFFSET 0 + OFFSET 0 SETTINGS readonly=1, + max_execution_time=60 ' --- # name: TestQuery.test_select_hogql_expressions.2 @@ -393,7 +418,8 @@ GROUP BY events.event ORDER BY count() DESC LIMIT 101 - OFFSET 0 + OFFSET 0 SETTINGS readonly=1, + max_execution_time=60 ' --- # name: TestQuery.test_select_hogql_expressions.3 @@ -406,6 +432,7 @@ GROUP BY events.event ORDER BY count() DESC, events.event ASC LIMIT 101 - OFFSET 0 + OFFSET 0 SETTINGS readonly=1, + max_execution_time=60 ' --- diff --git a/posthog/hogql/constants.py b/posthog/hogql/constants.py index 244029c7c59bb..9d1aecf083883 100644 --- a/posthog/hogql/constants.py +++ b/posthog/hogql/constants.py @@ -1,4 +1,8 @@ # HogQL -> ClickHouse allowed transformations +from typing import Optional + +from pydantic import BaseModel, Extra + CLICKHOUSE_FUNCTIONS = { # arithmetic "abs": "abs", @@ -103,3 +107,11 @@ # Never return more rows than this in top level HogQL SELECT statements DEFAULT_RETURNED_ROWS = 100 MAX_SELECT_RETURNED_ROWS = 65535 + +# Settings applied on top of all HogQL queries. +class HogQLSettings(BaseModel): + class Config: + extra = Extra.forbid + + readonly: Optional[int] = 1 + max_execution_time: Optional[int] = 60 diff --git a/posthog/hogql/printer.py b/posthog/hogql/printer.py index c362c10486c1f..edd7a68e8cc3c 100644 --- a/posthog/hogql/printer.py +++ b/posthog/hogql/printer.py @@ -1,9 +1,12 @@ +import re from dataclasses import dataclass from typing import List, Literal, Optional, Union, cast +from clickhouse_driver.util.escape import escape_param + from ee.clickhouse.materialized_columns.columns import TablesWithMaterializedColumns, get_materialized_columns from posthog.hogql import ast -from posthog.hogql.constants import CLICKHOUSE_FUNCTIONS, HOGQL_AGGREGATIONS, MAX_SELECT_RETURNED_ROWS +from posthog.hogql.constants import CLICKHOUSE_FUNCTIONS, HOGQL_AGGREGATIONS, MAX_SELECT_RETURNED_ROWS, HogQLSettings from posthog.hogql.context import HogQLContext from posthog.hogql.database import Table, create_hogql_database from posthog.hogql.print_string import print_clickhouse_identifier, print_hogql_identifier @@ -33,9 +36,10 @@ def print_ast( context: HogQLContext, dialect: Literal["hogql", "clickhouse"], stack: Optional[List[ast.SelectQuery]] = None, + settings: Optional[HogQLSettings] = None, ) -> str: prepared_ast = prepare_ast_for_printing(node=node, context=context, dialect=dialect, stack=stack) - return print_prepared_ast(node=prepared_ast, context=context, dialect=dialect, stack=stack) + return print_prepared_ast(node=prepared_ast, context=context, dialect=dialect, stack=stack, settings=settings) def prepare_ast_for_printing( @@ -64,9 +68,10 @@ def print_prepared_ast( context: HogQLContext, dialect: Literal["hogql", "clickhouse"], stack: Optional[List[ast.SelectQuery]] = None, + settings: Optional[HogQLSettings] = None, ) -> str: # _Printer also adds a team_id guard if printing clickhouse - return _Printer(context=context, dialect=dialect, stack=stack or []).visit(node) + return _Printer(context=context, dialect=dialect, stack=stack or [], settings=settings).visit(node) @dataclass @@ -79,17 +84,38 @@ class _Printer(Visitor): # NOTE: Call "print_ast()", not this class directly. def __init__( - self, context: HogQLContext, dialect: Literal["hogql", "clickhouse"], stack: Optional[List[ast.AST]] = None + self, + context: HogQLContext, + dialect: Literal["hogql", "clickhouse"], + stack: Optional[List[ast.AST]] = None, + settings: Optional[HogQLSettings] = None, ): self.context = context self.dialect = dialect - # Keep track of all traversed nodes. - self.stack: List[ast.AST] = stack or [] + self.stack: List[ast.AST] = stack or [] # Keep track of all traversed nodes. + self.settings = settings def visit(self, node: ast.AST): self.stack.append(node) response = super().visit(node) self.stack.pop() + + if len(self.stack) == 0 and self.dialect == "clickhouse" and self.settings: + if not isinstance(node, ast.SelectQuery) and not isinstance(node, ast.SelectUnionQuery): + raise ValueError("Settings can only be applied to SELECT queries") + settings = [] + for key, value in self.settings: + if not isinstance(value, (int, float, str)): + raise ValueError(f"Setting {key} must be a string, int, or float") + if not re.match(r"^[a-zA-Z0-9_]+$", key): + raise ValueError(f"Setting {key} is not supported") + if isinstance(value, int) or isinstance(value, float): + settings.append(f"{key}={value}") + else: + settings.append(f"{key}={escape_param(value)}") + if len(settings) > 0: + response += f" SETTINGS {', '.join(settings)}" + return response def visit_select_union_query(self, node: ast.SelectUnionQuery): diff --git a/posthog/hogql/query.py b/posthog/hogql/query.py index c9d2193341f97..5ac01af6ef960 100644 --- a/posthog/hogql/query.py +++ b/posthog/hogql/query.py @@ -4,7 +4,7 @@ from posthog.clickhouse.client.connection import Workload from posthog.hogql import ast -from posthog.hogql.constants import DEFAULT_RETURNED_ROWS +from posthog.hogql.constants import DEFAULT_RETURNED_ROWS, HogQLSettings from posthog.hogql.hogql import HogQLContext from posthog.hogql.parser import parse_select from posthog.hogql.placeholders import assert_no_placeholders, replace_placeholders @@ -32,6 +32,7 @@ def execute_hogql_query( query_type: str = "hogql_query", placeholders: Optional[Dict[str, ast.Expr]] = None, workload: Workload = Workload.ONLINE, + settings: Optional[HogQLSettings] = None, ) -> HogQLQueryResponse: if isinstance(query, ast.SelectQuery): select_query = query @@ -69,7 +70,9 @@ def execute_hogql_query( clickhouse_context = HogQLContext( team_id=team.pk, enable_select_queries=True, person_on_events_mode=team.person_on_events_mode ) - clickhouse = print_ast(select_query, clickhouse_context, "clickhouse") + clickhouse = print_ast( + select_query, context=clickhouse_context, dialect="clickhouse", settings=settings or HogQLSettings() + ) results, types = insight_sync_execute( clickhouse, diff --git a/posthog/hogql/test/__snapshots__/test_database.ambr b/posthog/hogql/test/__snapshots__/test_database.ambr index e15d2967c51f4..f5f15ea0e90e7 100644 --- a/posthog/hogql/test/__snapshots__/test_database.ambr +++ b/posthog/hogql/test/__snapshots__/test_database.ambr @@ -231,6 +231,24 @@ "type": "lazy_table", "table": "persons" } + ], + "groups": [ + { + "key": "index", + "type": "integer" + }, + { + "key": "key", + "type": "string" + }, + { + "key": "created_at", + "type": "datetime" + }, + { + "key": "properties", + "type": "json" + } ] } ' @@ -463,6 +481,24 @@ "type": "lazy_table", "table": "persons" } + ], + "groups": [ + { + "key": "index", + "type": "integer" + }, + { + "key": "key", + "type": "string" + }, + { + "key": "created_at", + "type": "datetime" + }, + { + "key": "properties", + "type": "json" + } ] } ' diff --git a/posthog/hogql/test/test_query.py b/posthog/hogql/test/test_query.py index 3b3310a2eaed4..133493d6c2a6a 100644 --- a/posthog/hogql/test/test_query.py +++ b/posthog/hogql/test/test_query.py @@ -46,7 +46,7 @@ def test_query(self): ) self.assertEqual( response.clickhouse, - f"SELECT count(), events.event FROM events WHERE and(equals(events.team_id, {self.team.id}), equals(replaceRegexpAll(JSONExtractRaw(events.properties, %(hogql_val_0)s), '^\"|\"$', ''), %(hogql_val_1)s)) GROUP BY events.event LIMIT 100", + f"SELECT count(), events.event FROM events WHERE and(equals(events.team_id, {self.team.id}), equals(replaceRegexpAll(JSONExtractRaw(events.properties, %(hogql_val_0)s), '^\"|\"$', ''), %(hogql_val_1)s)) GROUP BY events.event LIMIT 100 SETTINGS readonly=1, max_execution_time=60", ) self.assertEqual( response.hogql, @@ -61,7 +61,7 @@ def test_query(self): ) self.assertEqual( response.clickhouse, - f"SELECT count, event FROM (SELECT count() AS count, events.event FROM events WHERE and(equals(events.team_id, {self.team.id}), equals(replaceRegexpAll(JSONExtractRaw(events.properties, %(hogql_val_0)s), '^\"|\"$', ''), %(hogql_val_1)s)) GROUP BY events.event) GROUP BY count, event LIMIT 100", + f"SELECT count, event FROM (SELECT count() AS count, events.event FROM events WHERE and(equals(events.team_id, {self.team.id}), equals(replaceRegexpAll(JSONExtractRaw(events.properties, %(hogql_val_0)s), '^\"|\"$', ''), %(hogql_val_1)s)) GROUP BY events.event) GROUP BY count, event LIMIT 100 SETTINGS readonly=1, max_execution_time=60", ) self.assertEqual( response.hogql, @@ -76,7 +76,7 @@ def test_query(self): ) self.assertEqual( response.clickhouse, - f"SELECT c.count, c.event FROM (SELECT count(*) AS count, events.event FROM events WHERE and(equals(events.team_id, {self.team.id}), equals(replaceRegexpAll(JSONExtractRaw(events.properties, %(hogql_val_0)s), '^\"|\"$', ''), %(hogql_val_1)s)) GROUP BY events.event) AS c GROUP BY c.count, c.event LIMIT 100", + f"SELECT c.count, c.event FROM (SELECT count(*) AS count, events.event FROM events WHERE and(equals(events.team_id, {self.team.id}), equals(replaceRegexpAll(JSONExtractRaw(events.properties, %(hogql_val_0)s), '^\"|\"$', ''), %(hogql_val_1)s)) GROUP BY events.event) AS c GROUP BY c.count, c.event LIMIT 100 SETTINGS readonly=1, max_execution_time=60", ) self.assertEqual( response.hogql, @@ -91,7 +91,7 @@ def test_query(self): ) self.assertEqual( response.clickhouse, - f"SELECT DISTINCT replaceRegexpAll(JSONExtractRaw(person.properties, %(hogql_val_0)s), '^\"|\"$', '') FROM person WHERE and(equals(person.team_id, {self.team.id}), equals(replaceRegexpAll(JSONExtractRaw(person.properties, %(hogql_val_1)s), '^\"|\"$', ''), %(hogql_val_2)s)) LIMIT 100", + f"SELECT DISTINCT replaceRegexpAll(JSONExtractRaw(person.properties, %(hogql_val_0)s), '^\"|\"$', '') FROM person WHERE and(equals(person.team_id, {self.team.id}), equals(replaceRegexpAll(JSONExtractRaw(person.properties, %(hogql_val_1)s), '^\"|\"$', ''), %(hogql_val_2)s)) LIMIT 100 SETTINGS readonly=1, max_execution_time=60", ) self.assertEqual( response.hogql, @@ -105,7 +105,7 @@ def test_query(self): ) self.assertEqual( response.clickhouse, - f"SELECT DISTINCT person_distinct_id2.person_id, person_distinct_id2.distinct_id FROM person_distinct_id2 WHERE equals(person_distinct_id2.team_id, {self.team.id}) LIMIT 100", + f"SELECT DISTINCT person_distinct_id2.person_id, person_distinct_id2.distinct_id FROM person_distinct_id2 WHERE equals(person_distinct_id2.team_id, {self.team.id}) LIMIT 100 SETTINGS readonly=1, max_execution_time=60", ) self.assertEqual( response.hogql, @@ -130,7 +130,7 @@ def test_query_joins_simple(self): ) self.assertEqual( response.clickhouse, - f"SELECT e.event, e.timestamp, pdi.distinct_id, p.id, replaceRegexpAll(JSONExtractRaw(p.properties, %(hogql_val_0)s), '^\"|\"$', '') FROM events AS e LEFT JOIN person_distinct_id2 AS pdi ON equals(pdi.distinct_id, e.distinct_id) LEFT JOIN person AS p ON equals(p.id, pdi.person_id) WHERE and(equals(p.team_id, {self.team.id}), equals(pdi.team_id, {self.team.id}), equals(e.team_id, {self.team.id})) LIMIT 100", + f"SELECT e.event, e.timestamp, pdi.distinct_id, p.id, replaceRegexpAll(JSONExtractRaw(p.properties, %(hogql_val_0)s), '^\"|\"$', '') FROM events AS e LEFT JOIN person_distinct_id2 AS pdi ON equals(pdi.distinct_id, e.distinct_id) LEFT JOIN person AS p ON equals(p.id, pdi.person_id) WHERE and(equals(p.team_id, {self.team.id}), equals(pdi.team_id, {self.team.id}), equals(e.team_id, {self.team.id})) LIMIT 100 SETTINGS readonly=1, max_execution_time=60", ) self.assertEqual( response.hogql, @@ -165,7 +165,7 @@ def test_query_joins_pdi(self): f"argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id FROM person_distinct_id2 WHERE " f"equals(person_distinct_id2.team_id, {self.team.id}) GROUP BY person_distinct_id2.distinct_id HAVING " f"equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0)) AS pdi ON " - f"equals(e.distinct_id, pdi.distinct_id) WHERE equals(e.team_id, {self.team.id}) LIMIT 100", + f"equals(e.distinct_id, pdi.distinct_id) WHERE equals(e.team_id, {self.team.id}) LIMIT 100 SETTINGS readonly=1, max_execution_time=60", ) self.assertEqual( response.hogql, @@ -183,7 +183,7 @@ def test_query_joins_events_pdi(self): ) self.assertEqual( response.clickhouse, - f"SELECT events.event, events.timestamp, events__pdi.distinct_id, events__pdi.person_id FROM events INNER JOIN (SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id, person_distinct_id2.distinct_id FROM person_distinct_id2 WHERE equals(person_distinct_id2.team_id, {self.team.pk}) GROUP BY person_distinct_id2.distinct_id HAVING equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0)) AS events__pdi ON equals(events.distinct_id, events__pdi.distinct_id) WHERE equals(events.team_id, {self.team.pk}) LIMIT 10", + f"SELECT events.event, events.timestamp, events__pdi.distinct_id, events__pdi.person_id FROM events INNER JOIN (SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id, person_distinct_id2.distinct_id FROM person_distinct_id2 WHERE equals(person_distinct_id2.team_id, {self.team.pk}) GROUP BY person_distinct_id2.distinct_id HAVING equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0)) AS events__pdi ON equals(events.distinct_id, events__pdi.distinct_id) WHERE equals(events.team_id, {self.team.pk}) LIMIT 10 SETTINGS readonly=1, max_execution_time=60", ) self.assertEqual( response.hogql, @@ -207,7 +207,7 @@ def test_query_joins_events_e_pdi(self): ) self.assertEqual( response.clickhouse, - f"SELECT e.event, e.timestamp, e__pdi.distinct_id, e__pdi.person_id FROM events AS e INNER JOIN (SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id, person_distinct_id2.distinct_id FROM person_distinct_id2 WHERE equals(person_distinct_id2.team_id, {self.team.pk}) GROUP BY person_distinct_id2.distinct_id HAVING equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0)) AS e__pdi ON equals(e.distinct_id, e__pdi.distinct_id) WHERE equals(e.team_id, {self.team.pk}) LIMIT 10", + f"SELECT e.event, e.timestamp, e__pdi.distinct_id, e__pdi.person_id FROM events AS e INNER JOIN (SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id, person_distinct_id2.distinct_id FROM person_distinct_id2 WHERE equals(person_distinct_id2.team_id, {self.team.pk}) GROUP BY person_distinct_id2.distinct_id HAVING equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0)) AS e__pdi ON equals(e.distinct_id, e__pdi.distinct_id) WHERE equals(e.team_id, {self.team.pk}) LIMIT 10 SETTINGS readonly=1, max_execution_time=60", ) self.assertEqual(response.results[0][0], "random event") self.assertEqual(response.results[0][2], "bla") @@ -231,7 +231,7 @@ def test_query_joins_pdi_persons(self): f"argMax(person.created_at, person.version) AS created_at, person.id FROM person WHERE " f"equals(person.team_id, {self.team.pk}) GROUP BY person.id HAVING equals(argMax(person.is_deleted, " f"person.version), 0)) AS pdi__person ON equals(pdi.person_id, pdi__person.id) WHERE " - f"equals(pdi.team_id, {self.team.pk}) LIMIT 10", + f"equals(pdi.team_id, {self.team.pk}) LIMIT 10 SETTINGS readonly=1, max_execution_time=60", ) self.assertEqual(response.results[0][0], "bla") self.assertEqual(response.results[0][1], datetime.datetime(2020, 1, 10, 0, 0)) @@ -254,7 +254,7 @@ def test_query_joins_pdi_person_properties(self): f"(SELECT argMax(replaceRegexpAll(JSONExtractRaw(person.properties, %(hogql_val_0)s), '^\"|\"$', ''), person.version) " f"AS properties___sneaky_mail, person.id FROM person WHERE equals(person.team_id, {self.team.pk}) GROUP BY person.id " f"HAVING equals(argMax(person.is_deleted, person.version), 0)) AS pdi__person ON " - f"equals(pdi.person_id, pdi__person.id) WHERE equals(pdi.team_id, {self.team.pk}) LIMIT 10", + f"equals(pdi.person_id, pdi__person.id) WHERE equals(pdi.team_id, {self.team.pk}) LIMIT 10 SETTINGS readonly=1, max_execution_time=60", ) self.assertEqual(response.results[0][0], "bla") self.assertEqual(response.results[0][1], "tim@posthog.com") @@ -276,7 +276,8 @@ def test_query_joins_events_pdi_person(self): f"person_distinct_id2.version), 0)) AS events__pdi ON equals(events.distinct_id, events__pdi.distinct_id) " f"INNER JOIN (SELECT person.id FROM person WHERE equals(person.team_id, {self.team.pk}) GROUP BY person.id HAVING " f"equals(argMax(person.is_deleted, person.version), 0)) AS events__pdi__person ON " - f"equals(events__pdi.person_id, events__pdi__person.id) WHERE equals(events.team_id, {self.team.pk}) LIMIT 10", + f"equals(events__pdi.person_id, events__pdi__person.id) WHERE equals(events.team_id, {self.team.pk}) LIMIT 10" + f" SETTINGS readonly=1, max_execution_time=60", ) self.assertEqual( response.hogql, @@ -305,7 +306,7 @@ def test_query_joins_events_pdi_person_properties(self): f"argMax(replaceRegexpAll(JSONExtractRaw(person.properties, %(hogql_val_0)s), '^\"|\"$', ''), person.version) " f"AS properties___sneaky_mail, person.id FROM person WHERE equals(person.team_id, {self.team.pk}) GROUP BY person.id HAVING " f"equals(argMax(person.is_deleted, person.version), 0)) AS events__pdi__person ON equals(events__pdi.person_id, " - f"events__pdi__person.id) WHERE equals(events.team_id, {self.team.pk}) LIMIT 10", + f"events__pdi__person.id) WHERE equals(events.team_id, {self.team.pk}) LIMIT 10 SETTINGS readonly=1, max_execution_time=60", ) self.assertEqual( response.hogql, @@ -333,7 +334,8 @@ def test_query_joins_events_pdi_e_person_properties(self): f"(SELECT argMax(replaceRegexpAll(JSONExtractRaw(person.properties, %(hogql_val_0)s), '^\"|\"$', ''), " f"person.version) AS properties___sneaky_mail, person.id FROM person WHERE equals(person.team_id, {self.team.pk}) " f"GROUP BY person.id HAVING equals(argMax(person.is_deleted, person.version), 0)) AS e__pdi__person ON " - f"equals(e__pdi.person_id, e__pdi__person.id) WHERE equals(e.team_id, {self.team.pk}) LIMIT 10", + f"equals(e__pdi.person_id, e__pdi__person.id) WHERE equals(e.team_id, {self.team.pk}) LIMIT 10" + f" SETTINGS readonly=1, max_execution_time=60", ) self.assertEqual( response.hogql, @@ -360,7 +362,8 @@ def test_query_joins_events_person_properties(self): f"e__pdi.distinct_id) INNER JOIN (SELECT argMax(replaceRegexpAll(JSONExtractRaw(person.properties, " f"%(hogql_val_0)s), '^\"|\"$', ''), person.version) AS properties___sneaky_mail, person.id FROM person WHERE " f"equals(person.team_id, {self.team.pk}) GROUP BY person.id HAVING equals(argMax(person.is_deleted, person.version), 0)) " - f"AS e__pdi__person ON equals(e__pdi.person_id, e__pdi__person.id) WHERE equals(e.team_id, {self.team.pk}) LIMIT 10", + f"AS e__pdi__person ON equals(e__pdi.person_id, e__pdi__person.id) WHERE equals(e.team_id, {self.team.pk}) LIMIT 10" + f" SETTINGS readonly=1, max_execution_time=60", ) self.assertEqual( response.hogql, @@ -385,7 +388,7 @@ def test_query_joins_events_person_properties_in_aggregration(self): f"%(hogql_val_0)s), '^\"|\"$', ''), person.version) AS properties___sneaky_mail, person.id FROM person WHERE " f"equals(person.team_id, {self.team.pk}) GROUP BY person.id HAVING equals(argMax(person.is_deleted, person.version), 0)) " f"AS s__pdi__person ON equals(s__pdi.person_id, s__pdi__person.id) WHERE equals(s.team_id, {self.team.pk}) " - f"GROUP BY s__pdi__person.properties___sneaky_mail LIMIT 10" + f"GROUP BY s__pdi__person.properties___sneaky_mail LIMIT 10 SETTINGS readonly=1, max_execution_time=60" ) self.assertEqual(response.clickhouse, expected) self.assertEqual( @@ -405,7 +408,8 @@ def test_select_person_on_events(self): response.clickhouse, f"SELECT replaceRegexpAll(JSONExtractRaw(s.person_properties, %(hogql_val_0)s), '^\"|\"$', ''), " f"count() FROM events AS s WHERE equals(s.team_id, {self.team.pk}) GROUP BY " - f"replaceRegexpAll(JSONExtractRaw(s.person_properties, %(hogql_val_1)s), '^\"|\"$', '') LIMIT 10", + f"replaceRegexpAll(JSONExtractRaw(s.person_properties, %(hogql_val_1)s), '^\"|\"$', '') LIMIT 10" + f" SETTINGS readonly=1, max_execution_time=60", ) self.assertEqual( response.hogql, @@ -433,7 +437,7 @@ def test_query_select_person_with_joins_without_poe(self): f"'^\"|\"$', ''), person.version) AS properties___sneaky_mail, person.id FROM person WHERE " f"equals(person.team_id, {self.team.pk}) GROUP BY person.id HAVING equals(argMax(person.is_deleted, person.version), 0)) " f"AS events__pdi__person ON equals(events__pdi.person_id, events__pdi__person.id) " - f"WHERE equals(events.team_id, {self.team.pk}) LIMIT 10", + f"WHERE equals(events.team_id, {self.team.pk}) LIMIT 10 SETTINGS readonly=1, max_execution_time=60", ) self.assertEqual( response.hogql, @@ -454,7 +458,7 @@ def test_query_select_person_with_poe_without_joins(self): ) self.assertEqual( response.clickhouse, - f"SELECT events.event, events.timestamp, events.person_id, replaceRegexpAll(JSONExtractRaw(events.person_properties, %(hogql_val_0)s), '^\"|\"$', '') FROM events WHERE equals(events.team_id, {self.team.pk}) LIMIT 10", + f"SELECT events.event, events.timestamp, events.person_id, replaceRegexpAll(JSONExtractRaw(events.person_properties, %(hogql_val_0)s), '^\"|\"$', '') FROM events WHERE equals(events.team_id, {self.team.pk}) LIMIT 10 SETTINGS readonly=1, max_execution_time=60", ) self.assertEqual( response.hogql, @@ -495,7 +499,7 @@ def test_prop_cohort_basic(self): ) self.assertEqual( response.clickhouse, - f"SELECT events.event, count() FROM events INNER JOIN (SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id, person_distinct_id2.distinct_id FROM person_distinct_id2 WHERE equals(person_distinct_id2.team_id, {self.team.pk}) GROUP BY person_distinct_id2.distinct_id HAVING equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0)) AS events__pdi ON equals(events.distinct_id, events__pdi.distinct_id) WHERE and(equals(events.team_id, {self.team.pk}), in(events__pdi.person_id, (SELECT cohortpeople.person_id FROM cohortpeople WHERE and(equals(cohortpeople.team_id, {self.team.pk}), equals(cohortpeople.cohort_id, {cohort.pk})) GROUP BY cohortpeople.person_id, cohortpeople.cohort_id, cohortpeople.version HAVING greater(sum(cohortpeople.sign), 0)))) GROUP BY events.event LIMIT 100", + f"SELECT events.event, count() FROM events INNER JOIN (SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id, person_distinct_id2.distinct_id FROM person_distinct_id2 WHERE equals(person_distinct_id2.team_id, {self.team.pk}) GROUP BY person_distinct_id2.distinct_id HAVING equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0)) AS events__pdi ON equals(events.distinct_id, events__pdi.distinct_id) WHERE and(equals(events.team_id, {self.team.pk}), in(events__pdi.person_id, (SELECT cohortpeople.person_id FROM cohortpeople WHERE and(equals(cohortpeople.team_id, {self.team.pk}), equals(cohortpeople.cohort_id, {cohort.pk})) GROUP BY cohortpeople.person_id, cohortpeople.cohort_id, cohortpeople.version HAVING greater(sum(cohortpeople.sign), 0)))) GROUP BY events.event LIMIT 100 SETTINGS readonly=1, max_execution_time=60", ) self.assertEqual(response.results, [("$pageview", 2)]) @@ -514,7 +518,7 @@ def test_prop_cohort_basic(self): f"SELECT events.event, count(*) FROM events WHERE and(equals(events.team_id, {self.team.pk}), in(events.person_id, " f"(SELECT cohortpeople.person_id FROM cohortpeople WHERE and(equals(cohortpeople.team_id, {self.team.pk}), " f"equals(cohortpeople.cohort_id, {cohort.pk})) GROUP BY cohortpeople.person_id, cohortpeople.cohort_id, " - f"cohortpeople.version HAVING greater(sum(cohortpeople.sign), 0)))) GROUP BY events.event LIMIT 100", + f"cohortpeople.version HAVING greater(sum(cohortpeople.sign), 0)))) GROUP BY events.event LIMIT 100 SETTINGS readonly=1, max_execution_time=60", ) self.assertEqual(response.results, [("$pageview", 2)]) @@ -547,7 +551,7 @@ def test_prop_cohort_static(self): self.assertEqual(response.results, [("$pageview", 1)]) self.assertEqual( response.clickhouse, - f"SELECT events.event, count() FROM events INNER JOIN (SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id, person_distinct_id2.distinct_id FROM person_distinct_id2 WHERE equals(person_distinct_id2.team_id, {self.team.pk}) GROUP BY person_distinct_id2.distinct_id HAVING equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0)) AS events__pdi ON equals(events.distinct_id, events__pdi.distinct_id) WHERE and(equals(events.team_id, {self.team.pk}), in(events__pdi.person_id, (SELECT person_static_cohort.person_id FROM person_static_cohort WHERE and(equals(person_static_cohort.team_id, {self.team.pk}), equals(person_static_cohort.cohort_id, {cohort.pk}))))) GROUP BY events.event LIMIT 100", + f"SELECT events.event, count() FROM events INNER JOIN (SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id, person_distinct_id2.distinct_id FROM person_distinct_id2 WHERE equals(person_distinct_id2.team_id, {self.team.pk}) GROUP BY person_distinct_id2.distinct_id HAVING equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0)) AS events__pdi ON equals(events.distinct_id, events__pdi.distinct_id) WHERE and(equals(events.team_id, {self.team.pk}), in(events__pdi.person_id, (SELECT person_static_cohort.person_id FROM person_static_cohort WHERE and(equals(person_static_cohort.team_id, {self.team.pk}), equals(person_static_cohort.cohort_id, {cohort.pk}))))) GROUP BY events.event LIMIT 100 SETTINGS readonly=1, max_execution_time=60", ) with override_settings(PERSON_ON_EVENTS_OVERRIDE=True): @@ -563,7 +567,7 @@ def test_prop_cohort_static(self): self.assertEqual(response.results, [("$pageview", 1)]) self.assertEqual( response.clickhouse, - f"SELECT events.event, count(*) FROM events WHERE and(equals(events.team_id, {self.team.pk}), in(events.person_id, (SELECT person_static_cohort.person_id FROM person_static_cohort WHERE and(equals(person_static_cohort.team_id, {self.team.pk}), equals(person_static_cohort.cohort_id, {cohort.pk}))))) GROUP BY events.event LIMIT 100", + f"SELECT events.event, count(*) FROM events WHERE and(equals(events.team_id, {self.team.pk}), in(events.person_id, (SELECT person_static_cohort.person_id FROM person_static_cohort WHERE and(equals(person_static_cohort.team_id, {self.team.pk}), equals(person_static_cohort.cohort_id, {cohort.pk}))))) GROUP BY events.event LIMIT 100 SETTINGS readonly=1, max_execution_time=60", ) def test_join_with_property_materialized_session_id(self): @@ -589,7 +593,7 @@ def test_join_with_property_materialized_session_id(self): ) self.assertEqual( response.clickhouse, - f"SELECT e.event, s.session_id FROM events AS e LEFT JOIN session_recording_events AS s ON equals(s.session_id, e.`$session_id`) WHERE and(equals(s.team_id, {self.team.pk}), equals(e.team_id, {self.team.pk}), isNotNull(e.`$session_id`)) LIMIT 10", + f"SELECT e.event, s.session_id FROM events AS e LEFT JOIN session_recording_events AS s ON equals(s.session_id, e.`$session_id`) WHERE and(equals(s.team_id, {self.team.pk}), equals(e.team_id, {self.team.pk}), isNotNull(e.`$session_id`)) LIMIT 10 SETTINGS readonly=1, max_execution_time=60", ) self.assertEqual(response.results, [("$pageview", "111"), ("$pageview", "111")]) @@ -599,7 +603,7 @@ def test_join_with_property_materialized_session_id(self): ) self.assertEqual( response.clickhouse, - f"SELECT e.event, s.session_id FROM session_recording_events AS s LEFT JOIN events AS e ON equals(e.`$session_id`, s.session_id) WHERE and(equals(e.team_id, {self.team.pk}), equals(s.team_id, {self.team.pk}), isNotNull(e.`$session_id`)) LIMIT 10", + f"SELECT e.event, s.session_id FROM session_recording_events AS s LEFT JOIN events AS e ON equals(e.`$session_id`, s.session_id) WHERE and(equals(e.team_id, {self.team.pk}), equals(s.team_id, {self.team.pk}), isNotNull(e.`$session_id`)) LIMIT 10 SETTINGS readonly=1, max_execution_time=60", ) self.assertEqual(response.results, [("$pageview", "111"), ("$pageview", "111")]) @@ -626,7 +630,7 @@ def test_join_with_property_not_materialized(self): ) self.assertEqual( response.clickhouse, - f"SELECT e.event, s.session_id FROM events AS e LEFT JOIN session_recording_events AS s ON equals(s.session_id, replaceRegexpAll(JSONExtractRaw(e.properties, %(hogql_val_0)s), '^\"|\"$', '')) WHERE and(equals(s.team_id, {self.team.pk}), equals(e.team_id, {self.team.pk}), isNotNull(replaceRegexpAll(JSONExtractRaw(e.properties, %(hogql_val_1)s), '^\"|\"$', ''))) LIMIT 10", + f"SELECT e.event, s.session_id FROM events AS e LEFT JOIN session_recording_events AS s ON equals(s.session_id, replaceRegexpAll(JSONExtractRaw(e.properties, %(hogql_val_0)s), '^\"|\"$', '')) WHERE and(equals(s.team_id, {self.team.pk}), equals(e.team_id, {self.team.pk}), isNotNull(replaceRegexpAll(JSONExtractRaw(e.properties, %(hogql_val_1)s), '^\"|\"$', ''))) LIMIT 10 SETTINGS readonly=1, max_execution_time=60", ) self.assertEqual(response.results, [("$pageview", "111"), ("$pageview", "111")]) @@ -636,6 +640,6 @@ def test_join_with_property_not_materialized(self): ) self.assertEqual( response.clickhouse, - f"SELECT e.event, s.session_id FROM session_recording_events AS s LEFT JOIN events AS e ON equals(replaceRegexpAll(JSONExtractRaw(e.properties, %(hogql_val_0)s), '^\"|\"$', ''), s.session_id) WHERE and(equals(e.team_id, {self.team.pk}), equals(s.team_id, {self.team.pk}), isNotNull(replaceRegexpAll(JSONExtractRaw(e.properties, %(hogql_val_1)s), '^\"|\"$', ''))) LIMIT 10", + f"SELECT e.event, s.session_id FROM session_recording_events AS s LEFT JOIN events AS e ON equals(replaceRegexpAll(JSONExtractRaw(e.properties, %(hogql_val_0)s), '^\"|\"$', ''), s.session_id) WHERE and(equals(e.team_id, {self.team.pk}), equals(s.team_id, {self.team.pk}), isNotNull(replaceRegexpAll(JSONExtractRaw(e.properties, %(hogql_val_1)s), '^\"|\"$', ''))) LIMIT 10 SETTINGS readonly=1, max_execution_time=60", ) self.assertEqual(response.results, [("$pageview", "111"), ("$pageview", "111")])