Edit on GitHub

sqlglot.dialects.clickhouse

  1from __future__ import annotations
  2
  3import typing as t
  4
  5from sqlglot import exp, generator, parser, tokens
  6from sqlglot.dialects.dialect import Dialect, inline_array_sql, rename_func, var_map_sql
  7from sqlglot.errors import ParseError
  8from sqlglot.parser import parse_var_map
  9from sqlglot.tokens import Token, TokenType
 10
 11
 12def _lower_func(sql: str) -> str:
 13    index = sql.index("(")
 14    return sql[:index].lower() + sql[index:]
 15
 16
 17class ClickHouse(Dialect):
 18    normalize_functions = None
 19    null_ordering = "nulls_are_last"
 20
 21    class Tokenizer(tokens.Tokenizer):
 22        COMMENTS = ["--", "#", "#!", ("/*", "*/")]
 23        IDENTIFIERS = ['"', "`"]
 24        BIT_STRINGS = [("0b", "")]
 25        HEX_STRINGS = [("0x", ""), ("0X", "")]
 26
 27        KEYWORDS = {
 28            **tokens.Tokenizer.KEYWORDS,
 29            "ASOF": TokenType.ASOF,
 30            "ATTACH": TokenType.COMMAND,
 31            "DATETIME64": TokenType.DATETIME64,
 32            "FINAL": TokenType.FINAL,
 33            "FLOAT32": TokenType.FLOAT,
 34            "FLOAT64": TokenType.DOUBLE,
 35            "GLOBAL": TokenType.GLOBAL,
 36            "INT128": TokenType.INT128,
 37            "INT16": TokenType.SMALLINT,
 38            "INT256": TokenType.INT256,
 39            "INT32": TokenType.INT,
 40            "INT64": TokenType.BIGINT,
 41            "INT8": TokenType.TINYINT,
 42            "MAP": TokenType.MAP,
 43            "TUPLE": TokenType.STRUCT,
 44            "UINT128": TokenType.UINT128,
 45            "UINT16": TokenType.USMALLINT,
 46            "UINT256": TokenType.UINT256,
 47            "UINT32": TokenType.UINT,
 48            "UINT64": TokenType.UBIGINT,
 49            "UINT8": TokenType.UTINYINT,
 50        }
 51
 52    class Parser(parser.Parser):
 53        FUNCTIONS = {
 54            **parser.Parser.FUNCTIONS,  # type: ignore
 55            "ANY": exp.AnyValue.from_arg_list,
 56            "MAP": parse_var_map,
 57            "MATCH": exp.RegexpLike.from_arg_list,
 58            "UNIQ": exp.ApproxDistinct.from_arg_list,
 59        }
 60
 61        FUNCTION_PARSERS = {
 62            **parser.Parser.FUNCTION_PARSERS,
 63            "QUANTILE": lambda self: self._parse_quantile(),
 64        }
 65
 66        FUNCTION_PARSERS.pop("MATCH")
 67
 68        NO_PAREN_FUNCTION_PARSERS = parser.Parser.NO_PAREN_FUNCTION_PARSERS.copy()
 69        NO_PAREN_FUNCTION_PARSERS.pop(TokenType.ANY)
 70
 71        RANGE_PARSERS = {
 72            **parser.Parser.RANGE_PARSERS,
 73            TokenType.GLOBAL: lambda self, this: self._match(TokenType.IN)
 74            and self._parse_in(this, is_global=True),
 75        }
 76
 77        # The PLACEHOLDER entry is popped because 1) it doesn't affect Clickhouse (it corresponds to
 78        # the postgres-specific JSONBContains parser) and 2) it makes parsing the ternary op simpler.
 79        COLUMN_OPERATORS = parser.Parser.COLUMN_OPERATORS.copy()
 80        COLUMN_OPERATORS.pop(TokenType.PLACEHOLDER)
 81
 82        JOIN_KINDS = {
 83            *parser.Parser.JOIN_KINDS,
 84            TokenType.ANY,
 85            TokenType.ASOF,
 86            TokenType.ANTI,
 87            TokenType.SEMI,
 88        }
 89
 90        TABLE_ALIAS_TOKENS = {*parser.Parser.TABLE_ALIAS_TOKENS} - {
 91            TokenType.ANY,
 92            TokenType.ASOF,
 93            TokenType.SEMI,
 94            TokenType.ANTI,
 95            TokenType.SETTINGS,
 96            TokenType.FORMAT,
 97        }
 98
 99        LOG_DEFAULTS_TO_LN = True
100
101        QUERY_MODIFIER_PARSERS = {
102            **parser.Parser.QUERY_MODIFIER_PARSERS,
103            "settings": lambda self: self._parse_csv(self._parse_conjunction)
104            if self._match(TokenType.SETTINGS)
105            else None,
106            "format": lambda self: self._parse_id_var() if self._match(TokenType.FORMAT) else None,
107        }
108
109        def _parse_expression(self, explicit_alias: bool = False) -> t.Optional[exp.Expression]:
110            return self._parse_alias(self._parse_ternary(), explicit=explicit_alias)
111
112        def _parse_ternary(self) -> t.Optional[exp.Expression]:
113            this = self._parse_conjunction()
114
115            if self._match(TokenType.PLACEHOLDER):
116                return self.expression(
117                    exp.If,
118                    this=this,
119                    true=self._parse_conjunction(),
120                    false=self._match(TokenType.COLON) and self._parse_conjunction(),
121                )
122
123            return this
124
125        def _parse_placeholder(self) -> t.Optional[exp.Expression]:
126            """
127            Parse a placeholder expression like SELECT {abc: UInt32} or FROM {table: Identifier}
128            https://clickhouse.com/docs/en/sql-reference/syntax#defining-and-using-query-parameters
129            """
130            if not self._match(TokenType.L_BRACE):
131                return None
132
133            this = self._parse_id_var()
134            self._match(TokenType.COLON)
135            kind = self._parse_types(check_func=False) or (
136                self._match_text_seq("IDENTIFIER") and "Identifier"
137            )
138
139            if not kind:
140                self.raise_error("Expecting a placeholder type or 'Identifier' for tables")
141            elif not self._match(TokenType.R_BRACE):
142                self.raise_error("Expecting }")
143
144            return self.expression(exp.Placeholder, this=this, kind=kind)
145
146        def _parse_in(
147            self, this: t.Optional[exp.Expression], is_global: bool = False
148        ) -> exp.Expression:
149            this = super()._parse_in(this)
150            this.set("is_global", is_global)
151            return this
152
153        def _parse_table(
154            self, schema: bool = False, alias_tokens: t.Optional[t.Collection[TokenType]] = None
155        ) -> t.Optional[exp.Expression]:
156            this = super()._parse_table(schema=schema, alias_tokens=alias_tokens)
157
158            if self._match(TokenType.FINAL):
159                this = self.expression(exp.Final, this=this)
160
161            return this
162
163        def _parse_position(self, haystack_first: bool = False) -> exp.Expression:
164            return super()._parse_position(haystack_first=True)
165
166        # https://clickhouse.com/docs/en/sql-reference/statements/select/with/
167        def _parse_cte(self) -> exp.Expression:
168            index = self._index
169            try:
170                # WITH <identifier> AS <subquery expression>
171                return super()._parse_cte()
172            except ParseError:
173                # WITH <expression> AS <identifier>
174                self._retreat(index)
175                statement = self._parse_statement()
176
177                if statement and isinstance(statement.this, exp.Alias):
178                    self.raise_error("Expected CTE to have alias")
179
180                return self.expression(exp.CTE, this=statement, alias=statement and statement.this)
181
182        def _parse_join_side_and_kind(
183            self,
184        ) -> t.Tuple[t.Optional[Token], t.Optional[Token], t.Optional[Token]]:
185            is_global = self._match(TokenType.GLOBAL) and self._prev
186            kind_pre = self._match_set(self.JOIN_KINDS, advance=False) and self._prev
187            if kind_pre:
188                kind = self._match_set(self.JOIN_KINDS) and self._prev
189                side = self._match_set(self.JOIN_SIDES) and self._prev
190                return is_global, side, kind
191            return (
192                is_global,
193                self._match_set(self.JOIN_SIDES) and self._prev,
194                self._match_set(self.JOIN_KINDS) and self._prev,
195            )
196
197        def _parse_join(self, skip_join_token: bool = False) -> t.Optional[exp.Expression]:
198            join = super()._parse_join(skip_join_token)
199
200            if join:
201                join.set("global", join.args.pop("natural", None))
202            return join
203
204        def _parse_function(
205            self, functions: t.Optional[t.Dict[str, t.Callable]] = None, anonymous: bool = False
206        ) -> t.Optional[exp.Expression]:
207            func = super()._parse_function(functions, anonymous)
208
209            if isinstance(func, exp.Anonymous):
210                params = self._parse_func_params(func)
211
212                if params:
213                    return self.expression(
214                        exp.ParameterizedAgg,
215                        this=func.this,
216                        expressions=func.expressions,
217                        params=params,
218                    )
219
220            return func
221
222        def _parse_func_params(
223            self, this: t.Optional[exp.Func] = None
224        ) -> t.Optional[t.List[t.Optional[exp.Expression]]]:
225            if self._match_pair(TokenType.R_PAREN, TokenType.L_PAREN):
226                return self._parse_csv(self._parse_lambda)
227            if self._match(TokenType.L_PAREN):
228                params = self._parse_csv(self._parse_lambda)
229                self._match_r_paren(this)
230                return params
231            return None
232
233        def _parse_quantile(self) -> exp.Quantile:
234            this = self._parse_lambda()
235            params = self._parse_func_params()
236            if params:
237                return self.expression(exp.Quantile, this=params[0], quantile=this)
238            return self.expression(exp.Quantile, this=this, quantile=exp.Literal.number(0.5))
239
240        def _parse_wrapped_id_vars(
241            self, optional: bool = False
242        ) -> t.List[t.Optional[exp.Expression]]:
243            return super()._parse_wrapped_id_vars(optional=True)
244
245    class Generator(generator.Generator):
246        STRUCT_DELIMITER = ("(", ")")
247
248        TYPE_MAPPING = {
249            **generator.Generator.TYPE_MAPPING,  # type: ignore
250            exp.DataType.Type.ARRAY: "Array",
251            exp.DataType.Type.BIGINT: "Int64",
252            exp.DataType.Type.DATETIME64: "DateTime64",
253            exp.DataType.Type.DOUBLE: "Float64",
254            exp.DataType.Type.FLOAT: "Float32",
255            exp.DataType.Type.INT: "Int32",
256            exp.DataType.Type.INT128: "Int128",
257            exp.DataType.Type.INT256: "Int256",
258            exp.DataType.Type.MAP: "Map",
259            exp.DataType.Type.NULLABLE: "Nullable",
260            exp.DataType.Type.SMALLINT: "Int16",
261            exp.DataType.Type.STRUCT: "Tuple",
262            exp.DataType.Type.TINYINT: "Int8",
263            exp.DataType.Type.UBIGINT: "UInt64",
264            exp.DataType.Type.UINT: "UInt32",
265            exp.DataType.Type.UINT128: "UInt128",
266            exp.DataType.Type.UINT256: "UInt256",
267            exp.DataType.Type.USMALLINT: "UInt16",
268            exp.DataType.Type.UTINYINT: "UInt8",
269        }
270
271        TRANSFORMS = {
272            **generator.Generator.TRANSFORMS,  # type: ignore
273            exp.AnyValue: rename_func("any"),
274            exp.ApproxDistinct: rename_func("uniq"),
275            exp.Array: inline_array_sql,
276            exp.CastToStrType: rename_func("CAST"),
277            exp.Final: lambda self, e: f"{self.sql(e, 'this')} FINAL",
278            exp.Map: lambda self, e: _lower_func(var_map_sql(self, e)),
279            exp.PartitionedByProperty: lambda self, e: f"PARTITION BY {self.sql(e, 'this')}",
280            exp.Quantile: lambda self, e: self.func("quantile", e.args.get("quantile"))
281            + f"({self.sql(e, 'this')})",
282            exp.RegexpLike: lambda self, e: f"match({self.format_args(e.this, e.expression)})",
283            exp.StrPosition: lambda self, e: f"position({self.format_args(e.this, e.args.get('substr'), e.args.get('position'))})",
284            exp.VarMap: lambda self, e: _lower_func(var_map_sql(self, e)),
285        }
286
287        PROPERTIES_LOCATION = {
288            **generator.Generator.PROPERTIES_LOCATION,  # type: ignore
289            exp.VolatileProperty: exp.Properties.Location.UNSUPPORTED,
290            exp.PartitionedByProperty: exp.Properties.Location.POST_SCHEMA,
291        }
292
293        JOIN_HINTS = False
294        TABLE_HINTS = False
295        EXPLICIT_UNION = True
296        GROUPINGS_SEP = ""
297
298        def cte_sql(self, expression: exp.CTE) -> str:
299            if isinstance(expression.this, exp.Alias):
300                return self.sql(expression, "this")
301
302            return super().cte_sql(expression)
303
304        def after_limit_modifiers(self, expression):
305            return super().after_limit_modifiers(expression) + [
306                self.seg("SETTINGS ") + self.expressions(expression, key="settings", flat=True)
307                if expression.args.get("settings")
308                else "",
309                self.seg("FORMAT ") + self.sql(expression, "format")
310                if expression.args.get("format")
311                else "",
312            ]
313
314        def parameterizedagg_sql(self, expression: exp.Anonymous) -> str:
315            params = self.expressions(expression, "params", flat=True)
316            return self.func(expression.name, *expression.expressions) + f"({params})"
317
318        def placeholder_sql(self, expression: exp.Placeholder) -> str:
319            return f"{{{expression.name}: {self.sql(expression, 'kind')}}}"
class ClickHouse(sqlglot.dialects.dialect.Dialect):
 18class ClickHouse(Dialect):
 19    normalize_functions = None
 20    null_ordering = "nulls_are_last"
 21
 22    class Tokenizer(tokens.Tokenizer):
 23        COMMENTS = ["--", "#", "#!", ("/*", "*/")]
 24        IDENTIFIERS = ['"', "`"]
 25        BIT_STRINGS = [("0b", "")]
 26        HEX_STRINGS = [("0x", ""), ("0X", "")]
 27
 28        KEYWORDS = {
 29            **tokens.Tokenizer.KEYWORDS,
 30            "ASOF": TokenType.ASOF,
 31            "ATTACH": TokenType.COMMAND,
 32            "DATETIME64": TokenType.DATETIME64,
 33            "FINAL": TokenType.FINAL,
 34            "FLOAT32": TokenType.FLOAT,
 35            "FLOAT64": TokenType.DOUBLE,
 36            "GLOBAL": TokenType.GLOBAL,
 37            "INT128": TokenType.INT128,
 38            "INT16": TokenType.SMALLINT,
 39            "INT256": TokenType.INT256,
 40            "INT32": TokenType.INT,
 41            "INT64": TokenType.BIGINT,
 42            "INT8": TokenType.TINYINT,
 43            "MAP": TokenType.MAP,
 44            "TUPLE": TokenType.STRUCT,
 45            "UINT128": TokenType.UINT128,
 46            "UINT16": TokenType.USMALLINT,
 47            "UINT256": TokenType.UINT256,
 48            "UINT32": TokenType.UINT,
 49            "UINT64": TokenType.UBIGINT,
 50            "UINT8": TokenType.UTINYINT,
 51        }
 52
 53    class Parser(parser.Parser):
 54        FUNCTIONS = {
 55            **parser.Parser.FUNCTIONS,  # type: ignore
 56            "ANY": exp.AnyValue.from_arg_list,
 57            "MAP": parse_var_map,
 58            "MATCH": exp.RegexpLike.from_arg_list,
 59            "UNIQ": exp.ApproxDistinct.from_arg_list,
 60        }
 61
 62        FUNCTION_PARSERS = {
 63            **parser.Parser.FUNCTION_PARSERS,
 64            "QUANTILE": lambda self: self._parse_quantile(),
 65        }
 66
 67        FUNCTION_PARSERS.pop("MATCH")
 68
 69        NO_PAREN_FUNCTION_PARSERS = parser.Parser.NO_PAREN_FUNCTION_PARSERS.copy()
 70        NO_PAREN_FUNCTION_PARSERS.pop(TokenType.ANY)
 71
 72        RANGE_PARSERS = {
 73            **parser.Parser.RANGE_PARSERS,
 74            TokenType.GLOBAL: lambda self, this: self._match(TokenType.IN)
 75            and self._parse_in(this, is_global=True),
 76        }
 77
 78        # The PLACEHOLDER entry is popped because 1) it doesn't affect Clickhouse (it corresponds to
 79        # the postgres-specific JSONBContains parser) and 2) it makes parsing the ternary op simpler.
 80        COLUMN_OPERATORS = parser.Parser.COLUMN_OPERATORS.copy()
 81        COLUMN_OPERATORS.pop(TokenType.PLACEHOLDER)
 82
 83        JOIN_KINDS = {
 84            *parser.Parser.JOIN_KINDS,
 85            TokenType.ANY,
 86            TokenType.ASOF,
 87            TokenType.ANTI,
 88            TokenType.SEMI,
 89        }
 90
 91        TABLE_ALIAS_TOKENS = {*parser.Parser.TABLE_ALIAS_TOKENS} - {
 92            TokenType.ANY,
 93            TokenType.ASOF,
 94            TokenType.SEMI,
 95            TokenType.ANTI,
 96            TokenType.SETTINGS,
 97            TokenType.FORMAT,
 98        }
 99
100        LOG_DEFAULTS_TO_LN = True
101
102        QUERY_MODIFIER_PARSERS = {
103            **parser.Parser.QUERY_MODIFIER_PARSERS,
104            "settings": lambda self: self._parse_csv(self._parse_conjunction)
105            if self._match(TokenType.SETTINGS)
106            else None,
107            "format": lambda self: self._parse_id_var() if self._match(TokenType.FORMAT) else None,
108        }
109
110        def _parse_expression(self, explicit_alias: bool = False) -> t.Optional[exp.Expression]:
111            return self._parse_alias(self._parse_ternary(), explicit=explicit_alias)
112
113        def _parse_ternary(self) -> t.Optional[exp.Expression]:
114            this = self._parse_conjunction()
115
116            if self._match(TokenType.PLACEHOLDER):
117                return self.expression(
118                    exp.If,
119                    this=this,
120                    true=self._parse_conjunction(),
121                    false=self._match(TokenType.COLON) and self._parse_conjunction(),
122                )
123
124            return this
125
126        def _parse_placeholder(self) -> t.Optional[exp.Expression]:
127            """
128            Parse a placeholder expression like SELECT {abc: UInt32} or FROM {table: Identifier}
129            https://clickhouse.com/docs/en/sql-reference/syntax#defining-and-using-query-parameters
130            """
131            if not self._match(TokenType.L_BRACE):
132                return None
133
134            this = self._parse_id_var()
135            self._match(TokenType.COLON)
136            kind = self._parse_types(check_func=False) or (
137                self._match_text_seq("IDENTIFIER") and "Identifier"
138            )
139
140            if not kind:
141                self.raise_error("Expecting a placeholder type or 'Identifier' for tables")
142            elif not self._match(TokenType.R_BRACE):
143                self.raise_error("Expecting }")
144
145            return self.expression(exp.Placeholder, this=this, kind=kind)
146
147        def _parse_in(
148            self, this: t.Optional[exp.Expression], is_global: bool = False
149        ) -> exp.Expression:
150            this = super()._parse_in(this)
151            this.set("is_global", is_global)
152            return this
153
154        def _parse_table(
155            self, schema: bool = False, alias_tokens: t.Optional[t.Collection[TokenType]] = None
156        ) -> t.Optional[exp.Expression]:
157            this = super()._parse_table(schema=schema, alias_tokens=alias_tokens)
158
159            if self._match(TokenType.FINAL):
160                this = self.expression(exp.Final, this=this)
161
162            return this
163
164        def _parse_position(self, haystack_first: bool = False) -> exp.Expression:
165            return super()._parse_position(haystack_first=True)
166
167        # https://clickhouse.com/docs/en/sql-reference/statements/select/with/
168        def _parse_cte(self) -> exp.Expression:
169            index = self._index
170            try:
171                # WITH <identifier> AS <subquery expression>
172                return super()._parse_cte()
173            except ParseError:
174                # WITH <expression> AS <identifier>
175                self._retreat(index)
176                statement = self._parse_statement()
177
178                if statement and isinstance(statement.this, exp.Alias):
179                    self.raise_error("Expected CTE to have alias")
180
181                return self.expression(exp.CTE, this=statement, alias=statement and statement.this)
182
183        def _parse_join_side_and_kind(
184            self,
185        ) -> t.Tuple[t.Optional[Token], t.Optional[Token], t.Optional[Token]]:
186            is_global = self._match(TokenType.GLOBAL) and self._prev
187            kind_pre = self._match_set(self.JOIN_KINDS, advance=False) and self._prev
188            if kind_pre:
189                kind = self._match_set(self.JOIN_KINDS) and self._prev
190                side = self._match_set(self.JOIN_SIDES) and self._prev
191                return is_global, side, kind
192            return (
193                is_global,
194                self._match_set(self.JOIN_SIDES) and self._prev,
195                self._match_set(self.JOIN_KINDS) and self._prev,
196            )
197
198        def _parse_join(self, skip_join_token: bool = False) -> t.Optional[exp.Expression]:
199            join = super()._parse_join(skip_join_token)
200
201            if join:
202                join.set("global", join.args.pop("natural", None))
203            return join
204
205        def _parse_function(
206            self, functions: t.Optional[t.Dict[str, t.Callable]] = None, anonymous: bool = False
207        ) -> t.Optional[exp.Expression]:
208            func = super()._parse_function(functions, anonymous)
209
210            if isinstance(func, exp.Anonymous):
211                params = self._parse_func_params(func)
212
213                if params:
214                    return self.expression(
215                        exp.ParameterizedAgg,
216                        this=func.this,
217                        expressions=func.expressions,
218                        params=params,
219                    )
220
221            return func
222
223        def _parse_func_params(
224            self, this: t.Optional[exp.Func] = None
225        ) -> t.Optional[t.List[t.Optional[exp.Expression]]]:
226            if self._match_pair(TokenType.R_PAREN, TokenType.L_PAREN):
227                return self._parse_csv(self._parse_lambda)
228            if self._match(TokenType.L_PAREN):
229                params = self._parse_csv(self._parse_lambda)
230                self._match_r_paren(this)
231                return params
232            return None
233
234        def _parse_quantile(self) -> exp.Quantile:
235            this = self._parse_lambda()
236            params = self._parse_func_params()
237            if params:
238                return self.expression(exp.Quantile, this=params[0], quantile=this)
239            return self.expression(exp.Quantile, this=this, quantile=exp.Literal.number(0.5))
240
241        def _parse_wrapped_id_vars(
242            self, optional: bool = False
243        ) -> t.List[t.Optional[exp.Expression]]:
244            return super()._parse_wrapped_id_vars(optional=True)
245
246    class Generator(generator.Generator):
247        STRUCT_DELIMITER = ("(", ")")
248
249        TYPE_MAPPING = {
250            **generator.Generator.TYPE_MAPPING,  # type: ignore
251            exp.DataType.Type.ARRAY: "Array",
252            exp.DataType.Type.BIGINT: "Int64",
253            exp.DataType.Type.DATETIME64: "DateTime64",
254            exp.DataType.Type.DOUBLE: "Float64",
255            exp.DataType.Type.FLOAT: "Float32",
256            exp.DataType.Type.INT: "Int32",
257            exp.DataType.Type.INT128: "Int128",
258            exp.DataType.Type.INT256: "Int256",
259            exp.DataType.Type.MAP: "Map",
260            exp.DataType.Type.NULLABLE: "Nullable",
261            exp.DataType.Type.SMALLINT: "Int16",
262            exp.DataType.Type.STRUCT: "Tuple",
263            exp.DataType.Type.TINYINT: "Int8",
264            exp.DataType.Type.UBIGINT: "UInt64",
265            exp.DataType.Type.UINT: "UInt32",
266            exp.DataType.Type.UINT128: "UInt128",
267            exp.DataType.Type.UINT256: "UInt256",
268            exp.DataType.Type.USMALLINT: "UInt16",
269            exp.DataType.Type.UTINYINT: "UInt8",
270        }
271
272        TRANSFORMS = {
273            **generator.Generator.TRANSFORMS,  # type: ignore
274            exp.AnyValue: rename_func("any"),
275            exp.ApproxDistinct: rename_func("uniq"),
276            exp.Array: inline_array_sql,
277            exp.CastToStrType: rename_func("CAST"),
278            exp.Final: lambda self, e: f"{self.sql(e, 'this')} FINAL",
279            exp.Map: lambda self, e: _lower_func(var_map_sql(self, e)),
280            exp.PartitionedByProperty: lambda self, e: f"PARTITION BY {self.sql(e, 'this')}",
281            exp.Quantile: lambda self, e: self.func("quantile", e.args.get("quantile"))
282            + f"({self.sql(e, 'this')})",
283            exp.RegexpLike: lambda self, e: f"match({self.format_args(e.this, e.expression)})",
284            exp.StrPosition: lambda self, e: f"position({self.format_args(e.this, e.args.get('substr'), e.args.get('position'))})",
285            exp.VarMap: lambda self, e: _lower_func(var_map_sql(self, e)),
286        }
287
288        PROPERTIES_LOCATION = {
289            **generator.Generator.PROPERTIES_LOCATION,  # type: ignore
290            exp.VolatileProperty: exp.Properties.Location.UNSUPPORTED,
291            exp.PartitionedByProperty: exp.Properties.Location.POST_SCHEMA,
292        }
293
294        JOIN_HINTS = False
295        TABLE_HINTS = False
296        EXPLICIT_UNION = True
297        GROUPINGS_SEP = ""
298
299        def cte_sql(self, expression: exp.CTE) -> str:
300            if isinstance(expression.this, exp.Alias):
301                return self.sql(expression, "this")
302
303            return super().cte_sql(expression)
304
305        def after_limit_modifiers(self, expression):
306            return super().after_limit_modifiers(expression) + [
307                self.seg("SETTINGS ") + self.expressions(expression, key="settings", flat=True)
308                if expression.args.get("settings")
309                else "",
310                self.seg("FORMAT ") + self.sql(expression, "format")
311                if expression.args.get("format")
312                else "",
313            ]
314
315        def parameterizedagg_sql(self, expression: exp.Anonymous) -> str:
316            params = self.expressions(expression, "params", flat=True)
317            return self.func(expression.name, *expression.expressions) + f"({params})"
318
319        def placeholder_sql(self, expression: exp.Placeholder) -> str:
320            return f"{{{expression.name}: {self.sql(expression, 'kind')}}}"
class ClickHouse.Tokenizer(sqlglot.tokens.Tokenizer):
22    class Tokenizer(tokens.Tokenizer):
23        COMMENTS = ["--", "#", "#!", ("/*", "*/")]
24        IDENTIFIERS = ['"', "`"]
25        BIT_STRINGS = [("0b", "")]
26        HEX_STRINGS = [("0x", ""), ("0X", "")]
27
28        KEYWORDS = {
29            **tokens.Tokenizer.KEYWORDS,
30            "ASOF": TokenType.ASOF,
31            "ATTACH": TokenType.COMMAND,
32            "DATETIME64": TokenType.DATETIME64,
33            "FINAL": TokenType.FINAL,
34            "FLOAT32": TokenType.FLOAT,
35            "FLOAT64": TokenType.DOUBLE,
36            "GLOBAL": TokenType.GLOBAL,
37            "INT128": TokenType.INT128,
38            "INT16": TokenType.SMALLINT,
39            "INT256": TokenType.INT256,
40            "INT32": TokenType.INT,
41            "INT64": TokenType.BIGINT,
42            "INT8": TokenType.TINYINT,
43            "MAP": TokenType.MAP,
44            "TUPLE": TokenType.STRUCT,
45            "UINT128": TokenType.UINT128,
46            "UINT16": TokenType.USMALLINT,
47            "UINT256": TokenType.UINT256,
48            "UINT32": TokenType.UINT,
49            "UINT64": TokenType.UBIGINT,
50            "UINT8": TokenType.UTINYINT,
51        }
class ClickHouse.Parser(sqlglot.parser.Parser):
 53    class Parser(parser.Parser):
 54        FUNCTIONS = {
 55            **parser.Parser.FUNCTIONS,  # type: ignore
 56            "ANY": exp.AnyValue.from_arg_list,
 57            "MAP": parse_var_map,
 58            "MATCH": exp.RegexpLike.from_arg_list,
 59            "UNIQ": exp.ApproxDistinct.from_arg_list,
 60        }
 61
 62        FUNCTION_PARSERS = {
 63            **parser.Parser.FUNCTION_PARSERS,
 64            "QUANTILE": lambda self: self._parse_quantile(),
 65        }
 66
 67        FUNCTION_PARSERS.pop("MATCH")
 68
 69        NO_PAREN_FUNCTION_PARSERS = parser.Parser.NO_PAREN_FUNCTION_PARSERS.copy()
 70        NO_PAREN_FUNCTION_PARSERS.pop(TokenType.ANY)
 71
 72        RANGE_PARSERS = {
 73            **parser.Parser.RANGE_PARSERS,
 74            TokenType.GLOBAL: lambda self, this: self._match(TokenType.IN)
 75            and self._parse_in(this, is_global=True),
 76        }
 77
 78        # The PLACEHOLDER entry is popped because 1) it doesn't affect Clickhouse (it corresponds to
 79        # the postgres-specific JSONBContains parser) and 2) it makes parsing the ternary op simpler.
 80        COLUMN_OPERATORS = parser.Parser.COLUMN_OPERATORS.copy()
 81        COLUMN_OPERATORS.pop(TokenType.PLACEHOLDER)
 82
 83        JOIN_KINDS = {
 84            *parser.Parser.JOIN_KINDS,
 85            TokenType.ANY,
 86            TokenType.ASOF,
 87            TokenType.ANTI,
 88            TokenType.SEMI,
 89        }
 90
 91        TABLE_ALIAS_TOKENS = {*parser.Parser.TABLE_ALIAS_TOKENS} - {
 92            TokenType.ANY,
 93            TokenType.ASOF,
 94            TokenType.SEMI,
 95            TokenType.ANTI,
 96            TokenType.SETTINGS,
 97            TokenType.FORMAT,
 98        }
 99
100        LOG_DEFAULTS_TO_LN = True
101
102        QUERY_MODIFIER_PARSERS = {
103            **parser.Parser.QUERY_MODIFIER_PARSERS,
104            "settings": lambda self: self._parse_csv(self._parse_conjunction)
105            if self._match(TokenType.SETTINGS)
106            else None,
107            "format": lambda self: self._parse_id_var() if self._match(TokenType.FORMAT) else None,
108        }
109
110        def _parse_expression(self, explicit_alias: bool = False) -> t.Optional[exp.Expression]:
111            return self._parse_alias(self._parse_ternary(), explicit=explicit_alias)
112
113        def _parse_ternary(self) -> t.Optional[exp.Expression]:
114            this = self._parse_conjunction()
115
116            if self._match(TokenType.PLACEHOLDER):
117                return self.expression(
118                    exp.If,
119                    this=this,
120                    true=self._parse_conjunction(),
121                    false=self._match(TokenType.COLON) and self._parse_conjunction(),
122                )
123
124            return this
125
126        def _parse_placeholder(self) -> t.Optional[exp.Expression]:
127            """
128            Parse a placeholder expression like SELECT {abc: UInt32} or FROM {table: Identifier}
129            https://clickhouse.com/docs/en/sql-reference/syntax#defining-and-using-query-parameters
130            """
131            if not self._match(TokenType.L_BRACE):
132                return None
133
134            this = self._parse_id_var()
135            self._match(TokenType.COLON)
136            kind = self._parse_types(check_func=False) or (
137                self._match_text_seq("IDENTIFIER") and "Identifier"
138            )
139
140            if not kind:
141                self.raise_error("Expecting a placeholder type or 'Identifier' for tables")
142            elif not self._match(TokenType.R_BRACE):
143                self.raise_error("Expecting }")
144
145            return self.expression(exp.Placeholder, this=this, kind=kind)
146
147        def _parse_in(
148            self, this: t.Optional[exp.Expression], is_global: bool = False
149        ) -> exp.Expression:
150            this = super()._parse_in(this)
151            this.set("is_global", is_global)
152            return this
153
154        def _parse_table(
155            self, schema: bool = False, alias_tokens: t.Optional[t.Collection[TokenType]] = None
156        ) -> t.Optional[exp.Expression]:
157            this = super()._parse_table(schema=schema, alias_tokens=alias_tokens)
158
159            if self._match(TokenType.FINAL):
160                this = self.expression(exp.Final, this=this)
161
162            return this
163
164        def _parse_position(self, haystack_first: bool = False) -> exp.Expression:
165            return super()._parse_position(haystack_first=True)
166
167        # https://clickhouse.com/docs/en/sql-reference/statements/select/with/
168        def _parse_cte(self) -> exp.Expression:
169            index = self._index
170            try:
171                # WITH <identifier> AS <subquery expression>
172                return super()._parse_cte()
173            except ParseError:
174                # WITH <expression> AS <identifier>
175                self._retreat(index)
176                statement = self._parse_statement()
177
178                if statement and isinstance(statement.this, exp.Alias):
179                    self.raise_error("Expected CTE to have alias")
180
181                return self.expression(exp.CTE, this=statement, alias=statement and statement.this)
182
183        def _parse_join_side_and_kind(
184            self,
185        ) -> t.Tuple[t.Optional[Token], t.Optional[Token], t.Optional[Token]]:
186            is_global = self._match(TokenType.GLOBAL) and self._prev
187            kind_pre = self._match_set(self.JOIN_KINDS, advance=False) and self._prev
188            if kind_pre:
189                kind = self._match_set(self.JOIN_KINDS) and self._prev
190                side = self._match_set(self.JOIN_SIDES) and self._prev
191                return is_global, side, kind
192            return (
193                is_global,
194                self._match_set(self.JOIN_SIDES) and self._prev,
195                self._match_set(self.JOIN_KINDS) and self._prev,
196            )
197
198        def _parse_join(self, skip_join_token: bool = False) -> t.Optional[exp.Expression]:
199            join = super()._parse_join(skip_join_token)
200
201            if join:
202                join.set("global", join.args.pop("natural", None))
203            return join
204
205        def _parse_function(
206            self, functions: t.Optional[t.Dict[str, t.Callable]] = None, anonymous: bool = False
207        ) -> t.Optional[exp.Expression]:
208            func = super()._parse_function(functions, anonymous)
209
210            if isinstance(func, exp.Anonymous):
211                params = self._parse_func_params(func)
212
213                if params:
214                    return self.expression(
215                        exp.ParameterizedAgg,
216                        this=func.this,
217                        expressions=func.expressions,
218                        params=params,
219                    )
220
221            return func
222
223        def _parse_func_params(
224            self, this: t.Optional[exp.Func] = None
225        ) -> t.Optional[t.List[t.Optional[exp.Expression]]]:
226            if self._match_pair(TokenType.R_PAREN, TokenType.L_PAREN):
227                return self._parse_csv(self._parse_lambda)
228            if self._match(TokenType.L_PAREN):
229                params = self._parse_csv(self._parse_lambda)
230                self._match_r_paren(this)
231                return params
232            return None
233
234        def _parse_quantile(self) -> exp.Quantile:
235            this = self._parse_lambda()
236            params = self._parse_func_params()
237            if params:
238                return self.expression(exp.Quantile, this=params[0], quantile=this)
239            return self.expression(exp.Quantile, this=this, quantile=exp.Literal.number(0.5))
240
241        def _parse_wrapped_id_vars(
242            self, optional: bool = False
243        ) -> t.List[t.Optional[exp.Expression]]:
244            return super()._parse_wrapped_id_vars(optional=True)

Parser consumes a list of tokens produced by the sqlglot.tokens.Tokenizer and produces a parsed syntax tree.

Arguments:
  • error_level: the desired error level. Default: ErrorLevel.RAISE
  • error_message_context: determines the amount of context to capture from a query string when displaying the error message (in number of characters). Default: 50.
  • index_offset: Index offset for arrays eg ARRAY[0] vs ARRAY[1] as the head of a list. Default: 0
  • alias_post_tablesample: If the table alias comes after tablesample. Default: False
  • max_errors: Maximum number of error messages to include in a raised ParseError. This is only relevant if error_level is ErrorLevel.RAISE. Default: 3
  • null_ordering: Indicates the default null ordering method to use if not explicitly set. Options are "nulls_are_small", "nulls_are_large", "nulls_are_last". Default: "nulls_are_small"
class ClickHouse.Generator(sqlglot.generator.Generator):
246    class Generator(generator.Generator):
247        STRUCT_DELIMITER = ("(", ")")
248
249        TYPE_MAPPING = {
250            **generator.Generator.TYPE_MAPPING,  # type: ignore
251            exp.DataType.Type.ARRAY: "Array",
252            exp.DataType.Type.BIGINT: "Int64",
253            exp.DataType.Type.DATETIME64: "DateTime64",
254            exp.DataType.Type.DOUBLE: "Float64",
255            exp.DataType.Type.FLOAT: "Float32",
256            exp.DataType.Type.INT: "Int32",
257            exp.DataType.Type.INT128: "Int128",
258            exp.DataType.Type.INT256: "Int256",
259            exp.DataType.Type.MAP: "Map",
260            exp.DataType.Type.NULLABLE: "Nullable",
261            exp.DataType.Type.SMALLINT: "Int16",
262            exp.DataType.Type.STRUCT: "Tuple",
263            exp.DataType.Type.TINYINT: "Int8",
264            exp.DataType.Type.UBIGINT: "UInt64",
265            exp.DataType.Type.UINT: "UInt32",
266            exp.DataType.Type.UINT128: "UInt128",
267            exp.DataType.Type.UINT256: "UInt256",
268            exp.DataType.Type.USMALLINT: "UInt16",
269            exp.DataType.Type.UTINYINT: "UInt8",
270        }
271
272        TRANSFORMS = {
273            **generator.Generator.TRANSFORMS,  # type: ignore
274            exp.AnyValue: rename_func("any"),
275            exp.ApproxDistinct: rename_func("uniq"),
276            exp.Array: inline_array_sql,
277            exp.CastToStrType: rename_func("CAST"),
278            exp.Final: lambda self, e: f"{self.sql(e, 'this')} FINAL",
279            exp.Map: lambda self, e: _lower_func(var_map_sql(self, e)),
280            exp.PartitionedByProperty: lambda self, e: f"PARTITION BY {self.sql(e, 'this')}",
281            exp.Quantile: lambda self, e: self.func("quantile", e.args.get("quantile"))
282            + f"({self.sql(e, 'this')})",
283            exp.RegexpLike: lambda self, e: f"match({self.format_args(e.this, e.expression)})",
284            exp.StrPosition: lambda self, e: f"position({self.format_args(e.this, e.args.get('substr'), e.args.get('position'))})",
285            exp.VarMap: lambda self, e: _lower_func(var_map_sql(self, e)),
286        }
287
288        PROPERTIES_LOCATION = {
289            **generator.Generator.PROPERTIES_LOCATION,  # type: ignore
290            exp.VolatileProperty: exp.Properties.Location.UNSUPPORTED,
291            exp.PartitionedByProperty: exp.Properties.Location.POST_SCHEMA,
292        }
293
294        JOIN_HINTS = False
295        TABLE_HINTS = False
296        EXPLICIT_UNION = True
297        GROUPINGS_SEP = ""
298
299        def cte_sql(self, expression: exp.CTE) -> str:
300            if isinstance(expression.this, exp.Alias):
301                return self.sql(expression, "this")
302
303            return super().cte_sql(expression)
304
305        def after_limit_modifiers(self, expression):
306            return super().after_limit_modifiers(expression) + [
307                self.seg("SETTINGS ") + self.expressions(expression, key="settings", flat=True)
308                if expression.args.get("settings")
309                else "",
310                self.seg("FORMAT ") + self.sql(expression, "format")
311                if expression.args.get("format")
312                else "",
313            ]
314
315        def parameterizedagg_sql(self, expression: exp.Anonymous) -> str:
316            params = self.expressions(expression, "params", flat=True)
317            return self.func(expression.name, *expression.expressions) + f"({params})"
318
319        def placeholder_sql(self, expression: exp.Placeholder) -> str:
320            return f"{{{expression.name}: {self.sql(expression, 'kind')}}}"

Generator interprets the given syntax tree and produces a SQL string as an output.

Arguments:
  • time_mapping (dict): the dictionary of custom time mappings in which the key represents a python time format and the output the target time format
  • time_trie (trie): a trie of the time_mapping keys
  • pretty (bool): if set to True the returned string will be formatted. Default: False.
  • quote_start (str): specifies which starting character to use to delimit quotes. Default: '.
  • quote_end (str): specifies which ending character to use to delimit quotes. Default: '.
  • identifier_start (str): specifies which starting character to use to delimit identifiers. Default: ".
  • identifier_end (str): specifies which ending character to use to delimit identifiers. Default: ".
  • bit_start (str): specifies which starting character to use to delimit bit literals. Default: None.
  • bit_end (str): specifies which ending character to use to delimit bit literals. Default: None.
  • hex_start (str): specifies which starting character to use to delimit hex literals. Default: None.
  • hex_end (str): specifies which ending character to use to delimit hex literals. Default: None.
  • byte_start (str): specifies which starting character to use to delimit byte literals. Default: None.
  • byte_end (str): specifies which ending character to use to delimit byte literals. Default: None.
  • identify (bool | str): 'always': always quote, 'safe': quote identifiers if they don't contain an upcase, True defaults to always.
  • normalize (bool): if set to True all identifiers will lower cased
  • string_escape (str): specifies a string escape character. Default: '.
  • identifier_escape (str): specifies an identifier escape character. Default: ".
  • pad (int): determines padding in a formatted string. Default: 2.
  • indent (int): determines the size of indentation in a formatted string. Default: 4.
  • unnest_column_only (bool): if true unnest table aliases are considered only as column aliases
  • normalize_functions (str): normalize function names, "upper", "lower", or None Default: "upper"
  • alias_post_tablesample (bool): if the table alias comes after tablesample Default: False
  • unsupported_level (ErrorLevel): determines the generator's behavior when it encounters unsupported expressions. Default ErrorLevel.WARN.
  • null_ordering (str): Indicates the default null ordering method to use if not explicitly set. Options are "nulls_are_small", "nulls_are_large", "nulls_are_last". Default: "nulls_are_small"
  • max_unsupported (int): Maximum number of unsupported messages to include in a raised UnsupportedError. This is only relevant if unsupported_level is ErrorLevel.RAISE. Default: 3
  • leading_comma (bool): if the the comma is leading or trailing in select statements Default: False
  • max_text_width: The max number of characters in a segment before creating new lines in pretty mode. The default is on the smaller end because the length only represents a segment and not the true line length. Default: 80
  • comments: Whether or not to preserve comments in the output SQL code. Default: True
def cte_sql(self, expression: sqlglot.expressions.CTE) -> str:
299        def cte_sql(self, expression: exp.CTE) -> str:
300            if isinstance(expression.this, exp.Alias):
301                return self.sql(expression, "this")
302
303            return super().cte_sql(expression)
def after_limit_modifiers(self, expression):
305        def after_limit_modifiers(self, expression):
306            return super().after_limit_modifiers(expression) + [
307                self.seg("SETTINGS ") + self.expressions(expression, key="settings", flat=True)
308                if expression.args.get("settings")
309                else "",
310                self.seg("FORMAT ") + self.sql(expression, "format")
311                if expression.args.get("format")
312                else "",
313            ]
def parameterizedagg_sql(self, expression: sqlglot.expressions.Anonymous) -> str:
315        def parameterizedagg_sql(self, expression: exp.Anonymous) -> str:
316            params = self.expressions(expression, "params", flat=True)
317            return self.func(expression.name, *expression.expressions) + f"({params})"
def placeholder_sql(self, expression: sqlglot.expressions.Placeholder) -> str:
319        def placeholder_sql(self, expression: exp.Placeholder) -> str:
320            return f"{{{expression.name}: {self.sql(expression, 'kind')}}}"
Inherited Members
sqlglot.generator.Generator
Generator
generate
unsupported
sep
seg
pad_comment
maybe_comment
wrap
no_identify
normalize_func
indent
sql
uncache_sql
cache_sql
characterset_sql
column_sql
columnposition_sql
columndef_sql
columnconstraint_sql
autoincrementcolumnconstraint_sql
compresscolumnconstraint_sql
generatedasidentitycolumnconstraint_sql
notnullcolumnconstraint_sql
primarykeycolumnconstraint_sql
uniquecolumnconstraint_sql
create_sql
clone_sql
describe_sql
prepend_ctes
with_sql
tablealias_sql
bitstring_sql
hexstring_sql
bytestring_sql
datatypesize_sql
datatype_sql
directory_sql
delete_sql
drop_sql
except_sql
except_op
fetch_sql
filter_sql
hint_sql
index_sql
identifier_sql
inputoutputformat_sql
national_sql
partition_sql
properties_sql
root_properties
properties
with_properties
locate_properties
property_sql
likeproperty_sql
fallbackproperty_sql
journalproperty_sql
freespaceproperty_sql
afterjournalproperty_sql
checksumproperty_sql
mergeblockratioproperty_sql
datablocksizeproperty_sql
blockcompressionproperty_sql
isolatedloadingproperty_sql
lockingproperty_sql
withdataproperty_sql
insert_sql
intersect_sql
intersect_op
introducer_sql
pseudotype_sql
onconflict_sql
returning_sql
rowformatdelimitedproperty_sql
table_sql
tablesample_sql
pivot_sql
tuple_sql
update_sql
values_sql
var_sql
into_sql
from_sql
group_sql
having_sql
join_sql
lambda_sql
lateral_sql
limit_sql
offset_sql
setitem_sql
set_sql
pragma_sql
lock_sql
literal_sql
loaddata_sql
null_sql
boolean_sql
order_sql
cluster_sql
distribute_sql
sort_sql
ordered_sql
matchrecognize_sql
query_modifiers
after_having_modifiers
select_sql
schema_sql
star_sql
parameter_sql
sessionparameter_sql
subquery_sql
qualify_sql
union_sql
union_op
unnest_sql
where_sql
window_sql
partition_by_sql
windowspec_sql
withingroup_sql
between_sql
bracket_sql
all_sql
any_sql
exists_sql
case_sql
constraint_sql
nextvaluefor_sql
extract_sql
trim_sql
concat_sql
check_sql
foreignkey_sql
primarykey_sql
unique_sql
if_sql
matchagainst_sql
jsonkeyvalue_sql
jsonobject_sql
openjsoncolumndef_sql
openjson_sql
in_sql
in_unnest_op
interval_sql
return_sql
reference_sql
anonymous_sql
paren_sql
neg_sql
not_sql
alias_sql
aliases_sql
attimezone_sql
add_sql
and_sql
connector_sql
bitwiseand_sql
bitwiseleftshift_sql
bitwisenot_sql
bitwiseor_sql
bitwiserightshift_sql
bitwisexor_sql
cast_sql
currentdate_sql
collate_sql
command_sql
comment_sql
mergetreettlaction_sql
mergetreettl_sql
transaction_sql
commit_sql
rollback_sql
altercolumn_sql
renametable_sql
altertable_sql
droppartition_sql
addconstraint_sql
distinct_sql
ignorenulls_sql
respectnulls_sql
intdiv_sql
dpipe_sql
div_sql
overlaps_sql
distance_sql
dot_sql
eq_sql
escape_sql
glob_sql
gt_sql
gte_sql
ilike_sql
ilikeany_sql
is_sql
like_sql
likeany_sql
similarto_sql
lt_sql
lte_sql
mod_sql
mul_sql
neq_sql
nullsafeeq_sql
nullsafeneq_sql
or_sql
slice_sql
sub_sql
trycast_sql
use_sql
binary
function_fallback_sql
func
format_args
text_width
format_time
expressions
op_expressions
naked_property
set_operation
tag_sql
token_sql
userdefinedfunction_sql
joinhint_sql
kwarg_sql
when_sql
merge_sql
tochar_sql