From 087872e7e21fc776ac37f95fa41a488bc834f09f Mon Sep 17 00:00:00 2001 From: fivetran-amrutabhimsenayachit Date: Wed, 2 Sep 2026 12:32:58 -0400 Subject: [PATCH 1/5] feat(optimizer): Fix the json path with single quote issue --- sqlglot-integration-tests | 2 +- sqlglot/dialects/dialect.py | 8 ++++---- sqlglot/generator.py | 9 ++++----- sqlglot/generators/bigquery.py | 28 ++++++++++++++++++++++------ sqlglot/generators/databricks.py | 1 + tests/test_jsonpath.py | 11 ++++++++++- 6 files changed, 42 insertions(+), 17 deletions(-) diff --git a/sqlglot-integration-tests b/sqlglot-integration-tests index b4603785d5..efddb0acdf 160000 --- a/sqlglot-integration-tests +++ b/sqlglot-integration-tests @@ -1 +1 @@ -Subproject commit b4603785d5871f30f7aef44b79f7d2a70489d95c +Subproject commit efddb0acdfe0d367572bd8238bf1b3a1643400fd diff --git a/sqlglot/dialects/dialect.py b/sqlglot/dialects/dialect.py index d77b6d396d..ce316d2331 100644 --- a/sqlglot/dialects/dialect.py +++ b/sqlglot/dialects/dialect.py @@ -2285,15 +2285,15 @@ def _json_extract_segments(self: Generator, expression: JSON_EXTRACT_TYPE) -> st segments = [] for segment in path.expressions: - escape = segment.args.get("quoted") path = self.sql(segment) if path: if isinstance(segment, exp.JSONPathPart) and ( quoted_index or not isinstance(segment, exp.JSONPathSubscript) ): - if escape: - path = self.escape_str(path) - + # Always escape path segments when wrapping them as SQL string literals, + # regardless of whether the key was quoted in the JSON path syntax. + # This ensures characters like single quotes are properly escaped for SQL. + path = self.escape_str(path) path = f"{self.dialect.QUOTE_START}{path}{self.dialect.QUOTE_END}" segments.append(path) diff --git a/sqlglot/generator.py b/sqlglot/generator.py index 8dde498c7a..e479fcdbbb 100644 --- a/sqlglot/generator.py +++ b/sqlglot/generator.py @@ -3840,7 +3840,7 @@ def json_path_part(self, expression: int | str | exp.JSONPathPart) -> str: if self._quote_json_path_key_using_brackets and self.JSON_PATH_SINGLE_QUOTE_ESCAPE: escaped = expression.replace("'", "\\'") - escaped = f"\\'{expression}\\'" + escaped = f"\\'{escaped}\\'" else: escaped = expression.replace('"', '\\"') escaped = f'"{escaped}"' @@ -5401,10 +5401,9 @@ def _jsonpathkey_sql(self, expression: exp.JSONPathKey) -> str: this = self.json_path_part(this) - if quoted and self.QUOTE_JSON_PATH: - # The whole path is rendered as a single quoted string literal, so the bracketed key - # (which may itself contain backslash-escaped quotes, e.g. ["x \"y\"z"]) must be - # escaped again for the outer string literal (-> ["x \\"y\\"z"]). + if self.QUOTE_JSON_PATH and not ( + self._quote_json_path_key_using_brackets and self.JSON_PATH_SINGLE_QUOTE_ESCAPE + ): this = self.escape_str(this) return ( diff --git a/sqlglot/generators/bigquery.py b/sqlglot/generators/bigquery.py index 2f6aa1e99b..8d8d32d948 100644 --- a/sqlglot/generators/bigquery.py +++ b/sqlglot/generators/bigquery.py @@ -34,6 +34,18 @@ JSON_EXTRACT_TYPE = t.Union[exp.JSONExtract, exp.JSONExtractScalar, exp.JSONExtractArray] DQUOTES_ESCAPING_JSON_FUNCTIONS = ("JSON_QUERY", "JSON_VALUE", "JSON_QUERY_ARRAY") +APOSTROPHE_UNSAFE_LEGACY_JSON_FUNCTIONS = { + exp.JSONExtract: "JSON_QUERY", + exp.JSONExtractArray: "JSON_QUERY_ARRAY", + exp.JSONExtractScalar: "JSON_VALUE", +} + + +def _has_apostrophe_json_path(expression: JSON_EXTRACT_TYPE) -> bool: + path = expression.expression + return isinstance(path, exp.JSONPath) and any( + isinstance(part, exp.JSONPathKey) and "'" in part.name for part in path.expressions + ) def _derived_table_values_to_unnest(self: BigQueryGenerator, expression: exp.Values) -> str: @@ -232,15 +244,19 @@ def _json_extract_sql(self: BigQueryGenerator, expression: JSON_EXTRACT_TYPE) -> name = expression.meta_get("name") or expression.sql_name() upper = name.upper() - dquote_escaping = upper in DQUOTES_ESCAPING_JSON_FUNCTIONS + if _has_apostrophe_json_path(expression): + upper = APOSTROPHE_UNSAFE_LEGACY_JSON_FUNCTIONS.get(type(expression), upper) - if dquote_escaping: - self._quote_json_path_key_using_brackets = False + dquote_escaping = upper in DQUOTES_ESCAPING_JSON_FUNCTIONS - sql = rename_func(upper)(self, expression) + try: + if dquote_escaping: + self._quote_json_path_key_using_brackets = False - if dquote_escaping: - self._quote_json_path_key_using_brackets = True + sql = rename_func(upper)(self, expression) + finally: + if dquote_escaping: + self._quote_json_path_key_using_brackets = True return sql diff --git a/sqlglot/generators/databricks.py b/sqlglot/generators/databricks.py index 98319538ce..0420286b20 100644 --- a/sqlglot/generators/databricks.py +++ b/sqlglot/generators/databricks.py @@ -101,6 +101,7 @@ def jsonpath_sql(self, expression: exp.JSONPath) -> str: path = super().jsonpath_sql(expression) if isinstance(expression.parent, exp.JSONExtractScalar): + path = self.escape_str(path) return f"{self.dialect.QUOTE_START}{path}{self.dialect.QUOTE_END}" return path diff --git a/tests/test_jsonpath.py b/tests/test_jsonpath.py index 7e846e886c..c2c0fecb98 100644 --- a/tests/test_jsonpath.py +++ b/tests/test_jsonpath.py @@ -47,6 +47,15 @@ def test_identity(self): with self.subTest(f"{selector} -> {expected}"): self.assertEqual(parse(selector).sql(), f"'{expected}'") + def test_apostrophe_escaping_is_scoped_to_keys(self): + for selector, expected in ( + ("$['it\\'s']", "$[\"it''s\"]"), + ("$[?@.a=='b']", "$[?@.a=='b']"), + ("$[?!(@.a=='b')]", "$[?!(@.a=='b')]"), + ): + with self.subTest(selector): + self.assertEqual(parse(selector).sql(), f"'{expected}'") + def test_union_preserves_falsey_members(self): for selector, expected in ( ("$[1,0]", exp.JSONPathUnion(expressions=[1, 0])), @@ -74,7 +83,7 @@ def test_cts_file(self): """$['a']""": """$.a""", """$['c']""": """$.c""", """$[' ']""": """$[" "]""", - """$['\\'']""": """$["\'"]""", + """$['\\'']""": """$["''"]""", """$['\\\\']""": """$["\\\\"]""", """$['\\/']""": """$["\\/"]""", """$['\\b']""": """$["\\b"]""", From b1da11af6b4ccb2cfc22375c10ba481b3a4dc125 Mon Sep 17 00:00:00 2001 From: fivetran-amrutabhimsenayachit Date: Fri, 4 Sep 2026 14:27:33 -0400 Subject: [PATCH 2/5] fix: address review comments --- sqlglot/generators/bigquery.py | 41 +++++++++++++------------------- sqlglot/generators/databricks.py | 1 - 2 files changed, 16 insertions(+), 26 deletions(-) diff --git a/sqlglot/generators/bigquery.py b/sqlglot/generators/bigquery.py index 8d8d32d948..09b7818323 100644 --- a/sqlglot/generators/bigquery.py +++ b/sqlglot/generators/bigquery.py @@ -34,18 +34,6 @@ JSON_EXTRACT_TYPE = t.Union[exp.JSONExtract, exp.JSONExtractScalar, exp.JSONExtractArray] DQUOTES_ESCAPING_JSON_FUNCTIONS = ("JSON_QUERY", "JSON_VALUE", "JSON_QUERY_ARRAY") -APOSTROPHE_UNSAFE_LEGACY_JSON_FUNCTIONS = { - exp.JSONExtract: "JSON_QUERY", - exp.JSONExtractArray: "JSON_QUERY_ARRAY", - exp.JSONExtractScalar: "JSON_VALUE", -} - - -def _has_apostrophe_json_path(expression: JSON_EXTRACT_TYPE) -> bool: - path = expression.expression - return isinstance(path, exp.JSONPath) and any( - isinstance(part, exp.JSONPathKey) and "'" in part.name for part in path.expressions - ) def _derived_table_values_to_unnest(self: BigQueryGenerator, expression: exp.Values) -> str: @@ -243,21 +231,13 @@ def _levenshtein_sql(self: BigQueryGenerator, expression: exp.Levenshtein) -> st def _json_extract_sql(self: BigQueryGenerator, expression: JSON_EXTRACT_TYPE) -> str: name = expression.meta_get("name") or expression.sql_name() upper = name.upper() - - if _has_apostrophe_json_path(expression): - upper = APOSTROPHE_UNSAFE_LEGACY_JSON_FUNCTIONS.get(type(expression), upper) - dquote_escaping = upper in DQUOTES_ESCAPING_JSON_FUNCTIONS - try: - if dquote_escaping: - self._quote_json_path_key_using_brackets = False - - sql = rename_func(upper)(self, expression) - finally: - if dquote_escaping: - self._quote_json_path_key_using_brackets = True - + if dquote_escaping: + self._quote_json_path_key_using_brackets = False + sql = rename_func(upper)(self, expression) + if dquote_escaping: + self._quote_json_path_key_using_brackets = True return sql @@ -315,6 +295,17 @@ class BigQueryGenerator(generator.Generator): exp.TsOrDsToDate, ) + def json_path_part(self, expression: int | str | exp.JSONPathPart) -> str: + if ( + isinstance(expression, str) + and self._quote_json_path_key_using_brackets + and self.JSON_PATH_SINGLE_QUOTE_ESCAPE + ): + escaped = expression.replace("'", "\\'") + return self.escape_str(f"'{escaped}'") + + return super().json_path_part(expression) + TRANSFORMS = { **generator.Generator.TRANSFORMS, exp.AIEmbed: rename_func("EMBED"), diff --git a/sqlglot/generators/databricks.py b/sqlglot/generators/databricks.py index 0420286b20..7611d29dd3 100644 --- a/sqlglot/generators/databricks.py +++ b/sqlglot/generators/databricks.py @@ -97,7 +97,6 @@ def timeserieskey_sql(self, expression: exp.TimeseriesKey) -> str: return f"{self.sql(expression, 'this')} TIMESERIES" def jsonpath_sql(self, expression: exp.JSONPath) -> str: - expression.set("escape", None) path = super().jsonpath_sql(expression) if isinstance(expression.parent, exp.JSONExtractScalar): From b4234b70b0dcc0411b8556bfed994907f5b8a0d3 Mon Sep 17 00:00:00 2001 From: geooo109 Date: Mon, 7 Sep 2026 14:32:39 +0300 Subject: [PATCH 3/5] escape JSON paths once in jsonpath_sql and drop the BigQuery override --- sqlglot-integration-tests | 2 +- sqlglot/dialects/dialect.py | 3 --- sqlglot/generator.py | 8 ++------ sqlglot/generators/bigquery.py | 11 ----------- tests/test_jsonpath.py | 14 +++----------- 5 files changed, 6 insertions(+), 32 deletions(-) diff --git a/sqlglot-integration-tests b/sqlglot-integration-tests index efddb0acdf..9273459d73 160000 --- a/sqlglot-integration-tests +++ b/sqlglot-integration-tests @@ -1 +1 @@ -Subproject commit efddb0acdfe0d367572bd8238bf1b3a1643400fd +Subproject commit 9273459d73bbaf4db8acb2d0c9b0a3edb4ee0fb2 diff --git a/sqlglot/dialects/dialect.py b/sqlglot/dialects/dialect.py index ce316d2331..a45ec3d952 100644 --- a/sqlglot/dialects/dialect.py +++ b/sqlglot/dialects/dialect.py @@ -2290,9 +2290,6 @@ def _json_extract_segments(self: Generator, expression: JSON_EXTRACT_TYPE) -> st if isinstance(segment, exp.JSONPathPart) and ( quoted_index or not isinstance(segment, exp.JSONPathSubscript) ): - # Always escape path segments when wrapping them as SQL string literals, - # regardless of whether the key was quoted in the JSON path syntax. - # This ensures characters like single quotes are properly escaped for SQL. path = self.escape_str(path) path = f"{self.dialect.QUOTE_START}{path}{self.dialect.QUOTE_END}" diff --git a/sqlglot/generator.py b/sqlglot/generator.py index e479fcdbbb..869a81e34b 100644 --- a/sqlglot/generator.py +++ b/sqlglot/generator.py @@ -3822,6 +3822,7 @@ def jsonpath_sql(self, expression: exp.JSONPath) -> str: path = self.expressions(expression, sep="", flat=True).lstrip(".") if self.QUOTE_JSON_PATH: + path = self.escape_str(path) path = f"{self.dialect.QUOTE_START}{path}{self.dialect.QUOTE_END}" return path @@ -3840,7 +3841,7 @@ def json_path_part(self, expression: int | str | exp.JSONPathPart) -> str: if self._quote_json_path_key_using_brackets and self.JSON_PATH_SINGLE_QUOTE_ESCAPE: escaped = expression.replace("'", "\\'") - escaped = f"\\'{escaped}\\'" + escaped = f"'{escaped}'" else: escaped = expression.replace('"', '\\"') escaped = f'"{escaped}"' @@ -5401,11 +5402,6 @@ def _jsonpathkey_sql(self, expression: exp.JSONPathKey) -> str: this = self.json_path_part(this) - if self.QUOTE_JSON_PATH and not ( - self._quote_json_path_key_using_brackets and self.JSON_PATH_SINGLE_QUOTE_ESCAPE - ): - this = self.escape_str(this) - return ( f"[{this}]" if self._quote_json_path_key_using_brackets and self.JSON_PATH_BRACKETED_KEY_SUPPORTED diff --git a/sqlglot/generators/bigquery.py b/sqlglot/generators/bigquery.py index 09b7818323..b54725e938 100644 --- a/sqlglot/generators/bigquery.py +++ b/sqlglot/generators/bigquery.py @@ -295,17 +295,6 @@ class BigQueryGenerator(generator.Generator): exp.TsOrDsToDate, ) - def json_path_part(self, expression: int | str | exp.JSONPathPart) -> str: - if ( - isinstance(expression, str) - and self._quote_json_path_key_using_brackets - and self.JSON_PATH_SINGLE_QUOTE_ESCAPE - ): - escaped = expression.replace("'", "\\'") - return self.escape_str(f"'{escaped}'") - - return super().json_path_part(expression) - TRANSFORMS = { **generator.Generator.TRANSFORMS, exp.AIEmbed: rename_func("EMBED"), diff --git a/tests/test_jsonpath.py b/tests/test_jsonpath.py index c2c0fecb98..d37f89d123 100644 --- a/tests/test_jsonpath.py +++ b/tests/test_jsonpath.py @@ -47,15 +47,6 @@ def test_identity(self): with self.subTest(f"{selector} -> {expected}"): self.assertEqual(parse(selector).sql(), f"'{expected}'") - def test_apostrophe_escaping_is_scoped_to_keys(self): - for selector, expected in ( - ("$['it\\'s']", "$[\"it''s\"]"), - ("$[?@.a=='b']", "$[?@.a=='b']"), - ("$[?!(@.a=='b')]", "$[?!(@.a=='b')]"), - ): - with self.subTest(selector): - self.assertEqual(parse(selector).sql(), f"'{expected}'") - def test_union_preserves_falsey_members(self): for selector, expected in ( ("$[1,0]", exp.JSONPathUnion(expressions=[1, 0])), @@ -83,7 +74,7 @@ def test_cts_file(self): """$['a']""": """$.a""", """$['c']""": """$.c""", """$[' ']""": """$[" "]""", - """$['\\'']""": """$["''"]""", + """$['\\'']""": """$["\'"]""", """$['\\\\']""": """$["\\\\"]""", """$['\\/']""": """$["\\/"]""", """$['\\b']""": """$["\\b"]""", @@ -165,4 +156,5 @@ def test_cts_file(self): pass else: path = parse(selector) - self.assertEqual(path.sql(), f"'{overrides.get(selector, selector)}'") + expected = overrides.get(selector, selector).replace("'", "''") + self.assertEqual(path.sql(), f"'{expected}'") From 0173f960ebd4a855f7ea98f48a1a3369a059ceda Mon Sep 17 00:00:00 2001 From: geooo109 Date: Mon, 7 Sep 2026 15:06:26 +0300 Subject: [PATCH 4/5] Restore original spacing in BigQuery JSON extract helper --- sqlglot/generators/bigquery.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/sqlglot/generators/bigquery.py b/sqlglot/generators/bigquery.py index b54725e938..2f6aa1e99b 100644 --- a/sqlglot/generators/bigquery.py +++ b/sqlglot/generators/bigquery.py @@ -231,13 +231,17 @@ def _levenshtein_sql(self: BigQueryGenerator, expression: exp.Levenshtein) -> st def _json_extract_sql(self: BigQueryGenerator, expression: JSON_EXTRACT_TYPE) -> str: name = expression.meta_get("name") or expression.sql_name() upper = name.upper() + dquote_escaping = upper in DQUOTES_ESCAPING_JSON_FUNCTIONS if dquote_escaping: self._quote_json_path_key_using_brackets = False + sql = rename_func(upper)(self, expression) + if dquote_escaping: self._quote_json_path_key_using_brackets = True + return sql From cd211c5477aff499e6c7fe73648c01264af3c92e Mon Sep 17 00:00:00 2001 From: geooo109 Date: Mon, 7 Sep 2026 15:37:14 +0300 Subject: [PATCH 5/5] sync tests --- sqlglot-integration-tests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqlglot-integration-tests b/sqlglot-integration-tests index 9273459d73..54d8263cc1 160000 --- a/sqlglot-integration-tests +++ b/sqlglot-integration-tests @@ -1 +1 @@ -Subproject commit 9273459d73bbaf4db8acb2d0c9b0a3edb4ee0fb2 +Subproject commit 54d8263cc1b318f2aad9a9c22b1ee6c7f247e1e6