chore: remove sqlparse (#33564)

Author: betodealmeida

pyproject.toml

@@ -94,7 +94,6 @@ dependencies = [
     "sqlalchemy>=1.4, <2",
     "sqlalchemy-utils>=0.38.3, <0.39",
     "sqlglot>=26.1.3, <27",
-    "sqlparse>=0.5.0",
     # newer pandas needs 0.9+
     "tabulate>=0.9.0, <1.0",
     "typing-extensions>=4, <5",
@@ -219,7 +218,7 @@ combine_as_imports = true
 include_trailing_comma = true
 line_length = 88
 known_first_party = "superset"
-known_third_party = "alembic, apispec, backoff, celery, click, colorama, cron_descriptor, croniter, cryptography, dateutil, deprecation, flask, flask_appbuilder, flask_babel, flask_caching, flask_compress, flask_jwt_extended, flask_login, flask_migrate, flask_sqlalchemy, flask_talisman, flask_testing, flask_wtf, freezegun, geohash, geopy, holidays, humanize, isodate, jinja2, jwt, markdown, markupsafe, marshmallow, msgpack, nh3, numpy, pandas, parameterized, parsedatetime, pgsanity, polyline, prison, progress, pyarrow, sqlalchemy_bigquery, pyhive, pyparsing, pytest, pytest_mock, pytz, redis, requests, selenium, setuptools, shillelagh, simplejson, slack, sqlalchemy, sqlalchemy_utils, sqlparse, typing_extensions, urllib3, werkzeug, wtforms, wtforms_json, yaml"
+known_third_party = "alembic, apispec, backoff, celery, click, colorama, cron_descriptor, croniter, cryptography, dateutil, deprecation, flask, flask_appbuilder, flask_babel, flask_caching, flask_compress, flask_jwt_extended, flask_login, flask_migrate, flask_sqlalchemy, flask_talisman, flask_testing, flask_wtf, freezegun, geohash, geopy, holidays, humanize, isodate, jinja2, jwt, markdown, markupsafe, marshmallow, msgpack, nh3, numpy, pandas, parameterized, parsedatetime, pgsanity, polyline, prison, progress, pyarrow, sqlalchemy_bigquery, pyhive, pyparsing, pytest, pytest_mock, pytz, redis, requests, selenium, setuptools, shillelagh, simplejson, slack, sqlalchemy, sqlalchemy_utils, typing_extensions, urllib3, werkzeug, wtforms, wtforms_json, yaml"
 multi_line_output = 3
 order_by_type = false
 

requirements/base.txt

@@ -379,8 +379,6 @@ sqlalchemy-utils==0.38.3
     #   flask-appbuilder
 sqlglot==26.17.1
     # via apache-superset (pyproject.toml)
-sqlparse==0.5.3
-    # via apache-superset (pyproject.toml)
 sshtunnel==0.4.0
     # via apache-superset (pyproject.toml)
 tabulate==0.9.0

requirements/development.txt

@@ -798,10 +798,6 @@ sqlglot==26.17.1
     #   apache-superset
 sqloxide==0.1.51
     # via apache-superset
-sqlparse==0.5.3
-    # via
-    #   -c requirements/base.txt
-    #   apache-superset
 sshtunnel==0.4.0
     # via
     #   -c requirements/base.txt

superset/commands/database/uploaders/base.py

@@ -35,7 +35,7 @@
 from superset.connectors.sqla.models import SqlaTable
 from superset.daos.database import DatabaseDAO
 from superset.models.core import Database
-from superset.sql_parse import Table
+from superset.sql.parse import Table
 from superset.utils.backports import StrEnum
 from superset.utils.core import get_user
 from superset.utils.decorators import on_error, transaction

superset/commands/dataset/create.py

@@ -33,7 +33,7 @@
 from superset.daos.dataset import DatasetDAO
 from superset.exceptions import SupersetSecurityException
 from superset.extensions import security_manager
-from superset.sql_parse import Table
+from superset.sql.parse import Table
 from superset.utils.decorators import on_error, transaction
 
 logger = logging.getLogger(__name__)

superset/commands/dataset/duplicate.py

@@ -36,7 +36,7 @@
 from superset.exceptions import SupersetErrorException
 from superset.extensions import db
 from superset.models.core import Database
-from superset.sql_parse import Table
+from superset.sql.parse import Table
 from superset.utils.decorators import on_error, transaction
 
 logger = logging.getLogger(__name__)

superset/commands/dataset/exceptions.py

@@ -26,7 +26,7 @@
     ImportFailedError,
     UpdateFailedError,
 )
-from superset.sql_parse import Table
+from superset.sql.parse import Table
 
 
 def get_dataset_exist_error_msg(table: Table) -> str:

superset/commands/dataset/importers/v1/utils.py

@@ -31,7 +31,7 @@
 from superset.commands.exceptions import ImportFailedError
 from superset.connectors.sqla.models import SqlaTable
 from superset.models.core import Database
-from superset.sql_parse import Table
+from superset.sql.parse import Table
 from superset.utils import json
 from superset.utils.core import get_user
 

superset/commands/dataset/update.py

@@ -47,7 +47,7 @@
 from superset.datasets.schemas import FolderSchema
 from superset.exceptions import SupersetSecurityException
 from superset.models.core import Database
-from superset.sql_parse import Table
+from superset.sql.parse import Table
 from superset.utils.decorators import on_error, transaction
 
 logger = logging.getLogger(__name__)

superset/common/query_object.py

@@ -24,6 +24,7 @@
 
 from flask import g
 from flask_babel import gettext as _
+from jinja2.exceptions import TemplateError
 from pandas import DataFrame
 
 from superset import feature_flag_manager
@@ -34,7 +35,7 @@
     QueryObjectValidationError,
 )
 from superset.extensions import event_logger
-from superset.sql_parse import sanitize_clause
+from superset.sql.parse import sanitize_clause
 from superset.superset_typing import Column, Metric, OrderBy
 from superset.utils import json, pandas_postprocessing
 from superset.utils.core import (
@@ -260,9 +261,11 @@ def metric_names(self) -> list[str]:
         """Return metrics names (labels), coerce adhoc metrics to strings."""
         return get_metric_names(
             self.metrics or [],
-            self.datasource.verbose_map
-            if self.datasource and hasattr(self.datasource, "verbose_map")
-            else None,
+            (
+                self.datasource.verbose_map
+                if self.datasource and hasattr(self.datasource, "verbose_map")
+                else None
+            ),
         )
 
     @property
@@ -298,11 +301,25 @@ def _validate_no_have_duplicate_labels(self) -> None:
             )
 
     def _sanitize_filters(self) -> None:
+        from superset.jinja_context import get_template_processor
+
         for param in ("where", "having"):
             clause = self.extras.get(param)
-            if clause:
+            if clause and self.datasource:
                 try:
-                    sanitized_clause = sanitize_clause(clause)
+                    database = self.datasource.database
+                    processor = get_template_processor(database=database)
+                    try:
+                        clause = processor.process_template(clause, force=True)
+                    except TemplateError as ex:
+                        raise QueryObjectValidationError(
+                            _(
+                                "Error in jinja expression in WHERE clause: %(msg)s",
+                                msg=ex.message,
+                            )
+                        ) from ex
+                    engine = database.db_engine_spec.engine
+                    sanitized_clause = sanitize_clause(clause, engine)
                     if sanitized_clause != clause:
                         self.extras[param] = sanitized_clause
                 except QueryClauseValidationException as ex: