Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 5 additions & 3 deletions sqlmesh/core/engine_adapter/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -1943,8 +1943,10 @@ def remove_managed_columns(
f"Column {updated_at_name} not found in {target_table}. Table must contain an `updated_at` timestamp for SCD Type 2"
)
time_data_type = target_columns_to_types[valid_from_name]
select_source_columns: t.List[t.Union[str, exp.Alias]] = [
col for col in unmanaged_columns_to_types if col != updated_at_name
select_source_columns: t.List[exp.Expr] = [
exp.column(col, quoted=True)
for col in unmanaged_columns_to_types
if col != updated_at_name
]
table_columns = [exp.column(c, quoted=True) for c in target_columns_to_types]
if updated_at_name:
Expand Down Expand Up @@ -2245,7 +2247,7 @@ def remove_managed_columns(
.with_(
"inserted_rows",
exp.select(
*unmanaged_columns_to_types,
*(exp.column(col, quoted=True) for col in unmanaged_columns_to_types),
insert_valid_from_start.as_(valid_from_col.this), # type: ignore
to_time_column(exp.null(), time_data_type, self.dialect, nullable=True).as_(
valid_to_col.this
Expand Down
1 change: 1 addition & 0 deletions sqlmesh/core/engine_adapter/base_postgres.py
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@ def columns(
)
if table.args.get("db"):
sql = sql.where(exp.column("nspname").eq(table.args["db"].name))
sql = sql.order_by("attnum")

self.execute(sql)
resp = self.cursor.fetchall()
Expand Down
14 changes: 14 additions & 0 deletions sqlmesh/core/engine_adapter/mixins.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
from sqlmesh.core.node import IntervalUnit
from sqlmesh.core.dialect import schema_
from sqlmesh.core.schema_diff import TableAlterOperation
from sqlmesh.utils import columns_to_types_all_known
from sqlmesh.utils.errors import SQLMeshError

if t.TYPE_CHECKING:
Expand Down Expand Up @@ -281,6 +282,19 @@ def _build_create_table_exp(
self.columns(temp_view_name)
)

if not (
columns_to_types_from_view
and columns_to_types_all_known(columns_to_types_from_view)
):
# View metadata is unavailable (e.g. a late-binding view the catalog doesn't expose).
# Never fall back to the LIMIT 0 CTAS since its column types (e.g. VARCHAR widths)
# are non-deterministic and the table could silently truncate data.
raise SQLMeshError(
f"Could not determine column types for table "
f"'{exp.to_table(table_name_or_schema).sql(dialect=self.dialect)}' from "
"its query. Specify the column types explicitly, e.g. by annotating the model."
)

schema = self._build_schema_exp(
exp.to_table(table_name_or_schema),
columns_to_types_from_view,
Expand Down
72 changes: 65 additions & 7 deletions sqlmesh/core/engine_adapter/redshift.py
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,59 @@ def columns(
sql = sql.where(exp.column("table_schema").eq(table.args["db"].name))

columns_raw = self.fetchall(sql, quote_identifiers=True)
if not columns_raw:
# svv_columns can return 0 rows for late-binding views referencing cross-database /
# datashare objects or schemas without USAGE metadata visibility. Fall back to
# pg_get_late_binding_view_cols() and then pg_catalog.pg_attribute.
late_binding_sql = (
exp.select("col_name", "col_type")
.from_(
exp.Table(
this=exp.func("pg_get_late_binding_view_cols"),
alias=exp.TableAlias(
this=exp.to_identifier("cols"),
columns=[
exp.ColumnDef(
this=exp.to_identifier("view_schema"),
kind=exp.DataType.build("NAME", dialect=self.dialect),
),
exp.ColumnDef(
this=exp.to_identifier("view_name"),
kind=exp.DataType.build("NAME", dialect=self.dialect),
),
exp.ColumnDef(
this=exp.to_identifier("col_name"),
kind=exp.DataType.build("NAME", dialect=self.dialect),
),
exp.ColumnDef(
this=exp.to_identifier("col_type"),
kind=exp.DataType.build("VARCHAR", dialect=self.dialect),
),
exp.ColumnDef(
this=exp.to_identifier("col_num"),
kind=exp.DataType.build("INT", dialect=self.dialect),
),
],
),
)
)
.where(exp.column("view_name").eq(table.alias_or_name))
.order_by("col_num", dialect=self.dialect)
)
if table.args.get("db"):
late_binding_sql = late_binding_sql.where(
exp.column("view_schema").eq(table.args["db"].name)
)
late_binding_cols = self.fetchall(late_binding_sql, quote_identifiers=True)
if late_binding_cols:
return {
col_name: exp.DataType.build(col_type, dialect=self.dialect, udt=True)
for col_name, col_type in late_binding_cols
}
try:
return super().columns(table_name, include_pseudo_columns=include_pseudo_columns)
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
except SQLMeshError:
return {}

def build_var_length_col(
column_name: str,
Expand Down Expand Up @@ -200,6 +253,9 @@ def _create_table_from_source_queries(
)
if self.table_exists(table_name):
return
# target_columns_to_types is deliberately not forwarded: inferred types can lack VARCHAR widths, and
# casting to them would truncate data. Keeping the LIMIT 0 CTAS lets VarcharSizeWorkaroundMixin derive
# the exact widths from a temp view instead.
super()._create_table_from_source_queries(
table_name,
source_queries,
Expand Down Expand Up @@ -228,13 +284,15 @@ def create_view(
underlying table without dropping the view first. This is a problem for us since we want to be able to
swap tables out from under views. Therefore, we create the view as non-binding.
"""
no_schema_binding = True
if isinstance(query_or_df, exp.Expr):
# We can't include NO SCHEMA BINDING if the query has a recursive CTE
has_recursive_cte = any(
w.args.get("recursive", False) for w in query_or_df.find_all(exp.With)
)
no_schema_binding = not has_recursive_cte
no_schema_binding: t.Optional[bool] = create_kwargs.pop("no_schema_binding", None)
if no_schema_binding is None:
no_schema_binding = True
if isinstance(query_or_df, exp.Expr):
# We can't include NO SCHEMA BINDING if the query has a recursive CTE
has_recursive_cte = any(
w.args.get("recursive", False) for w in query_or_df.find_all(exp.With)
)
no_schema_binding = not has_recursive_cte

return super().create_view(
view_name,
Expand Down
6 changes: 4 additions & 2 deletions sqlmesh/utils/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -342,8 +342,10 @@ def type_is_known(d_type: t.Union[exp.DataType, exp.ColumnDef]) -> bool:


def columns_to_types_all_known(columns_to_types: t.Dict[str, exp.DataType]) -> bool:
"""Checks that all column types are known and not NULL."""
return all(type_is_known(expression) for expression in columns_to_types.values())
"""Checks that all column types are non-empty, known, and not NULL."""
return bool(columns_to_types) and all(
type_is_known(expression) for expression in columns_to_types.values()
)


class Verbosity(IntEnum):
Expand Down
1 change: 1 addition & 0 deletions tests/core/engine_adapter/test_base_postgres.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ def test_columns(make_mocked_engine_adapter: t.Callable):
'JOIN "pg_catalog"."pg_class" ON "pg_class"."oid" = "attrelid" '
'JOIN "pg_catalog"."pg_namespace" ON "pg_namespace"."oid" = "relnamespace" '
"""WHERE ("attnum" > 0 AND NOT "attisdropped" AND "relname" = 'table') AND "nspname" = 'db'"""
' ORDER BY "attnum"'
)
assert resp == {"col": exp.DataType.build("INT")}

Expand Down
24 changes: 24 additions & 0 deletions tests/core/engine_adapter/test_mssql.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
from sqlmesh.core import dialect as d
from sqlmesh.core.engine_adapter.shared import DataObject, DataObjectType, SourceQuery
from sqlmesh.utils.date import to_ds
from sqlmesh.utils.errors import SQLMeshError
from tests.core.engine_adapter import to_sql_calls

pytestmark = [pytest.mark.engine, pytest.mark.mssql]
Expand Down Expand Up @@ -1055,3 +1056,26 @@ def test_comments(make_mocked_engine_adapter: t.Callable, mocker: MockerFixture)
adapter._build_create_comment_table_exp(table, comment),
adapter._build_create_comment_column_exp(table, "a", comment),
]


def test_ctas_varchar_workaround_empty_columns_fallback(
make_mocked_engine_adapter: t.Callable, mocker: MockerFixture
):
adapter = make_mocked_engine_adapter(MSSQLEngineAdapter)
mocker.patch("sqlmesh.core.engine_adapter.base.random_id", return_value="test_random_id")
mocker.patch(
"sqlmesh.core.engine_adapter.mssql.MSSQLEngineAdapter.table_exists", return_value=False
)
mocker.patch("sqlmesh.core.engine_adapter.mssql.MSSQLEngineAdapter.columns", return_value={})

with pytest.raises(SQLMeshError, match="Could not determine column types"):
adapter.ctas(
table_name="test_schema.test_table",
query_or_df=parse_one("SELECT a, b FROM tbl WHERE FALSE LIMIT 0"),
exists=False,
)

assert to_sql_calls(adapter) == [
"CREATE VIEW [__temp_ctas_test_random_id] AS SELECT [a], [b] FROM [tbl];",
"DROP VIEW IF EXISTS [__temp_ctas_test_random_id];",
]
Loading