diff --git a/sqlmesh/core/context.py b/sqlmesh/core/context.py index fd7f8ad071..332fca4c48 100644 --- a/sqlmesh/core/context.py +++ b/sqlmesh/core/context.py @@ -1888,6 +1888,29 @@ def plan_builder( and self.config.virtual_environment_mode.is_full ) + schema_differ_overrides: t.Dict[t.Optional[str], t.Dict[str, t.Any]] = {} + has_schema_differ_overrides = False + try: + default_overrides = self.connection_config.schema_differ_overrides or {} + schema_differ_overrides[None] = default_overrides + schema_differ_overrides[self.selected_gateway] = default_overrides + if default_overrides: + has_schema_differ_overrides = True + except ConfigError: + pass + for cfg in self.configs.values(): + if isinstance(cfg.gateways, dict): + for gw_name in cfg.gateways: + if gw_name not in schema_differ_overrides: + try: + conn = cfg.get_connection(gw_name) + except ConfigError: + continue + gw_overrides = conn.schema_differ_overrides or {} + schema_differ_overrides[gw_name] = gw_overrides + if gw_overrides: + has_schema_differ_overrides = True + return self.PLAN_BUILDER_TYPE( context_diff=context_diff, start=start, @@ -1931,6 +1954,9 @@ def plan_builder( }, explain=explain or False, ignore_cron=ignore_cron or False, + schema_differ_overrides=( + schema_differ_overrides if has_schema_differ_overrides else None + ), ) def apply( diff --git a/sqlmesh/core/plan/builder.py b/sqlmesh/core/plan/builder.py index a6307a9ffd..8396eb42d2 100644 --- a/sqlmesh/core/plan/builder.py +++ b/sqlmesh/core/plan/builder.py @@ -7,6 +7,8 @@ from functools import cached_property from datetime import datetime +from sqlglot import exp + from sqlmesh.core.console import PlanBuilderConsole, get_console from sqlmesh.core.config import ( @@ -16,6 +18,7 @@ ) from sqlmesh.core.context_diff import ContextDiff from sqlmesh.core.environment import EnvironmentNamingInfo +from sqlmesh.core.model import Model, SCDType2ByTimeKind from sqlmesh.core.plan.common import should_force_rebuild, is_breaking_kind_change from sqlmesh.core.plan.definition import ( Plan, @@ -27,6 +30,8 @@ get_schema_differ, has_drop_alteration, has_additive_alteration, + prepare_columns_for_static_diff, + TableAlterDropColumnOperation, TableAlterOperation, ) from sqlmesh.core.snapshot import ( @@ -36,7 +41,7 @@ ) from sqlmesh.core.snapshot.categorizer import categorize_change from sqlmesh.core.snapshot.definition import Interval, SnapshotId -from sqlmesh.utils import columns_to_types_all_known, random_id +from sqlmesh.utils import random_id from sqlmesh.utils.dag import DAG from sqlmesh.utils.date import ( TimeLike, @@ -52,6 +57,25 @@ logger = logging.getLogger(__name__) +def _physical_columns_to_types(model: Model) -> t.Optional[t.Dict[str, exp.DataType]]: + """Returns the model's columns as they are materialized in the physical table. + + SCD Type 2 by time models always create the `updated_at` column with `time_data_type`, + regardless of the type that the model's query projects. + + Table creation only forces this for annotated models. For unannotated ones the comparison is + still consistent: both sides are overridden the same way, and `valid_from` / `valid_to` already + use `time_data_type`. + """ + columns_to_types = model.columns_to_types + if columns_to_types is not None and isinstance(model.kind, SCDType2ByTimeKind): + columns_to_types = dict(columns_to_types) + updated_at = model.kind.updated_at_name.name + if updated_at in columns_to_types: + columns_to_types[updated_at] = model.kind.time_data_type + return columns_to_types + + class PlanBuilder: """Plan Builder constructs a Plan based on user choices for how they want to backfill, preview, etc. their changes. @@ -98,6 +122,8 @@ class PlanBuilder: end_override_per_model: A mapping of model FQNs to target end dates. ignore_cron: Whether to ignore the node's cron schedule when computing missing intervals. explain: Whether to explain the plan instead of applying it. + schema_differ_overrides: Optional mapping from gateway name (or None for the default + gateway) to SchemaDiffer keyword argument overrides for static schema diffing. """ def __init__( @@ -138,6 +164,7 @@ def __init__( console: t.Optional[PlanBuilderConsole] = None, user_provided_flags: t.Optional[t.Dict[str, UserProvidedFlags]] = None, selected_models: t.Optional[t.Set[str]] = None, + schema_differ_overrides: t.Optional[t.Dict[t.Optional[str], t.Dict[str, t.Any]]] = None, ): self._context_diff = context_diff self._no_gaps = no_gaps @@ -212,6 +239,10 @@ def __init__( gateway_managed=self._context_diff.gateway_managed_virtual_layer, ) + self._schema_differ_overrides: t.Dict[t.Optional[str], t.Dict[str, t.Any]] = ( + schema_differ_overrides or {} + ) + self._logged_schema_warnings: t.Set[t.Tuple[SnapshotId, bool]] = set() self._latest_plan: t.Optional[Plan] = None @property @@ -310,8 +341,8 @@ def build(self) -> Plan: dag = self._build_dag() directly_modified, indirectly_modified = self._build_directly_and_indirectly_modified(dag) - self._check_destructive_additive_changes(directly_modified) self._categorize_snapshots(dag, indirectly_modified) + self._check_destructive_additive_changes(dag) self._adjust_snapshot_intervals() deployability_index = ( @@ -584,68 +615,154 @@ def _adjust_snapshot_intervals(self) -> None: if new.is_forward_only: new.dev_intervals = new.intervals.copy() - def _check_destructive_additive_changes(self, directly_modified: t.Set[SnapshotId]) -> None: - for s_id in sorted(directly_modified): + def _is_breaking_kind_or_seed_modification(self, s_id: SnapshotId) -> bool: + if s_id.name not in self._context_diff.modified_snapshots: + return False + snapshot = self._context_diff.snapshots[s_id] + if snapshot.is_seed: + return True + new, old = self._context_diff.modified_snapshots[s_id.name] + return is_breaking_kind_change(old, new) + + def _compute_snapshot_forward_only(self, s_id: SnapshotId) -> bool: + if not (self._forward_only or self._is_forward_only_change(s_id)): + return False + # Breaking kind changes and seed changes can't be forward-only. + return not self._is_breaking_kind_or_seed_modification(s_id) + + def _check_destructive_additive_changes(self, dag: DAG[SnapshotId]) -> None: + effective_forward_only: t.Dict[SnapshotId, bool] = {} + + for s_id in dag: + snapshot = self._context_diff.snapshots.get(s_id) + if not snapshot: + continue + + if snapshot.is_seed or self._is_breaking_kind_or_seed_modification(s_id): + is_fo = False + elif ( + snapshot.is_forward_only or self._forward_only or self._is_forward_only_change(s_id) + ): + is_fo = True + elif self._context_diff.indirectly_modified(s_id.name): + upstream_new_fo = { + effective_forward_only[p_id] + for p_id in dag.upstream(s_id) + if p_id in effective_forward_only + and self._is_new_snapshot(self._context_diff.snapshots[p_id]) + } + is_fo = upstream_new_fo == {True} + else: + is_fo = False + effective_forward_only[s_id] = is_fo + if s_id.name not in self._context_diff.modified_snapshots: continue + if not ( + self._context_diff.directly_modified(s_id.name) + or self._context_diff.indirectly_modified(s_id.name) + ): + continue + if not snapshot.is_model or not snapshot.is_materialized or snapshot.is_seed: + continue + + is_final_indirect_non_breaking = ( + snapshot.categorized + and snapshot.is_indirect_non_breaking + and all( + self._context_diff.snapshots[p_id].categorized + for p_id in dag.upstream(s_id) + if p_id in self._context_diff.snapshots + ) + ) + if not (is_fo or is_final_indirect_non_breaking): + continue - snapshot = self._context_diff.snapshots[s_id] needs_destructive_check = snapshot.needs_destructive_check( self._allow_destructive_models ) needs_additive_check = snapshot.needs_additive_check(self._allow_additive_models) - # should we raise/warn if this snapshot has/inherits a destructive change? - should_raise_or_warn = (self._is_forward_only_change(s_id) or self._forward_only) and ( - needs_destructive_check or needs_additive_check - ) - - if not should_raise_or_warn or not snapshot.is_model: - continue - new, old = self._context_diff.modified_snapshots[snapshot.name] - # we must know all columns_to_types to determine whether a change is destructive - old_columns_to_types = old.model.columns_to_types or {} - new_columns_to_types = new.model.columns_to_types or {} + if not (needs_destructive_check or needs_additive_check): + continue + comparable = prepare_columns_for_static_diff( + _physical_columns_to_types(old.model), _physical_columns_to_types(new.model) + ) + if comparable is None: + continue - if columns_to_types_all_known(old_columns_to_types) and columns_to_types_all_known( - new_columns_to_types - ): - alter_operations = t.cast( - t.List[TableAlterOperation], - get_schema_differ(snapshot.model.dialect).compare_columns( - new.name, - old_columns_to_types, - new_columns_to_types, - ignore_destructive=new.model.on_destructive_change.is_ignore, - ignore_additive=new.model.on_additive_change.is_ignore, - ), - ) + old_columns_to_types, new_columns_to_types = comparable + if snapshot.model_gateway in self._schema_differ_overrides: + differ_overrides = self._schema_differ_overrides[snapshot.model_gateway] or None + elif snapshot.model_gateway is None: + differ_overrides = self._schema_differ_overrides.get(None) or None + else: + differ_overrides = None + + alter_operations = t.cast( + t.List[TableAlterOperation], + get_schema_differ( + snapshot.model.dialect, overrides=differ_overrides + ).compare_columns( + new.name, + old_columns_to_types, + new_columns_to_types, + ignore_destructive=new.model.on_destructive_change.is_ignore, + ignore_additive=new.model.on_additive_change.is_ignore, + ), + ) - snapshot_name = snapshot.name - model_dialect = snapshot.model.dialect + if old.model.on_additive_change.is_ignore: + # Columns that the previous definition added under an ignored additive policy may + # never have reached its physical table, so the saved definitions can't establish + # that dropping them is destructive. That is left to the physical validation. + alter_operations = [ + op + for op in alter_operations + if not isinstance(op, TableAlterDropColumnOperation) + ] - if needs_destructive_check and has_drop_alteration(alter_operations): + snapshot_name = snapshot.name + model_dialect = snapshot.model.dialect + + if needs_destructive_check and has_drop_alteration(alter_operations): + if snapshot.model.on_destructive_change.is_warn: + if (s_id, True) not in self._logged_schema_warnings: + self._console.log_destructive_change( + snapshot_name, + alter_operations, + model_dialect, + error=False, + ) + self._logged_schema_warnings.add((s_id, True)) + elif snapshot.model.on_destructive_change.is_error: self._console.log_destructive_change( snapshot_name, alter_operations, model_dialect, - error=not snapshot.model.on_destructive_change.is_warn, + error=True, ) - if snapshot.model.on_destructive_change.is_error: - raise PlanError( - "Plan requires a destructive change to a forward-only model." + raise PlanError("Plan requires a destructive change to a forward-only model.") + + if needs_additive_check and has_additive_alteration(alter_operations): + if snapshot.model.on_additive_change.is_warn: + if (s_id, False) not in self._logged_schema_warnings: + self._console.log_additive_change( + snapshot_name, + alter_operations, + model_dialect, + error=False, ) - - if needs_additive_check and has_additive_alteration(alter_operations): + self._logged_schema_warnings.add((s_id, False)) + elif snapshot.model.on_additive_change.is_error: self._console.log_additive_change( snapshot_name, alter_operations, model_dialect, - error=not snapshot.model.on_additive_change.is_warn, + error=True, ) - if snapshot.model.on_additive_change.is_error: - raise PlanError("Plan requires an additive change to a forward-only model.") + raise PlanError("Plan requires an additive change to a forward-only model.") def _categorize_snapshots( self, dag: DAG[SnapshotId], indirectly_modified: SnapshotMapping @@ -663,12 +780,7 @@ def _categorize_snapshots( if not snapshot or not self._is_new_snapshot(snapshot): continue - forward_only = self._forward_only or self._is_forward_only_change(s_id) - if forward_only and s_id.name in self._context_diff.modified_snapshots: - new, old = self._context_diff.modified_snapshots[s_id.name] - if is_breaking_kind_change(old, new) or snapshot.is_seed: - # Breaking kind changes and seed changes can't be forward-only. - forward_only = False + forward_only = self._compute_snapshot_forward_only(s_id) if s_id in self._choices: snapshot.categorize_as(self._choices[s_id], forward_only) diff --git a/sqlmesh/core/schema_diff.py b/sqlmesh/core/schema_diff.py index ecf38b18a8..a294962854 100644 --- a/sqlmesh/core/schema_diff.py +++ b/sqlmesh/core/schema_diff.py @@ -11,7 +11,7 @@ from sqlglot import exp from sqlglot.helper import ensure_list, seq_get -from sqlmesh.utils import columns_to_types_to_struct +from sqlmesh.utils import columns_to_types_to_struct, type_is_known from sqlmesh.utils.pydantic import PydanticModel from sqlmesh.utils.errors import SQLMeshError @@ -648,7 +648,7 @@ def _alter_operation( if self._is_coerceable_type(current_type, new_type): return [] if self._is_compatible_type(current_type, new_type): - if ignore_additive: + if ignore_destructive if self.treat_alter_data_type_as_destructive else ignore_additive: return [] struct.expressions.pop(pos) struct.expressions.insert(pos, new_kwarg) @@ -667,7 +667,7 @@ def _alter_operation( return [] return self._drop_operation( columns, - root_struct, + struct, pos, root_struct, table_name, @@ -843,6 +843,186 @@ def get_additive_column_names(alter_expressions: t.List[TableAlterOperation]) -> ] +class _UncomparableTypesError(Exception): + """Raised when a type pair can't be safely normalized for a name-based static comparison.""" + + +def _is_unknown_leaf(dtype: exp.DataType) -> bool: + return dtype.is_type(exp.DataType.Type.UNKNOWN, exp.DataType.Type.NULL) + + +def _sanitize_unknown_types(dtype: exp.DataType) -> exp.DataType: + if type_is_known(dtype): + return dtype + + placeholder = exp.DataType.build("INT") + if _is_unknown_leaf(dtype): + return placeholder + + sanitized = dtype.copy() + new_expressions: t.List[exp.Expr] = [] + for expr in dtype.expressions: + if isinstance(expr, exp.ColumnDef): + col_copy = expr.copy() + col_copy.set( + "kind", + _sanitize_unknown_types(expr.kind) + if isinstance(expr.kind, exp.DataType) + else placeholder, + ) + new_expressions.append(col_copy) + elif isinstance(expr, exp.DataType): + new_expressions.append(_sanitize_unknown_types(expr)) + else: + new_expressions.append(expr.copy()) + sanitized.set("expressions", new_expressions) + return sanitized + + +def _prepare_type_pair_for_static_diff( + old_type: exp.DataType, new_type: exp.DataType +) -> t.Tuple[exp.DataType, exp.DataType]: + if type_is_known(old_type) and type_is_known(new_type): + return old_type, new_type + + placeholder = exp.DataType.build("INT") + if _is_unknown_leaf(old_type) or _is_unknown_leaf(new_type): + return placeholder, placeholder + + if old_type.is_type(exp.DataType.Type.STRUCT) and new_type.is_type(exp.DataType.Type.STRUCT): + if not old_type.expressions or not new_type.expressions: + return placeholder, placeholder + if all( + isinstance(e, exp.ColumnDef) and e.this is not None for e in old_type.expressions + ) and all( + isinstance(e, exp.ColumnDef) and e.this is not None for e in new_type.expressions + ): + old_col_defs = t.cast(t.List[exp.ColumnDef], old_type.expressions) + new_col_defs = t.cast(t.List[exp.ColumnDef], new_type.expressions) + new_by_name = {f.name: f for f in new_col_defs} + + shared_fields: t.Dict[str, t.Tuple[exp.DataType, exp.DataType]] = {} + norm_old_exprs: t.List[exp.ColumnDef] = [] + for f_old in old_col_defs: + old_kind = f_old.kind or placeholder + f_old_copy = f_old.copy() + if f_old.name in new_by_name: + new_kind = new_by_name[f_old.name].kind or placeholder + norm_old_k, norm_new_k = _prepare_type_pair_for_static_diff(old_kind, new_kind) + shared_fields[f_old.name] = (norm_old_k, norm_new_k) + f_old_copy.set("kind", norm_old_k) + else: + f_old_copy.set("kind", _sanitize_unknown_types(old_kind)) + norm_old_exprs.append(f_old_copy) + + norm_new_exprs: t.List[exp.ColumnDef] = [] + for f_new in new_col_defs: + f_new_copy = f_new.copy() + if f_new.name in shared_fields: + f_new_copy.set("kind", shared_fields[f_new.name][1]) + else: + f_new_copy.set("kind", _sanitize_unknown_types(f_new.kind or placeholder)) + norm_new_exprs.append(f_new_copy) + + norm_old_struct = old_type.copy() + norm_old_struct.set("expressions", norm_old_exprs) + norm_new_struct = new_type.copy() + norm_new_struct.set("expressions", norm_new_exprs) + return norm_old_struct, norm_new_struct + # Positional (unnamed) fields can't go through the name-based comparison. + raise _UncomparableTypesError() + + if old_type.is_type(exp.DataType.Type.ARRAY) and new_type.is_type(exp.DataType.Type.ARRAY): + if ( + not old_type.expressions + or not new_type.expressions + or not isinstance(old_type.expressions[0], exp.DataType) + or not isinstance(new_type.expressions[0], exp.DataType) + ): + arr_placeholder = exp.DataType.build("ARRAY") + return arr_placeholder, arr_placeholder.copy() + norm_old_elem, norm_new_elem = _prepare_type_pair_for_static_diff( + old_type.expressions[0], new_type.expressions[0] + ) + norm_old_arr = old_type.copy() + norm_old_arr.set("expressions", [norm_old_elem]) + norm_new_arr = new_type.copy() + norm_new_arr.set("expressions", [norm_new_elem]) + return norm_old_arr, norm_new_arr + + if ( + old_type.is_type(exp.DataType.Type.MAP) + and new_type.is_type(exp.DataType.Type.MAP) + and len(old_type.expressions) == 2 + and len(new_type.expressions) == 2 + and isinstance(old_type.expressions[0], exp.DataType) + and isinstance(old_type.expressions[1], exp.DataType) + and isinstance(new_type.expressions[0], exp.DataType) + and isinstance(new_type.expressions[1], exp.DataType) + ): + norm_old_k, norm_new_k = _prepare_type_pair_for_static_diff( + old_type.expressions[0], new_type.expressions[0] + ) + norm_old_v, norm_new_v = _prepare_type_pair_for_static_diff( + old_type.expressions[1], new_type.expressions[1] + ) + norm_old_map = old_type.copy() + norm_old_map.set("expressions", [norm_old_k, norm_old_v]) + norm_new_map = new_type.copy() + norm_new_map.set("expressions", [norm_new_k, norm_new_v]) + return norm_old_map, norm_new_map + + return _sanitize_unknown_types(old_type), _sanitize_unknown_types(new_type) + + +def prepare_columns_for_static_diff( + old_columns_to_types: t.Optional[t.Dict[str, exp.DataType]], + new_columns_to_types: t.Optional[t.Dict[str, exp.DataType]], +) -> t.Optional[t.Tuple[t.Dict[str, exp.DataType], t.Dict[str, exp.DataType]]]: + """Prepares old and new column mappings for static schema diffing at plan time. + + Returns None if either mapping is missing or empty (e.g., unexpanded SELECT *), or if a shared + column has a partially-unknown STRUCT with positional fields that can't be compared by name. + For columns (and nested STRUCT/ARRAY/MAP fields) present in both old and new where either side + has an unknown or NULL type, normalizes both sides to the same placeholder type so shared + unknown-type columns/fields do not trigger false-positive destructive type changes while still + detecting dropped/added columns/fields and type changes among known-type columns/fields, + preserving original projection column order. + """ + if not old_columns_to_types or not new_columns_to_types: + return None + + # The differ mutates type trees in place, so never hand it the model's own (cached) nodes. + old_columns_to_types = {c: d.copy() for c, d in old_columns_to_types.items()} + new_columns_to_types = {c: d.copy() for c, d in new_columns_to_types.items()} + + comparable_old: t.Dict[str, exp.DataType] = {} + shared_new: t.Dict[str, exp.DataType] = {} + + for col, old_type in old_columns_to_types.items(): + if col in new_columns_to_types: + try: + norm_old, norm_new = _prepare_type_pair_for_static_diff( + old_type, new_columns_to_types[col] + ) + except _UncomparableTypesError: + # Defer to the physical validation performed at evaluation time. + return None + comparable_old[col] = norm_old + shared_new[col] = norm_new + else: + comparable_old[col] = _sanitize_unknown_types(old_type) + + comparable_new: t.Dict[str, exp.DataType] = {} + for col, new_type in new_columns_to_types.items(): + if col in shared_new: + comparable_new[col] = shared_new[col] + else: + comparable_new[col] = _sanitize_unknown_types(new_type) + + return comparable_old, comparable_new + + def get_schema_differ( dialect: str, overrides: t.Optional[t.Dict[str, t.Any]] = None ) -> SchemaDiffer: diff --git a/sqlmesh/core/snapshot/evaluator.py b/sqlmesh/core/snapshot/evaluator.py index 345f8816e6..997fd4d8b0 100644 --- a/sqlmesh/core/snapshot/evaluator.py +++ b/sqlmesh/core/snapshot/evaluator.py @@ -755,6 +755,7 @@ def _evaluate_snapshot( # If there are no existing intervals yet; only consider this a first insert for the first snapshot in the batch if target_table_exists is None: target_table_exists = adapter.table_exists(target_table_name) + was_creating = not target_table_exists is_first_insert = ( not _intervals(snapshot, deployability_index) or not target_table_exists ) and batch_index == 0 @@ -812,6 +813,8 @@ def _evaluate_snapshot( render_kwargs={**render_statements_kwargs, "inside_transaction": True}, ) + cloned_in_dev = False + created_before_insert = False if not target_table_exists or (model.is_seed and not snapshot.intervals): # Only create the empty table if the columns were provided explicitly by the user should_create_empty_table = ( @@ -832,6 +835,7 @@ def _evaluate_snapshot( allow_destructive_snapshots=allow_destructive_snapshots, allow_additive_snapshots=allow_additive_snapshots, ) + cloned_in_dev = True runtime_stage = RuntimeStage.EVALUATING target_table_exists = True elif should_create_empty_table or model.is_seed or model.kind.is_scd_type_2: @@ -845,6 +849,15 @@ def _evaluate_snapshot( dry_run=False, run_pre_post_statements=False, ) + created_before_insert = True + if not is_snapshot_deployable: + self._check_dev_schema_compatibility( + snapshot=snapshot, + dev_table_name=target_table_name, + deployability_index=deployability_index, + allow_destructive_snapshots=allow_destructive_snapshots, + allow_additive_snapshots=allow_additive_snapshots, + ) runtime_stage = RuntimeStage.EVALUATING target_table_exists = True @@ -880,6 +893,20 @@ def _evaluate_snapshot( batch_index=batch_index, ) + if ( + was_creating + and not is_snapshot_deployable + and not cloned_in_dev + and not created_before_insert + ): + self._check_dev_schema_compatibility( + snapshot=snapshot, + dev_table_name=target_table_name, + deployability_index=deployability_index, + allow_destructive_snapshots=allow_destructive_snapshots, + allow_additive_snapshots=allow_additive_snapshots, + ) + evaluation_strategy.run_post_statements( snapshot=snapshot, render_kwargs={**render_statements_kwargs, "inside_transaction": True}, @@ -954,15 +981,24 @@ def create_snapshot( ) else: is_table_deployable = deployability_index.is_deployable(snapshot) + target_table_name = snapshot.table_name(is_deployable=is_table_deployable) self._execute_create( snapshot=snapshot, - table_name=snapshot.table_name(is_deployable=is_table_deployable), + table_name=target_table_name, is_table_deployable=is_table_deployable, deployability_index=deployability_index, create_render_kwargs=create_render_kwargs, rendered_physical_properties=rendered_physical_properties, dry_run=True, ) + if not is_table_deployable: + self._check_dev_schema_compatibility( + snapshot=snapshot, + dev_table_name=target_table_name, + deployability_index=deployability_index, + allow_destructive_snapshots=allow_destructive_snapshots, + allow_additive_snapshots=allow_additive_snapshots, + ) evaluation_strategy.run_post_statements( snapshot=snapshot, render_kwargs={**create_render_kwargs, "inside_transaction": False} @@ -1154,6 +1190,59 @@ def _clone_snapshot_in_dev( adapter.drop_table(target_table_name) raise + def _check_dev_schema_compatibility( + self, + snapshot: Snapshot, + dev_table_name: str, + deployability_index: DeployabilityIndex, + allow_destructive_snapshots: t.Set[str], + allow_additive_snapshots: t.Set[str], + ) -> None: + if ( + not snapshot.is_model + or not snapshot.is_materialized + or snapshot.is_seed + or snapshot.is_managed + or snapshot.is_dbt_custom + or not snapshot.is_no_rebuild + or not snapshot.previous_versions + or deployability_index.is_deployable(snapshot) + ): + return + + needs_destructive = snapshot.needs_destructive_check(allow_destructive_snapshots) + needs_additive = snapshot.needs_additive_check(allow_additive_snapshots) + if not (needs_destructive or needs_additive): + return + + adapter = self.get_adapter(snapshot.model.gateway) + prod_table_name = snapshot.table_name(is_deployable=True) + try: + if ( + prod_table_name == dev_table_name + or not adapter.table_exists(prod_table_name) + or not adapter.table_exists(dev_table_name) + ): + return + + alter_operations = adapter.get_alter_operations( + prod_table_name, + dev_table_name, + ignore_destructive=snapshot.model.on_destructive_change.is_ignore, + ignore_additive=snapshot.model.on_additive_change.is_ignore, + ) + _check_destructive_schema_change( + snapshot, alter_operations, allow_destructive_snapshots + ) + _check_additive_schema_change(snapshot, alter_operations, allow_additive_snapshots) + except Exception: + # Don't leave an unvalidated preview behind: a retry would skip creating it again. + try: + adapter.drop_table(dev_table_name) + except Exception: + logger.exception("Failed to drop the dev preview table '%s'", dev_table_name) + raise + @observe_snapshot("physical", "schema_migration") def _migrate_snapshot( self, @@ -3402,9 +3491,10 @@ def _check_destructive_schema_change( ) ) return - raise DestructiveChangeError( - format_destructive_change_msg(snapshot_name, alter_operations, model_dialect) - ) + if snapshot.model.on_destructive_change.is_error: + raise DestructiveChangeError( + format_destructive_change_msg(snapshot_name, alter_operations, model_dialect) + ) def _check_additive_schema_change( @@ -3416,8 +3506,12 @@ def _check_additive_schema_change( if not isinstance(snapshot.model.kind, _Incremental): return - if snapshot.needs_additive_check(allow_additive_snapshots) and has_additive_alteration( - alter_operations + # Snapshots that get a fresh physical table can't have an additive change applied to them. + # Pinned physical versions (and non-VDE modes) are migrated in place even when they rebuild. + if ( + (snapshot.is_no_rebuild or snapshot.requires_schema_migration_in_prod) + and snapshot.needs_additive_check(allow_additive_snapshots) + and has_additive_alteration(alter_operations) ): # Note: IGNORE filtering is applied before this function is called # so if we reach here, additive changes are not being ignored diff --git a/tests/core/integration/test_forward_only.py b/tests/core/integration/test_forward_only.py index 2dddf18efd..68af45b757 100644 --- a/tests/core/integration/test_forward_only.py +++ b/tests/core/integration/test_forward_only.py @@ -1,12 +1,21 @@ from __future__ import annotations +from pathlib import Path import typing as t import numpy as np # noqa: TID253 import pandas as pd # noqa: TID253 import pytest +from sqlglot import exp import time_machine +from pytest_mock.plugin import MockerFixture from sqlmesh.core import dialect as d +from sqlmesh.core.config import ( + Config, + DuckDBConnectionConfig, + GatewayConfig, + ModelDefaultsConfig, +) from sqlmesh.core.context import Context from sqlmesh.core.config.categorizer import CategorizerConfig from sqlmesh.core.model import ( @@ -19,6 +28,7 @@ SnapshotChangeCategory, ) from sqlmesh.utils.date import to_datetime, to_timestamp +from sqlmesh.utils.errors import AdditiveChangeError, DestructiveChangeError, PlanError from tests.core.integration.utils import add_projection_to_model pytestmark = pytest.mark.slow @@ -1495,3 +1505,337 @@ def test_new_forward_only_model_same_dev_environment(init_and_plan_context: t.Ca df = context.fetchdf("SELECT * FROM memory.sushi__dev.new_model").replace({np.nan: None}) assert df.to_dict() == {"ds": {0: "2023-01-07"}, "b": {0: 1}} + + +def _make_duckdb_context(project_path: Path) -> Context: + cfg = Config( + gateways={"default": GatewayConfig(connection=DuckDBConnectionConfig())}, + default_gateway="default", + model_defaults=ModelDefaultsConfig(dialect="duckdb"), + ) + return Context(paths=[project_path], config=cfg) + + +def _assert_failed_on_db_down(spy: t.Any, error_type: t.Type[Exception]) -> None: + """Asserts that dev apply failed because schema validation of `db.down` raised `error_type`.""" + errors = [e for call in spy.call_args_list for e in call.args[0]] + assert any( + isinstance(e.__cause__, error_type) and '"db"."down"' in str(e.__cause__) for e in errors + ), [repr(e.__cause__) for e in errors] + + +def test_unannotated_star_destructive_and_additive_caught_in_dev_apply( + tmp_path: Path, mocker: MockerFixture +): + """Gap S6 (SQL SELECT *): On non-cloning engines (DuckDB, SUPPORTS_CLONING=False), destructive + and disallowed additive changes through unannotated SELECT * chains must fail during dev apply, + leave prod untouched, and clean up the temporary __dev preview table.""" + models_dir = tmp_path / "models" + models_dir.mkdir(parents=True) + (models_dir / "up.sql").write_text( + """ + MODEL (name db.up, kind FULL); + SELECT * FROM raw_src; + """ + ) + (models_dir / "down.sql").write_text( + """ + MODEL ( + name db.down, + kind INCREMENTAL_BY_TIME_RANGE ( + time_column ds, + forward_only true, + on_additive_change error + ), + start '2023-01-01' + ); + SELECT * FROM db.up WHERE ds BETWEEN @start_ds AND @end_ds; + """ + ) + + context = _make_duckdb_context(tmp_path) + failed_models_spy = mocker.spy(context.console, "log_failed_models") + assert context.engine_adapter.SUPPORTS_CLONING is False + assert context.get_model("db.up").columns_to_types is None + assert context.get_model("db.down").columns_to_types is None + + context.engine_adapter.execute( + "CREATE TABLE raw_src AS SELECT 1 AS id, 'x' AS b, '2023-01-01' AS ds" + ) + context.plan("prod", execution_time="2023-01-02", auto_apply=True, no_prompts=True) + assert set(context.engine_adapter.columns("db.down")) == {"id", "b", "ds"} + + # 1. Drop column 'b' in raw_src and touch db.up -> dev apply fails, prod intact, __dev dropped + context.engine_adapter.execute("ALTER TABLE raw_src DROP COLUMN b") + (models_dir / "up.sql").write_text( + """ + MODEL (name db.up, kind FULL); + SELECT * FROM raw_src WHERE id > 0; + """ + ) + context.load() + + failed_models_spy.reset_mock() + with pytest.raises(PlanError): + context.plan("dev", execution_time="2023-01-02", auto_apply=True, no_prompts=True) + _assert_failed_on_db_down(failed_models_spy, DestructiveChangeError) + + down_snapshot = context.get_snapshot("db.down", raise_if_missing=True) + assert set(context.engine_adapter.columns("db.down")) == {"id", "b", "ds"} + prod_env = context.state_reader.get_environment("prod") + assert prod_env is not None and prod_env.finalized_ts is not None + assert not context.engine_adapter.table_exists(down_snapshot.table_name(is_deployable=False)) + + # 2. Add column 'c' in raw_src with on_additive_change=error -> dev apply fails and drops __dev + context.engine_adapter.execute( + "CREATE OR REPLACE TABLE raw_src AS SELECT 1 AS id, 'x' AS b, 2 AS c, '2023-01-01' AS ds" + ) + (models_dir / "up.sql").write_text( + """ + MODEL (name db.up, kind FULL); + SELECT * FROM raw_src WHERE id >= 1; + """ + ) + context.load() + + failed_models_spy.reset_mock() + with pytest.raises(PlanError): + context.plan("dev", execution_time="2023-01-02", auto_apply=True, no_prompts=True) + _assert_failed_on_db_down(failed_models_spy, AdditiveChangeError) + + down_snapshot_add = context.get_snapshot("db.down", raise_if_missing=True) + assert not context.engine_adapter.table_exists( + down_snapshot_add.table_name(is_deployable=False) + ) + assert set(context.engine_adapter.columns("db.down")) == {"id", "b", "ds"} + + # 3. Re-apply destructive drop with allow_destructive_models -> succeeds in dev AND prod + context.engine_adapter.execute( + "CREATE OR REPLACE TABLE raw_src AS SELECT 1 AS id, '2023-01-01' AS ds" + ) + (models_dir / "up.sql").write_text( + """ + MODEL (name db.up, kind FULL); + SELECT * FROM raw_src WHERE id = 1; + """ + ) + context.load() + + context.plan( + "dev", + execution_time="2023-01-02", + auto_apply=True, + no_prompts=True, + allow_destructive_models=["db.up+"], + ) + assert set(context.engine_adapter.columns("db__dev.down")) == {"id", "ds"} + + context.plan( + "prod", + execution_time="2023-01-02", + auto_apply=True, + no_prompts=True, + allow_destructive_models=["db.down"], + ) + assert set(context.engine_adapter.columns("db.down")) == {"id", "ds"} + + +def test_unannotated_python_model_downstream_forward_only_destructive_caught_in_dev( + tmp_path: Path, mocker: MockerFixture +): + """Gap S6 (Python @model without columns metadata): Dynamic Python SQL model upstream of a + forward_only=True SELECT * model surfaces destructive column drops during dev apply.""" + models_dir = tmp_path / "models" + models_dir.mkdir(parents=True) + (models_dir / "py_up.py").write_text( + """ +from sqlglot import parse_one +from sqlmesh import model + +@model("db.py_up", is_sql=True, kind="full") +def execute(evaluator): + if evaluator.runtime_stage == "loading": + return parse_one("SELECT * FROM raw_src") + return parse_one("SELECT 1 AS id, 'x' AS b, '2023-01-01' AS ds") +""" + ) + (models_dir / "down.sql").write_text( + """ + MODEL ( + name db.down, + kind INCREMENTAL_BY_TIME_RANGE (time_column ds, forward_only true), + start '2023-01-01' + ); + SELECT * FROM db.py_up WHERE ds BETWEEN @start_ds AND @end_ds; + """ + ) + + context = _make_duckdb_context(tmp_path) + failed_models_spy = mocker.spy(context.console, "log_failed_models") + assert context.get_model("db.py_up").columns_to_types is None + assert context.get_model("db.down").columns_to_types is None + + context.plan("prod", execution_time="2023-01-02", auto_apply=True, no_prompts=True) + assert set(context.engine_adapter.columns("db.down")) == {"id", "b", "ds"} + + # Drop column 'b' in db.py_up + (models_dir / "py_up.py").write_text( + """ +from sqlglot import parse_one +from sqlmesh import model + +@model("db.py_up", is_sql=True, kind="full") +def execute(evaluator): + if evaluator.runtime_stage == "loading": + return parse_one("SELECT * FROM raw_src") + return parse_one("SELECT 1 AS id, '2023-01-01' AS ds") +""" + ) + context.load() + + failed_models_spy.reset_mock() + with pytest.raises(PlanError): + context.plan("dev", execution_time="2023-01-02", auto_apply=True, no_prompts=True) + _assert_failed_on_db_down(failed_models_spy, DestructiveChangeError) + + down_snapshot = context.get_snapshot("db.down", raise_if_missing=True) + assert set(context.engine_adapter.columns("db.down")) == {"id", "b", "ds"} + assert not context.engine_adapter.table_exists(down_snapshot.table_name(is_deployable=False)) + + # Allowing db.down succeeds in both dev and prod + context.plan( + "dev", + execution_time="2023-01-02", + auto_apply=True, + no_prompts=True, + allow_destructive_models=["db.down"], + ) + context.plan( + "prod", + execution_time="2023-01-02", + auto_apply=True, + no_prompts=True, + allow_destructive_models=["db.down"], + ) + assert set(context.engine_adapter.columns("db.down")) == {"id", "ds"} + + +def test_forward_only_dev_plan_destructive_change_blocks_prod_plan_until_allowed( + tmp_path: Path, +): + """Gap S9 end-to-end: Applying a destructive change in dev via --forward-only + --allow-destructive-model stores a forward_only=True snapshot in state. A subsequent prod plan + without --allow-destructive-model must fail immediately at plan_builder.build() before touching + prod, and then succeed once allow_destructive_models is provided.""" + models_dir = tmp_path / "models" + models_dir.mkdir(parents=True) + (models_dir / "m.sql").write_text( + """ + MODEL ( + name db.m, + kind INCREMENTAL_BY_TIME_RANGE (time_column ds), + start '2023-01-01' + ); + SELECT 1::INT AS id, 'x'::TEXT AS val, '2023-01-01'::DATE AS ds; + """ + ) + + context = _make_duckdb_context(tmp_path) + context.plan("prod", execution_time="2023-01-02", auto_apply=True, no_prompts=True) + assert context.engine_adapter.columns("db.m")["val"] == exp.DataType.build("TEXT") + + # Destructive column type change TEXT -> INT applied to dev as --forward-only + (models_dir / "m.sql").write_text( + """ + MODEL ( + name db.m, + kind INCREMENTAL_BY_TIME_RANGE (time_column ds), + start '2023-01-01' + ); + SELECT 1::INT AS id, 1::INT AS val, '2023-01-01'::DATE AS ds; + """ + ) + context.load() + context.plan( + "dev", + execution_time="2023-01-02", + forward_only=True, + allow_destructive_models=["db.m"], + auto_apply=True, + no_prompts=True, + ) + + # Prod plan without allow_destructive_models MUST fail at PlanBuilder.build() before touching prod + with pytest.raises( + PlanError, match=r"Plan requires a destructive change to a forward-only model\." + ): + context.plan_builder("prod", execution_time="2023-01-02").build() + + prod_env = context.state_reader.get_environment("prod") + assert prod_env is not None and prod_env.finalized_ts is not None + assert context.engine_adapter.columns("db.m")["val"] == exp.DataType.build("TEXT") + + # Prod plan WITH allow_destructive_models succeeds + context.plan( + "prod", + execution_time="2023-01-02", + allow_destructive_models=["db.m"], + auto_apply=True, + no_prompts=True, + ) + assert context.engine_adapter.columns("db.m")["val"] == exp.DataType.build("INT") + + +def test_drop_after_ignored_additive_change_still_caught_by_physical_validation( + tmp_path: Path, mocker: MockerFixture +): + """The static plan check defers drops when the previous definition ignored additive changes + (the physical table may not have those columns), but dropping a column that really exists + physically must still be rejected by the physical validation during dev apply.""" + models_dir = tmp_path / "models" + models_dir.mkdir(parents=True) + + def write_model(columns: str, *, on_additive: str) -> None: + (models_dir / "down.sql").write_text( + f""" + MODEL ( + name db.down, + kind INCREMENTAL_BY_TIME_RANGE ( + time_column ds, + forward_only true, + on_destructive_change error, + on_additive_change {on_additive} + ), + start '2023-01-01' + ); + SELECT {columns} FROM raw_src WHERE ds BETWEEN @start_ds AND @end_ds; + """ + ) + + write_model("id::INT AS id, val::TEXT AS val, ds::DATE AS ds", on_additive="error") + context = _make_duckdb_context(tmp_path) + failed_models_spy = mocker.spy(context.console, "log_failed_models") + context.engine_adapter.execute( + "CREATE TABLE raw_src AS SELECT 1 AS id, 'x' AS val, 2 AS c, '2023-01-01' AS ds" + ) + context.plan("prod", execution_time="2023-01-02", auto_apply=True, no_prompts=True) + assert set(context.engine_adapter.columns("db.down")) == {"id", "val", "ds"} + + # Adds `c` under on_additive_change=ignore: the physical table never gets it. + write_model( + "id::INT AS id, val::TEXT AS val, ds::DATE AS ds, c::INT AS c", on_additive="ignore" + ) + context.load() + context.plan("prod", execution_time="2023-01-02", auto_apply=True, no_prompts=True) + assert set(context.engine_adapter.columns("db.down")) == {"id", "val", "ds"} + + # Drops `c` (never physical, deferred) and `val` (physical, must be rejected). + write_model("id::INT AS id, ds::DATE AS ds", on_additive="ignore") + context.load() + failed_models_spy.reset_mock() + with pytest.raises(PlanError): + context.plan("dev", execution_time="2023-01-02", auto_apply=True, no_prompts=True) + _assert_failed_on_db_down(failed_models_spy, DestructiveChangeError) + assert set(context.engine_adapter.columns("db.down")) == {"id", "val", "ds"} + down_snapshot = context.get_snapshot("db.down", raise_if_missing=True) + assert not context.engine_adapter.table_exists(down_snapshot.table_name(is_deployable=False)) diff --git a/tests/core/test_plan.py b/tests/core/test_plan.py index add85d2eec..4b4462731a 100644 --- a/tests/core/test_plan.py +++ b/tests/core/test_plan.py @@ -1,11 +1,21 @@ import logging import typing as t from datetime import timedelta +from pathlib import Path +from tempfile import TemporaryDirectory from unittest.mock import patch import pytest +from sqlmesh.core.config import ( + Config, + DuckDBConnectionConfig, + GatewayConfig, + ModelDefaultsConfig, + PostgresConnectionConfig, +) from sqlmesh.core.console import TerminalConsole +from sqlmesh.utils import UniqueKeyDict from sqlmesh.utils.metaprogramming import Executable from tests.core.test_table_diff import create_test_console import time_machine @@ -17,16 +27,21 @@ from sqlmesh.core.context_diff import ContextDiff from sqlmesh.core.environment import EnvironmentNamingInfo, EnvironmentStatements from sqlmesh.core.model import ( + EmbeddedKind, ExternalModel, FullKind, + IncrementalByPartitionKind, IncrementalByTimeRangeKind, IncrementalUnmanagedKind, + Model, + SCDType2ByTimeKind, SeedKind, SeedModel, SqlModel, ModelKindName, ) from sqlmesh.core.model.kind import OnDestructiveChange, OnAdditiveChange, ViewKind +from sqlmesh.core.model.schema import update_model_schemas from sqlmesh.core.model.seed import Seed from sqlmesh.core.plan import Plan, PlanBuilder, SnapshotIntervals from sqlmesh.core.snapshot import ( @@ -104,7 +119,7 @@ def test_forward_only_dev(make_snapshot, mocker: MockerFixture): SqlModel( name="a", dialect="duckdb", - query=parse_one("select 1, ds"), + query=parse_one("select 1 as a, ds"), kind=dict(name=ModelKindName.INCREMENTAL_BY_TIME_RANGE, time_column="ds"), ) ) @@ -114,7 +129,7 @@ def test_forward_only_dev(make_snapshot, mocker: MockerFixture): SqlModel( **{ **snapshot.model.dict(), - "query": parse_one("select 2, ds"), + "query": parse_one("select 2 as a, ds"), } ) ) @@ -1697,7 +1712,7 @@ def test_forward_only_models(make_snapshot, mocker: MockerFixture): snapshot = make_snapshot( SqlModel( name="a", - query=parse_one("select 1, ds"), + query=parse_one("select 1 as a, ds"), dialect="duckdb", kind=IncrementalByTimeRangeKind(time_column="ds", forward_only=True), ) @@ -1706,7 +1721,7 @@ def test_forward_only_models(make_snapshot, mocker: MockerFixture): updated_snapshot = make_snapshot( SqlModel( name="a", - query=parse_one("select 3, ds"), + query=parse_one("select 3 as a, ds"), kind=IncrementalByTimeRangeKind(time_column="ds", forward_only=True), dialect="duckdb", ) @@ -1801,7 +1816,7 @@ def test_forward_only_models_model_kind_changed_to_incremental_by_time_range( snapshot = make_snapshot( SqlModel( name="a", - query=parse_one("select 1, ds"), + query=parse_one("select 1 as a, ds"), kind=IncrementalUnmanagedKind(), partitioned_by=partitioned_by, ) @@ -1810,7 +1825,7 @@ def test_forward_only_models_model_kind_changed_to_incremental_by_time_range( updated_snapshot = make_snapshot( SqlModel( name="a", - query=parse_one("select 3, ds"), + query=parse_one("select 3 as a, ds"), kind=IncrementalByTimeRangeKind(time_column="ds", forward_only=True), ) ) @@ -2869,7 +2884,7 @@ def test_unaligned_start_model_with_forward_only_preview(make_snapshot): snapshot_a = make_snapshot( SqlModel( name="a", - query=parse_one("select 1, ds"), + query=parse_one("select 1 as a, ds"), kind=dict( name=ModelKindName.INCREMENTAL_BY_TIME_RANGE, forward_only=True, time_column="ds" ), @@ -2880,7 +2895,7 @@ def test_unaligned_start_model_with_forward_only_preview(make_snapshot): new_snapshot_a = make_snapshot( SqlModel( name="a", - query=parse_one("select 2, ds"), + query=parse_one("select 2 as a, ds"), kind=dict( name=ModelKindName.INCREMENTAL_BY_TIME_RANGE, forward_only=True, time_column="ds" ), @@ -2939,7 +2954,7 @@ def _make_forward_only_preview_context_diff(make_snapshot): old_snapshot = make_snapshot( SqlModel( name="a", - query=parse_one("select 1, ds"), + query=parse_one("select 1 as a, ds"), kind=dict( name=ModelKindName.INCREMENTAL_BY_TIME_RANGE, forward_only=True, @@ -2953,7 +2968,7 @@ def _make_forward_only_preview_context_diff(make_snapshot): new_snapshot = make_snapshot( SqlModel( name="a", - query=parse_one("select 2, ds"), + query=parse_one("select 2 as a, ds"), kind=dict( name=ModelKindName.INCREMENTAL_BY_TIME_RANGE, forward_only=True, @@ -3385,7 +3400,7 @@ def test_set_choice_for_forward_only_model(make_snapshot): snapshot = make_snapshot( SqlModel( name="a", - query=parse_one("select 1, ds"), + query=parse_one("select 1 as a, ds"), dialect="duckdb", kind=IncrementalByTimeRangeKind(time_column="ds", forward_only=True), ) @@ -3394,7 +3409,7 @@ def test_set_choice_for_forward_only_model(make_snapshot): updated_snapshot = make_snapshot( SqlModel( name="a", - query=parse_one("select 3, ds"), + query=parse_one("select 3 as a, ds"), kind=IncrementalByTimeRangeKind(time_column="ds", forward_only=True), dialect="duckdb", ) @@ -4510,3 +4525,1371 @@ def test_forward_only_indirect_change_to_materialized_view(make_snapshot): # Forward-only indirect changes to MVs should not always be classified as indirect breaking. # Instead, we want to preserve the standard categorization. assert snapshot_b_new.change_category == SnapshotChangeCategory.INDIRECT_NON_BREAKING + + +def _hydrate_model_schemas(*models: Model) -> None: + dag: DAG[str] = DAG() + model_map: UniqueKeyDict[str, Model] = UniqueKeyDict("models") + for m in models: + model_map[m.fqn] = m + dag.add(m.fqn, m.depends_on) + with TemporaryDirectory() as tmp_dir: + update_model_schemas(dag, model_map, Path(tmp_dir)) + + +def _build_snapshots_and_diff( + make_snapshot: t.Callable, + old_models: t.Sequence[Model], + new_models: t.Sequence[Model], + *, + environment: str = "dev", + is_new_environment: bool = True, + new_snapshot_names: t.Optional[t.Set[str]] = None, +) -> t.Tuple[t.Dict[str, Snapshot], t.Dict[str, Snapshot], ContextDiff]: + _hydrate_model_schemas(*old_models) + _hydrate_model_schemas(*new_models) + + old_nodes = {m.fqn: m for m in old_models} + new_nodes = {m.fqn: m for m in new_models} + + old_snapshots: t.Dict[str, Snapshot] = {} + for m in old_models: + snap = make_snapshot(m, nodes=old_nodes) + snap.categorize_as(SnapshotChangeCategory.BREAKING) + old_snapshots[m.fqn] = snap + + new_snapshots: t.Dict[str, Snapshot] = {} + for m in new_models: + snap = make_snapshot(m, nodes=new_nodes) + if m.fqn in old_snapshots: + snap.previous_versions = old_snapshots[m.fqn].all_versions + new_snapshots[m.fqn] = snap + + modified_snapshots = { + fqn: (new_snap, old_snapshots[fqn]) + for fqn, new_snap in new_snapshots.items() + if fqn in old_snapshots and new_snap.fingerprint != old_snapshots[fqn].fingerprint + } + snapshots_by_id = {snap.snapshot_id: snap for snap in new_snapshots.values()} + if new_snapshot_names is None: + new_snapshots_by_id = { + snap.snapshot_id: snap + for fqn, snap in new_snapshots.items() + if fqn in modified_snapshots + } + else: + new_snapshots_by_id = { + snap.snapshot_id: snap + for fqn, snap in new_snapshots.items() + if fqn in new_snapshot_names + } + + context_diff = ContextDiff( + environment=environment, + is_new_environment=is_new_environment, + is_unfinalized_environment=False, + normalize_environment_name=True, + create_from="prod", + create_from_env_exists=True, + added=set(), + removed_snapshots={}, + modified_snapshots=modified_snapshots, + snapshots=snapshots_by_id, + new_snapshots=new_snapshots_by_id, + previous_plan_id=None, + previously_promoted_snapshot_ids=set(), + previous_finalized_snapshots=None, + previous_gateway_managed_virtual_layer=False, + gateway_managed_virtual_layer=False, + environment_statements=[], + ) + return old_snapshots, new_snapshots, context_diff + + +def test_indirectly_modified_forward_only_model_destructive_change( + make_snapshot, mocker: MockerFixture +): + """Gap S2 (destructive): Upstream FULL model incompatibly alters a column type or drops a + nested struct field while an indirectly_modified downstream forward-only model selects it.""" + up_old = SqlModel( + name="db.up", + dialect="postgres", + kind=FullKind(), + query=parse_one( + "SELECT 1::INT AS id, '1'::TEXT AS val, '2022-01-01'::DATE AS ds", read="postgres" + ), + ) + up_new = SqlModel( + name="db.up", + dialect="postgres", + kind=FullKind(), + query=parse_one( + "SELECT 1::INT AS id, 1::INT AS val, '2022-01-01'::DATE AS ds", read="postgres" + ), + ) + + def _make_time_range_diff(on_destructive: OnDestructiveChange) -> ContextDiff: + down_old = SqlModel( + name="db.down", + dialect="postgres", + kind=IncrementalByTimeRangeKind( + time_column="ds", + forward_only=True, + on_destructive_change=on_destructive, + ), + query=parse_one("SELECT val, ds FROM db.up", read="postgres"), + ) + down_new = SqlModel( + name="db.down", + dialect="postgres", + kind=IncrementalByTimeRangeKind( + time_column="ds", + forward_only=True, + on_destructive_change=on_destructive, + ), + query=parse_one("SELECT val, ds FROM db.up", read="postgres"), + ) + _, _, diff = _build_snapshots_and_diff( + make_snapshot, [up_old.copy(), down_old], [up_new.copy(), down_new] + ) + assert diff.indirectly_modified('"db"."down"') + return diff + + # 1. ERROR: logs destructive change with error=True and raises PlanError + console = TerminalConsole() + log_destructive_spy = mocker.spy(console, "log_destructive_change") + diff_error = _make_time_range_diff(OnDestructiveChange.ERROR) + with pytest.raises( + PlanError, match=r"Plan requires a destructive change to a forward-only model\." + ): + PlanBuilder(diff_error, console=console).build() + assert log_destructive_spy.call_count == 1 + assert log_destructive_spy.call_args[0][0] == '"db"."down"' + assert log_destructive_spy.call_args[1]["error"] is True + + # 2. allow_destructive_models overrides ERROR without logging or raising + log_destructive_spy.reset_mock() + diff_allowed = _make_time_range_diff(OnDestructiveChange.ERROR) + assert PlanBuilder( + diff_allowed, + allow_destructive_models={'"db"."down"'}, + console=console, + ).build() + assert log_destructive_spy.call_count == 0 + + # 3. WARN: logs destructive change with error=False without raising + log_destructive_spy.reset_mock() + diff_warn = _make_time_range_diff(OnDestructiveChange.WARN) + assert PlanBuilder(diff_warn, console=console).build() + assert log_destructive_spy.call_count == 1 + assert log_destructive_spy.call_args[0][0] == '"db"."down"' + assert log_destructive_spy.call_args[1]["error"] is False + + # 4. ALLOW & IGNORE: succeed silently without logging + for permissive_mode in (OnDestructiveChange.ALLOW, OnDestructiveChange.IGNORE): + log_destructive_spy.reset_mock() + diff_permissive = _make_time_range_diff(permissive_mode) + assert PlanBuilder(diff_permissive, console=console).build() + assert log_destructive_spy.call_count == 0 + + # 5. INCREMENTAL_BY_PARTITION (forward_only=True by default) + part_old = SqlModel( + name="db.part_down", + dialect="postgres", + kind=IncrementalByPartitionKind(), + partitioned_by=["ds"], + query=parse_one("SELECT val, ds FROM db.up", read="postgres"), + ) + part_new = SqlModel( + name="db.part_down", + dialect="postgres", + kind=IncrementalByPartitionKind(), + partitioned_by=["ds"], + query=parse_one("SELECT val, ds FROM db.up", read="postgres"), + ) + _, _, diff_part = _build_snapshots_and_diff( + make_snapshot, [up_old.copy(), part_old], [up_new.copy(), part_new] + ) + with pytest.raises( + PlanError, match=r"Plan requires a destructive change to a forward-only model\." + ): + PlanBuilder(diff_part).build() + + # 6. SCD_TYPE_2_BY_TIME (forward_only=True by default) + scd_old = SqlModel( + name="db.scd_down", + dialect="postgres", + kind=SCDType2ByTimeKind(unique_key=["id"], updated_at_name="ds"), + query=parse_one("SELECT id, val, ds FROM db.up", read="postgres"), + ) + scd_new = SqlModel( + name="db.scd_down", + dialect="postgres", + kind=SCDType2ByTimeKind(unique_key=["id"], updated_at_name="ds"), + query=parse_one("SELECT id, val, ds FROM db.up", read="postgres"), + ) + _, _, diff_scd = _build_snapshots_and_diff( + make_snapshot, [up_old.copy(), scd_old], [up_new.copy(), scd_new] + ) + with pytest.raises( + PlanError, match=r"Plan requires a destructive change to a forward-only model\." + ): + PlanBuilder(diff_scd).build() + + # 7. STRUCT nested field drop on indirectly_modified forward-only downstream model (databricks) + struct_up_old = SqlModel( + name="db.struct_up", + dialect="databricks", + kind=FullKind(), + query=parse_one( + "SELECT CAST(NAMED_STRUCT('a', 1, 'b', 'x') AS STRUCT) AS info, " + "CAST('2022-01-01' AS DATE) AS ds", + read="databricks", + ), + ) + struct_up_new = SqlModel( + name="db.struct_up", + dialect="databricks", + kind=FullKind(), + query=parse_one( + "SELECT CAST(NAMED_STRUCT('a', 1) AS STRUCT) AS info, " + "CAST('2022-01-01' AS DATE) AS ds", + read="databricks", + ), + ) + struct_down_old = SqlModel( + name="db.struct_down", + dialect="databricks", + kind=IncrementalByTimeRangeKind(time_column="ds", forward_only=True), + query=parse_one("SELECT info, ds FROM db.struct_up", read="databricks"), + ) + struct_down_new = SqlModel( + name="db.struct_down", + dialect="databricks", + kind=IncrementalByTimeRangeKind(time_column="ds", forward_only=True), + query=parse_one("SELECT info, ds FROM db.struct_up", read="databricks"), + ) + _, _, diff_struct_drop = _build_snapshots_and_diff( + make_snapshot, [struct_up_old, struct_down_old], [struct_up_new, struct_down_new] + ) + assert diff_struct_drop.indirectly_modified('"db"."struct_down"') + with pytest.raises( + PlanError, match=r"Plan requires a destructive change to a forward-only model\." + ): + PlanBuilder(diff_struct_drop).build() + + +def test_indirectly_modified_forward_only_model_additive_change( + make_snapshot, mocker: MockerFixture +): + """Gap S2 (additive): Upstream FULL model triggers a compatible type widening or nested struct + field addition on an indirectly_modified downstream forward-only model.""" + up_old = SqlModel( + name="db.up", + dialect="postgres", + kind=FullKind(), + query=parse_one( + "SELECT 'x'::VARCHAR(10) AS val, '2022-01-01'::DATE AS ds", read="postgres" + ), + ) + up_new = SqlModel( + name="db.up", + dialect="postgres", + kind=FullKind(), + query=parse_one( + "SELECT 'x'::VARCHAR(50) AS val, '2022-01-01'::DATE AS ds", read="postgres" + ), + ) + + def _make_additive_diff(on_additive: OnAdditiveChange) -> ContextDiff: + down_old = SqlModel( + name="db.down", + dialect="postgres", + kind=IncrementalByTimeRangeKind( + time_column="ds", + forward_only=True, + on_additive_change=on_additive, + ), + query=parse_one("SELECT val, ds FROM db.up", read="postgres"), + ) + down_new = SqlModel( + name="db.down", + dialect="postgres", + kind=IncrementalByTimeRangeKind( + time_column="ds", + forward_only=True, + on_additive_change=on_additive, + ), + query=parse_one("SELECT val, ds FROM db.up", read="postgres"), + ) + _, _, diff = _build_snapshots_and_diff( + make_snapshot, [up_old.copy(), down_old], [up_new.copy(), down_new] + ) + assert diff.indirectly_modified('"db"."down"') + return diff + + # 1. ERROR: logs additive change with error=True and raises PlanError + console = TerminalConsole() + log_additive_spy = mocker.spy(console, "log_additive_change") + diff_error = _make_additive_diff(OnAdditiveChange.ERROR) + with pytest.raises( + PlanError, match=r"Plan requires an additive change to a forward-only model\." + ): + PlanBuilder(diff_error, console=console).build() + assert log_additive_spy.call_count == 1 + assert log_additive_spy.call_args[0][0] == '"db"."down"' + assert log_additive_spy.call_args[1]["error"] is True + + # 2. allow_additive_models overrides ERROR without logging or raising + log_additive_spy.reset_mock() + diff_allowed = _make_additive_diff(OnAdditiveChange.ERROR) + assert PlanBuilder( + diff_allowed, + allow_additive_models={'"db"."down"'}, + console=console, + ).build() + assert log_additive_spy.call_count == 0 + + # 3. WARN: logs additive change with error=False without raising + log_additive_spy.reset_mock() + diff_warn = _make_additive_diff(OnAdditiveChange.WARN) + assert PlanBuilder(diff_warn, console=console).build() + assert log_additive_spy.call_count == 1 + assert log_additive_spy.call_args[0][0] == '"db"."down"' + assert log_additive_spy.call_args[1]["error"] is False + + # 4. ALLOW & IGNORE: succeed silently without logging + for permissive_mode in (OnAdditiveChange.ALLOW, OnAdditiveChange.IGNORE): + log_additive_spy.reset_mock() + diff_permissive = _make_additive_diff(permissive_mode) + assert PlanBuilder(diff_permissive, console=console).build() + assert log_additive_spy.call_count == 0 + + # 5. STRUCT nested field addition on indirectly_modified forward-only model (databricks) + struct_up_old = SqlModel( + name="db.struct_up", + dialect="databricks", + kind=FullKind(), + query=parse_one( + "SELECT CAST(NAMED_STRUCT('a', 1) AS STRUCT) AS info, " + "CAST('2022-01-01' AS DATE) AS ds", + read="databricks", + ), + ) + struct_up_new = SqlModel( + name="db.struct_up", + dialect="databricks", + kind=FullKind(), + query=parse_one( + "SELECT CAST(NAMED_STRUCT('a', 1, 'b', 'x') AS STRUCT) AS info, " + "CAST('2022-01-01' AS DATE) AS ds", + read="databricks", + ), + ) + struct_down_old = SqlModel( + name="db.struct_down", + dialect="databricks", + kind=IncrementalByTimeRangeKind( + time_column="ds", + forward_only=True, + on_additive_change=OnAdditiveChange.ERROR, + ), + query=parse_one("SELECT info, ds FROM db.struct_up", read="databricks"), + ) + struct_down_new = SqlModel( + name="db.struct_down", + dialect="databricks", + kind=IncrementalByTimeRangeKind( + time_column="ds", + forward_only=True, + on_additive_change=OnAdditiveChange.ERROR, + ), + query=parse_one("SELECT info, ds FROM db.struct_up", read="databricks"), + ) + _, _, diff_struct_add = _build_snapshots_and_diff( + make_snapshot, [struct_up_old, struct_down_old], [struct_up_new, struct_down_new] + ) + assert diff_struct_add.indirectly_modified('"db"."struct_down"') + with pytest.raises( + PlanError, match=r"Plan requires an additive change to a forward-only model\." + ): + PlanBuilder(diff_struct_add).build() + + +def test_indirect_downstream_inherits_forward_only_and_view_embedded_handling( + make_snapshot, mocker: MockerFixture +): + """Gap S3 + #2576 non-regression + VIEW/EMBEDDED intermediate & leaf handling.""" + # 1. Scenario S3: upstream forward_only=True (on_destructive_change=ALLOW) alters val TEXT->INT; + # downstream has model.forward_only=False (default on_destructive_change=ERROR) and inherits + # forward_only=True from all_upstream_forward_only == {True}, so PlanBuilder MUST raise PlanError. + up_old = SqlModel( + name="db.up", + dialect="postgres", + kind=IncrementalByTimeRangeKind( + time_column="ds", + forward_only=True, + on_destructive_change=OnDestructiveChange.ALLOW, + ), + query=parse_one("SELECT '1'::TEXT AS val, '2022-01-01'::DATE AS ds", read="postgres"), + ) + up_new = SqlModel( + name="db.up", + dialect="postgres", + kind=IncrementalByTimeRangeKind( + time_column="ds", + forward_only=True, + on_destructive_change=OnDestructiveChange.ALLOW, + ), + query=parse_one("SELECT 1::INT AS val, '2022-01-01'::DATE AS ds", read="postgres"), + ) + down_old = SqlModel( + name="db.down", + dialect="postgres", + kind=IncrementalByTimeRangeKind(time_column="ds", forward_only=False), + query=parse_one("SELECT val, ds FROM db.up", read="postgres"), + ) + down_new = SqlModel( + name="db.down", + dialect="postgres", + kind=IncrementalByTimeRangeKind(time_column="ds", forward_only=False), + query=parse_one("SELECT val, ds FROM db.up", read="postgres"), + ) + _, new_snaps_s3, diff_s3 = _build_snapshots_and_diff( + make_snapshot, [up_old, down_old], [up_new, down_new] + ) + with pytest.raises( + PlanError, match=r"Plan requires a destructive change to a forward-only model\." + ): + PlanBuilder(diff_s3, forward_only=False).build() + assert new_snaps_s3['"db"."down"'].is_forward_only + + # 2. #2576 non-regression: upstream drops unused_col, downstream only selects keep_col and ds + # (downstream's own schema did NOT change) -> succeeds with zero errors/warnings. + up_drop_old = SqlModel( + name="db.up", + dialect="postgres", + kind=IncrementalByTimeRangeKind( + time_column="ds", + forward_only=True, + on_destructive_change=OnDestructiveChange.ALLOW, + ), + query=parse_one( + "SELECT 1::INT AS keep_col, 'drop_me'::TEXT AS unused_col, '2022-01-01'::DATE AS ds", + read="postgres", + ), + ) + up_drop_new = SqlModel( + name="db.up", + dialect="postgres", + kind=IncrementalByTimeRangeKind( + time_column="ds", + forward_only=True, + on_destructive_change=OnDestructiveChange.ALLOW, + ), + query=parse_one( + "SELECT 1::INT AS keep_col, '2022-01-01'::DATE AS ds", + read="postgres", + ), + ) + down_keep_old = SqlModel( + name="db.down", + dialect="postgres", + kind=IncrementalByTimeRangeKind(time_column="ds", forward_only=False), + query=parse_one("SELECT keep_col, ds FROM db.up", read="postgres"), + ) + down_keep_new = SqlModel( + name="db.down", + dialect="postgres", + kind=IncrementalByTimeRangeKind(time_column="ds", forward_only=False), + query=parse_one("SELECT keep_col, ds FROM db.up", read="postgres"), + ) + _, new_snaps_keep, diff_keep = _build_snapshots_and_diff( + make_snapshot, [up_drop_old, down_keep_old], [up_drop_new, down_keep_new] + ) + console = TerminalConsole() + log_destructive_spy = mocker.spy(console, "log_destructive_change") + assert PlanBuilder(diff_keep, forward_only=False, console=console).build() + assert new_snaps_keep['"db"."down"'].is_forward_only + assert log_destructive_spy.call_count == 0 + + # 3. VIEW leaf does NOT raise PlanError (views recreate via CREATE OR REPLACE VIEW) + view_leaf_old = SqlModel( + name="db.v_leaf", + dialect="postgres", + kind=ViewKind(), + query=parse_one("SELECT val, ds FROM db.up", read="postgres"), + ) + view_leaf_new = SqlModel( + name="db.v_leaf", + dialect="postgres", + kind=ViewKind(), + query=parse_one("SELECT val, ds FROM db.up", read="postgres"), + ) + _, _, diff_view_leaf = _build_snapshots_and_diff( + make_snapshot, [up_old.copy(), view_leaf_old], [up_new.copy(), view_leaf_new] + ) + assert PlanBuilder(diff_view_leaf, forward_only=True, console=console).build() + assert log_destructive_spy.call_count == 0 + + # 4. Materialized forward_only=True model downstream of EMBEDDED -> VIEW DOES raise PlanError + emb_old = SqlModel( + name="db.emb", + dialect="postgres", + kind=EmbeddedKind(), + query=parse_one("SELECT '1'::TEXT AS val, '2022-01-01'::DATE AS ds", read="postgres"), + ) + emb_new = SqlModel( + name="db.emb", + dialect="postgres", + kind=EmbeddedKind(), + query=parse_one("SELECT 1::INT AS val, '2022-01-01'::DATE AS ds", read="postgres"), + ) + mid_view_old = SqlModel( + name="db.mid_view", + dialect="postgres", + kind=ViewKind(), + query=parse_one("SELECT val, ds FROM db.emb", read="postgres"), + ) + mid_view_new = SqlModel( + name="db.mid_view", + dialect="postgres", + kind=ViewKind(), + query=parse_one("SELECT val, ds FROM db.emb", read="postgres"), + ) + leaf_fo_old = SqlModel( + name="db.leaf_fo", + dialect="postgres", + kind=IncrementalByTimeRangeKind(time_column="ds", forward_only=True), + query=parse_one("SELECT val, ds FROM db.mid_view", read="postgres"), + ) + leaf_fo_new = SqlModel( + name="db.leaf_fo", + dialect="postgres", + kind=IncrementalByTimeRangeKind(time_column="ds", forward_only=True), + query=parse_one("SELECT val, ds FROM db.mid_view", read="postgres"), + ) + _, _, diff_chain = _build_snapshots_and_diff( + make_snapshot, + [emb_old, mid_view_old, leaf_fo_old], + [emb_new, mid_view_new, leaf_fo_new], + ) + with pytest.raises( + PlanError, match=r"Plan requires a destructive change to a forward-only model\." + ): + PlanBuilder(diff_chain).build() + + +def test_forward_only_plan_indirect_downstream_destructive_change(make_snapshot): + """Gap S4: PlanBuilder(..., forward_only=True) where upstream FULL model alters column type and + indirectly_modified downstream INCREMENTAL_BY_TIME_RANGE(forward_only=False) selects it.""" + up_old = SqlModel( + name="db.up", + dialect="postgres", + kind=FullKind(), + query=parse_one("SELECT '1'::TEXT AS val, '2022-01-01'::DATE AS ds", read="postgres"), + ) + up_new = SqlModel( + name="db.up", + dialect="postgres", + kind=FullKind(), + query=parse_one("SELECT 1::INT AS val, '2022-01-01'::DATE AS ds", read="postgres"), + ) + down_old = SqlModel( + name="db.down", + dialect="postgres", + kind=IncrementalByTimeRangeKind(time_column="ds", forward_only=False), + query=parse_one("SELECT val, ds FROM db.up", read="postgres"), + ) + down_new = SqlModel( + name="db.down", + dialect="postgres", + kind=IncrementalByTimeRangeKind(time_column="ds", forward_only=False), + query=parse_one("SELECT val, ds FROM db.up", read="postgres"), + ) + _, _, diff = _build_snapshots_and_diff(make_snapshot, [up_old, down_old], [up_new, down_new]) + + with pytest.raises( + PlanError, match=r"Plan requires a destructive change to a forward-only model\." + ): + PlanBuilder(diff, forward_only=True).build() + + _, _, diff_allowed = _build_snapshots_and_diff( + make_snapshot, [up_old.copy(), down_old.copy()], [up_new.copy(), down_new.copy()] + ) + assert PlanBuilder( + diff_allowed, + forward_only=True, + allow_destructive_models={'"db"."down"'}, + ).build() + + +def test_partially_unknown_columns_to_types_schema_checks(make_snapshot, mocker: MockerFixture): + """Gap S5: Partially UNKNOWN-typed columns_to_types still detects dropped columns, added + columns, and incompatible known-type alterations without false-positiving on shared UNKNOWNs.""" + + def _make_direct_diff( + old_sql: str, + new_sql: str, + on_destructive: OnDestructiveChange = OnDestructiveChange.ERROR, + on_additive: OnAdditiveChange = OnAdditiveChange.ALLOW, + ) -> ContextDiff: + kind = IncrementalByTimeRangeKind( + time_column="ds", + forward_only=True, + on_destructive_change=on_destructive, + on_additive_change=on_additive, + ) + m_old = SqlModel( + name="db.m", + dialect="duckdb", + kind=kind, + query=parse_one(old_sql, read="duckdb"), + ) + m_new = SqlModel( + name="db.m", + dialect="duckdb", + kind=kind, + query=parse_one(new_sql, read="duckdb"), + ) + _, _, diff = _build_snapshots_and_diff(make_snapshot, [m_old], [m_new]) + return diff + + base_sql = ( + "SELECT 'a'::TEXT AS typed_col, 'drop_me'::TEXT AS dropped_col, " + "some_udf(x) AS unk_col, '2022-01-01'::DATE AS ds FROM raw_tbl" + ) + + # 1. Dropping a typed column alongside an UNKNOWN column raises PlanError + diff_drop_typed = _make_direct_diff( + base_sql, + "SELECT 'a'::TEXT AS typed_col, some_udf(x) AS unk_col, '2022-01-01'::DATE AS ds FROM raw_tbl", + ) + with pytest.raises( + PlanError, match=r"Plan requires a destructive change to a forward-only model\." + ): + PlanBuilder(diff_drop_typed).build() + + # 2. Dropping the UNKNOWN column itself raises PlanError + diff_drop_unk = _make_direct_diff( + base_sql, + "SELECT 'a'::TEXT AS typed_col, 'drop_me'::TEXT AS dropped_col, '2022-01-01'::DATE AS ds FROM raw_tbl", + ) + with pytest.raises( + PlanError, match=r"Plan requires a destructive change to a forward-only model\." + ): + PlanBuilder(diff_drop_unk).build() + + # 3. Adding a known-typed OR UNKNOWN-typed column with on_additive_change=ERROR raises PlanError + for added_expr in ("1::INT AS added_col", "other_udf(x) AS added_col"): + diff_add = _make_direct_diff( + base_sql, + f"SELECT 'a'::TEXT AS typed_col, 'drop_me'::TEXT AS dropped_col, " + f"some_udf(x) AS unk_col, {added_expr}, '2022-01-01'::DATE AS ds FROM raw_tbl", + on_additive=OnAdditiveChange.ERROR, + ) + with pytest.raises( + PlanError, match=r"Plan requires an additive change to a forward-only model\." + ): + PlanBuilder(diff_add).build() + + # 4. Incompatibly changing typed_col ('a'::TEXT -> 1::INT) while unk_col stays UNKNOWN raises PlanError + diff_retype = _make_direct_diff( + base_sql, + "SELECT 1::INT AS typed_col, 'drop_me'::TEXT AS dropped_col, " + "some_udf(x) AS unk_col, '2022-01-01'::DATE AS ds FROM raw_tbl", + ) + with pytest.raises( + PlanError, match=r"Plan requires a destructive change to a forward-only model\." + ): + PlanBuilder(diff_retype).build() + + # 5. Shared column keeping UNKNOWN (or INT -> UNKNOWN) with no drops/incompatible changes does NOT false-positive + console = TerminalConsole() + log_destructive_spy = mocker.spy(console, "log_destructive_change") + diff_shared_unk = _make_direct_diff( + "SELECT 1::INT AS col_a, some_udf(x) AS unk_col, '2022-01-01'::DATE AS ds FROM raw_tbl", + "SELECT col_a, other_udf(x) AS unk_col, '2022-01-01'::DATE AS ds FROM raw_tbl", + ) + assert PlanBuilder(diff_shared_unk, console=console).build() + assert log_destructive_spy.call_count == 0 + + +def test_forward_only_change_to_unannotated_star_does_not_false_positive( + make_snapshot, mocker: MockerFixture +): + """Bug C2: Changing a forward-only model from known columns_to_types to unexpanded SELECT * + (columns_to_types is None) must NOT coerce None -> {} and falsely claim all columns dropped.""" + kind = IncrementalByTimeRangeKind( + time_column="ds", + forward_only=True, + on_destructive_change=OnDestructiveChange.ERROR, + ) + m_old = SqlModel( + name="db.m", + dialect="duckdb", + kind=kind, + query=parse_one("SELECT 1::INT AS a, '2022-01-01'::DATE AS ds", read="duckdb"), + ) + m_new = SqlModel( + name="db.m", + dialect="duckdb", + kind=kind, + query=parse_one("SELECT * FROM unregistered_raw_tbl", read="duckdb"), + ) + assert m_old.columns_to_types is not None + assert m_new.columns_to_types is None + + _, _, diff = _build_snapshots_and_diff(make_snapshot, [m_old], [m_new]) + console = TerminalConsole() + log_destructive_spy = mocker.spy(console, "log_destructive_change") + assert PlanBuilder(diff, console=console).build() + assert log_destructive_spy.call_count == 0 + + +def test_interactive_categorization_indirect_non_breaking_schema_checks(make_snapshot): + """Gaps S7 & S8 + --no-auto-categorization: + 1. First build() while upstream is uncategorized must NOT prematurely raise PlanError. + 2. set_choice(up, BREAKING) -> downstream INDIRECT_BREAKING (full rebuild) -> succeeds. + 3. set_choice(up, NON_BREAKING) -> downstream INDIRECT_NON_BREAKING (is_no_rebuild) -> raises PlanError. + """ + + def _make_s7_diff() -> t.Tuple[Snapshot, Snapshot, ContextDiff]: + up_old = SqlModel( + name="db.up", + dialect="postgres", + kind=FullKind(), + query=parse_one("SELECT '1'::TEXT AS val, '2022-01-01'::DATE AS ds", read="postgres"), + ) + up_new = SqlModel( + name="db.up", + dialect="postgres", + kind=FullKind(), + query=parse_one("SELECT 1::INT AS val, '2022-01-01'::DATE AS ds", read="postgres"), + ) + down_old = SqlModel( + name="db.down", + dialect="postgres", + kind=IncrementalByTimeRangeKind( + time_column="ds", + forward_only=False, + on_destructive_change=OnDestructiveChange.ERROR, + ), + query=parse_one("SELECT val, ds FROM db.up", read="postgres"), + ) + down_new = SqlModel( + name="db.down", + dialect="postgres", + kind=IncrementalByTimeRangeKind( + time_column="ds", + forward_only=False, + on_destructive_change=OnDestructiveChange.ERROR, + ), + query=parse_one("SELECT val, ds FROM db.up", read="postgres"), + ) + _, new_snaps, diff = _build_snapshots_and_diff( + make_snapshot, [up_old, down_old], [up_new, down_new] + ) + return new_snaps['"db"."up"'], new_snaps['"db"."down"'], diff + + # S7 (destructive): uncategorized -> BREAKING succeeds -> NON_BREAKING raises PlanError + up_snap_b, down_snap_b, diff_s7_breaking = _make_s7_diff() + builder_s7_breaking = PlanBuilder(diff_s7_breaking, auto_categorization_enabled=False) + assert builder_s7_breaking.build() + assert not up_snap_b.categorized + assert builder_s7_breaking.set_choice(up_snap_b, SnapshotChangeCategory.BREAKING).build() + assert down_snap_b.change_category == SnapshotChangeCategory.INDIRECT_BREAKING + assert not down_snap_b.is_no_rebuild + + up_snap_nb, down_snap_nb, diff_s7_nb = _make_s7_diff() + builder_s7_nb = PlanBuilder(diff_s7_nb, auto_categorization_enabled=False) + assert builder_s7_nb.build() + with pytest.raises( + PlanError, match=r"Plan requires a destructive change to a forward-only model\." + ): + builder_s7_nb.set_choice(up_snap_nb, SnapshotChangeCategory.NON_BREAKING).build() + assert down_snap_nb.is_indirect_non_breaking + assert down_snap_nb.is_no_rebuild + + # S8 (additive): uncategorized -> BREAKING succeeds -> NON_BREAKING raises PlanError + def _make_s8_diff() -> t.Tuple[Snapshot, Snapshot, ContextDiff]: + up_old = SqlModel( + name="db.up", + dialect="postgres", + kind=FullKind(), + query=parse_one( + "SELECT 'x'::VARCHAR(10) AS val, '2022-01-01'::DATE AS ds", read="postgres" + ), + ) + up_new = SqlModel( + name="db.up", + dialect="postgres", + kind=FullKind(), + query=parse_one( + "SELECT 'x'::VARCHAR(50) AS val, '2022-01-01'::DATE AS ds", read="postgres" + ), + ) + down_old = SqlModel( + name="db.down", + dialect="postgres", + kind=IncrementalByTimeRangeKind( + time_column="ds", + forward_only=False, + on_additive_change=OnAdditiveChange.ERROR, + ), + query=parse_one("SELECT val, ds FROM db.up", read="postgres"), + ) + down_new = SqlModel( + name="db.down", + dialect="postgres", + kind=IncrementalByTimeRangeKind( + time_column="ds", + forward_only=False, + on_additive_change=OnAdditiveChange.ERROR, + ), + query=parse_one("SELECT val, ds FROM db.up", read="postgres"), + ) + _, new_snaps, diff = _build_snapshots_and_diff( + make_snapshot, [up_old, down_old], [up_new, down_new] + ) + return new_snaps['"db"."up"'], new_snaps['"db"."down"'], diff + + up_s8_b, down_s8_b, diff_s8_b = _make_s8_diff() + builder_s8_b = PlanBuilder(diff_s8_b, auto_categorization_enabled=False) + assert builder_s8_b.build() + assert builder_s8_b.set_choice(up_s8_b, SnapshotChangeCategory.BREAKING).build() + assert down_s8_b.change_category == SnapshotChangeCategory.INDIRECT_BREAKING + + up_s8_nb, down_s8_nb, diff_s8_nb = _make_s8_diff() + builder_s8_nb = PlanBuilder(diff_s8_nb, auto_categorization_enabled=False) + assert builder_s8_nb.build() + with pytest.raises( + PlanError, match=r"Plan requires an additive change to a forward-only model\." + ): + builder_s8_nb.set_choice(up_s8_nb, SnapshotChangeCategory.NON_BREAKING).build() + assert down_s8_nb.is_indirect_non_breaking + assert down_s8_nb.is_no_rebuild + + +def test_precategorized_forward_only_snapshot_checked_on_prod_plan(make_snapshot): + """Gap S9: A snapshot pre-categorized in dev via categorize_as(BREAKING, forward_only=True) + and already persisted in state (new_snapshots={}) must still be checked on a prod plan.""" + m_old = SqlModel( + name="db.m", + dialect="postgres", + kind=IncrementalByTimeRangeKind(time_column="ds", forward_only=False), + query=parse_one("SELECT '1'::TEXT AS val, '2022-01-01'::DATE AS ds", read="postgres"), + ) + m_new = SqlModel( + name="db.m", + dialect="postgres", + kind=IncrementalByTimeRangeKind(time_column="ds", forward_only=False), + query=parse_one("SELECT 1::INT AS val, '2022-01-01'::DATE AS ds", read="postgres"), + ) + _, new_snaps, prod_diff = _build_snapshots_and_diff( + make_snapshot, + [m_old], + [m_new], + environment="prod", + is_new_environment=False, + new_snapshot_names=set(), + ) + m_snap = new_snaps['"db"."m"'] + m_snap.categorize_as(SnapshotChangeCategory.BREAKING, forward_only=True) + assert m_snap.is_forward_only + assert not prod_diff.new_snapshots + + with pytest.raises( + PlanError, match=r"Plan requires a destructive change to a forward-only model\." + ): + PlanBuilder(prod_diff, forward_only=False).build() + + assert PlanBuilder( + prod_diff, + forward_only=False, + allow_destructive_models={'"db"."m"'}, + ).build() + + +def test_breaking_kind_change_in_forward_only_plan_skips_destructive_check(make_snapshot): + """Bug B1: Breaking kind change (FullKind -> IncrementalByTimeRangeKind) while dropping a + column in PlanBuilder(..., forward_only=True) forces forward_only=False (full rebuild) and + must succeed without raising PlanError.""" + m_old = SqlModel( + name="db.m", + dialect="duckdb", + kind=FullKind(), + query=parse_one( + "SELECT 1::INT AS id, 'drop_me'::TEXT AS b, '2022-01-01'::DATE AS ds", + read="duckdb", + ), + ) + m_new = SqlModel( + name="db.m", + dialect="duckdb", + kind=IncrementalByTimeRangeKind(time_column="ds"), + query=parse_one("SELECT 1::INT AS id, '2022-01-01'::DATE AS ds", read="duckdb"), + ) + _, new_snaps, diff = _build_snapshots_and_diff(make_snapshot, [m_old], [m_new]) + + plan = PlanBuilder(diff, forward_only=True).build() + assert plan is not None + assert not new_snaps['"db"."m"'].is_forward_only + assert new_snaps['"db"."m"'].change_category == SnapshotChangeCategory.BREAKING + + +def test_schema_warnings_logged_at_most_once_per_plan_builder(make_snapshot, mocker: MockerFixture): + """Idempotent schema warning logging: repeated build() calls after set_start / set_end on the + same PlanBuilder instance must log each warning at most once per (snapshot_id, change_kind).""" + kind = IncrementalByTimeRangeKind( + time_column="ds", + forward_only=True, + on_destructive_change=OnDestructiveChange.WARN, + on_additive_change=OnAdditiveChange.WARN, + ) + m_old = SqlModel( + name="db.m", + dialect="duckdb", + kind=kind, + query=parse_one( + "SELECT 1::INT AS id, 'drop_me'::TEXT AS old_col, '2022-01-01'::DATE AS ds", + read="duckdb", + ), + ) + m_new = SqlModel( + name="db.m", + dialect="duckdb", + kind=kind, + query=parse_one( + "SELECT 1::INT AS id, 'added'::TEXT AS new_col, '2022-01-01'::DATE AS ds", + read="duckdb", + ), + ) + _, _, diff = _build_snapshots_and_diff(make_snapshot, [m_old], [m_new]) + + console = TerminalConsole() + log_destructive_spy = mocker.spy(console, "log_destructive_change") + log_additive_spy = mocker.spy(console, "log_additive_change") + + plan_builder = PlanBuilder(diff, is_dev=True, console=console) + plan_builder.build() + plan_builder.set_start("2022-01-02") + plan_builder.build() + plan_builder.set_end("2022-01-03") + plan_builder.build() + + assert log_destructive_spy.call_count == 1 + assert log_additive_spy.call_count == 1 + + +def test_plan_builder_schema_differ_overrides(make_snapshot): + """Gap S10 (unit): PlanBuilder(..., schema_differ_overrides=...) applies both default (None) + and per-gateway overrides matching snapshot.model_gateway.""" + + def _make_widening_diff(gateway: t.Optional[str] = None) -> ContextDiff: + m_old = SqlModel( + name="db.m", + dialect="postgres", + gateway=gateway, + kind=IncrementalByTimeRangeKind(time_column="ds", forward_only=True), + query=parse_one( + "SELECT 'x'::VARCHAR(10) AS val, '2022-01-01'::DATE AS ds", read="postgres" + ), + ) + m_new = SqlModel( + name="db.m", + dialect="postgres", + gateway=gateway, + kind=IncrementalByTimeRangeKind(time_column="ds", forward_only=True), + query=parse_one( + "SELECT 'x'::VARCHAR(50) AS val, '2022-01-01'::DATE AS ds", read="postgres" + ), + ) + _, _, diff = _build_snapshots_and_diff(make_snapshot, [m_old], [m_new]) + return diff + + # Without overrides, VARCHAR(10) -> VARCHAR(50) in postgres is compatible (additive, allowed) + assert PlanBuilder(_make_widening_diff()).build() + + # With default (None) override treat_alter_data_type_as_destructive=True -> raises PlanError + with pytest.raises( + PlanError, match=r"Plan requires a destructive change to a forward-only model\." + ): + PlanBuilder( + _make_widening_diff(), + schema_differ_overrides={None: {"treat_alter_data_type_as_destructive": True}}, + ).build() + + # Per-gateway override: gw_a unaffected, gw_b raises PlanError + gateway_overrides: t.Dict[t.Optional[str], t.Dict[str, t.Any]] = { + "gw_b": {"treat_alter_data_type_as_destructive": True} + } + assert PlanBuilder( + _make_widening_diff(gateway="gw_a"), + schema_differ_overrides=gateway_overrides, + ).build() + + with pytest.raises( + PlanError, match=r"Plan requires a destructive change to a forward-only model\." + ): + PlanBuilder( + _make_widening_diff(gateway="gw_b"), + schema_differ_overrides=gateway_overrides, + ).build() + + # Default gateway (None) overrides must NOT bleed onto an explicitly named secondary gateway + assert PlanBuilder( + _make_widening_diff(gateway="gw_other"), + schema_differ_overrides={ + None: {"treat_alter_data_type_as_destructive": True}, + "gw_a": {"treat_alter_data_type_as_destructive": True}, + "gw_other": {}, + }, + ).build() + + +def test_context_plan_builder_plumbs_schema_differ_overrides_without_connecting( + tmp_path: Path, mocker: MockerFixture +): + """Gap S10 (Context plumbing): Context.plan_builder reads ConnectionConfig.schema_differ_overrides + for both the default gateway and named multi-gateway configs without instantiating uninitialized + secondary EngineAdapters.""" + # 1. Default gateway DuckDBConnectionConfig with schema_differ_overrides catches DECIMAL widening + proj_default = tmp_path / "proj_default" + (proj_default / "models").mkdir(parents=True) + (proj_default / "models" / "m.sql").write_text( + """ + MODEL ( + name db.m, + kind INCREMENTAL_BY_TIME_RANGE (time_column ds, forward_only true), + start '2023-01-01' + ); + SELECT 1::DECIMAL(10, 2) AS val, '2023-01-01'::DATE AS ds; + """ + ) + cfg_default = Config( + gateways={ + "default": GatewayConfig( + connection=DuckDBConnectionConfig( + schema_differ_overrides={"treat_alter_data_type_as_destructive": True} + ) + ) + }, + default_gateway="default", + model_defaults=ModelDefaultsConfig(dialect="duckdb"), + ) + ctx_default = Context(paths=[proj_default], config=cfg_default) + ctx_default.plan("prod", execution_time="2023-01-02", auto_apply=True, no_prompts=True) + + (proj_default / "models" / "m.sql").write_text( + """ + MODEL ( + name db.m, + kind INCREMENTAL_BY_TIME_RANGE (time_column ds, forward_only true), + start '2023-01-01' + ); + SELECT 1::DECIMAL(12, 2) AS val, '2023-01-01'::DATE AS ds; + """ + ) + ctx_default.load() + with pytest.raises( + PlanError, match=r"Plan requires a destructive change to a forward-only model\." + ): + ctx_default.plan_builder("dev", execution_time="2023-01-02").build() + + # 2. Multi-gateway config: secondary Postgres gateway with unreachable host is NOT connected + # when Context.plan_builder collects per-gateway schema_differ_overrides + proj_multi = tmp_path / "proj_multi" + (proj_multi / "models").mkdir(parents=True) + (proj_multi / "models" / "a.sql").write_text( + """ + MODEL (name db.a, kind FULL); + SELECT 1::INT AS id; + """ + ) + pg_conn = PostgresConnectionConfig( + host="unreachable.invalid", + port=5432, + user="u", + password="p", + database="d", + schema_differ_overrides={"treat_alter_data_type_as_destructive": True}, + ) + cfg_multi = Config( + gateways={ + "gw_a": GatewayConfig(connection=DuckDBConnectionConfig()), + "gw_b": GatewayConfig(connection=pg_conn), + }, + default_gateway="gw_a", + model_defaults=ModelDefaultsConfig(dialect="duckdb"), + ) + pg_adapter_spy = mocker.spy(PostgresConnectionConfig, "create_engine_adapter") + plan_builder_init_spy = mocker.spy(PlanBuilder, "__init__") + ctx_multi = Context(paths=[proj_multi], config=cfg_multi) + pg_adapter_spy.reset_mock() + + pb = ctx_multi.plan_builder("dev", execution_time="2023-01-02") + assert pb.build() + assert pg_adapter_spy.call_count == 0 + passed_overrides = plan_builder_init_spy.call_args.kwargs.get("schema_differ_overrides") or {} + assert passed_overrides.get("gw_b") == {"treat_alter_data_type_as_destructive": True} + + +def test_plan_builder_partially_unknown_nested_struct_and_positional_add( + make_snapshot, mocker: MockerFixture +): + # 1. Partially-unknown STRUCT field drop in forward-only model must raise PlanError + kind_err = IncrementalByTimeRangeKind(time_column="ds", forward_only=True) + m_old_struct = SqlModel( + name="db.m", + dialect="databricks", + kind=kind_err, + query=parse_one( + "SELECT STRUCT(1 AS a, 'x' AS b, NULL AS u) AS info, CAST('2023-01-01' AS DATE) AS ds", + read="databricks", + ), + ) + m_new_struct = SqlModel( + name="db.m", + dialect="databricks", + kind=kind_err, + query=parse_one( + "SELECT STRUCT(1 AS a, NULL AS u) AS info, CAST('2023-01-01' AS DATE) AS ds", + read="databricks", + ), + ) + _, _, diff_struct = _build_snapshots_and_diff(make_snapshot, [m_old_struct], [m_new_struct]) + with pytest.raises( + PlanError, match=r"Plan requires a destructive change to a forward-only model\." + ): + PlanBuilder(diff_struct).build() + + # 2. Positional column addition in the middle preserves AFTER clause on Databricks + kind_warn = IncrementalByTimeRangeKind( + time_column="ds", + forward_only=True, + on_additive_change=OnAdditiveChange.WARN, + ) + m_old_pos = SqlModel( + name="db.m", + dialect="databricks", + kind=kind_warn, + query=parse_one("SELECT 1 AS a, CAST('2023-01-01' AS DATE) AS ds", read="databricks"), + ) + m_new_pos = SqlModel( + name="db.m", + dialect="databricks", + kind=kind_warn, + query=parse_one( + "SELECT 1 AS a, 2 AS b_mid, CAST('2023-01-01' AS DATE) AS ds", read="databricks" + ), + ) + _, _, diff_pos = _build_snapshots_and_diff(make_snapshot, [m_old_pos], [m_new_pos]) + console = TerminalConsole() + log_additive_spy = mocker.spy(console, "log_additive_change") + assert PlanBuilder(diff_pos, console=console).build() + assert log_additive_spy.call_count == 1 + alter_ops = log_additive_spy.call_args.args[1] + assert [op.expression.sql("databricks") for op in alter_ops] == [ + "ALTER TABLE `db`.`m` ADD COLUMN b_mid INT AFTER a" + ] + + +def test_context_plan_builder_gateway_default_connection_fallback_and_precedence( + tmp_path: Path, mocker: MockerFixture +): + from sqlmesh.core.config import ( + Config, + DuckDBConnectionConfig, + GatewayConfig, + ModelDefaultsConfig, + PostgresConnectionConfig, + ) + from sqlmesh.core.context import Context + + proj = tmp_path / "proj_gw_fallback" + (proj / "models").mkdir(parents=True) + (proj / "models" / "a.sql").write_text( + """ + MODEL (name db.a, kind FULL); + SELECT 1::INT AS id; + """ + ) + + pg_default_conn = PostgresConnectionConfig( + host="unreachable.invalid", + port=5432, + user="u", + password="p", + database="d", + schema_differ_overrides={"treat_alter_data_type_as_destructive": True}, + ) + cfg = Config( + default_connection=pg_default_conn, + gateways={ + "gw_duck": GatewayConfig(connection=DuckDBConnectionConfig()), + # connection=None should inherit default_connection's schema_differ_overrides + "gw_state_only": GatewayConfig( + connection=None, state_connection=DuckDBConnectionConfig() + ), + }, + default_gateway="gw_duck", + model_defaults=ModelDefaultsConfig(dialect="duckdb"), + ) + + pg_adapter_spy = mocker.spy(PostgresConnectionConfig, "create_engine_adapter") + plan_builder_init_spy = mocker.spy(PlanBuilder, "__init__") + ctx = Context(paths=[proj], config=cfg) + pg_adapter_spy.reset_mock() + + pb = ctx.plan_builder("dev", execution_time="2023-01-02") + assert pb.build() + assert pg_adapter_spy.call_count == 0 + passed_overrides = plan_builder_init_spy.call_args.kwargs.get("schema_differ_overrides") or {} + assert passed_overrides.get("gw_duck") == {} + assert passed_overrides.get(None) == {} + assert passed_overrides.get("gw_state_only") == {"treat_alter_data_type_as_destructive": True} + + +def test_treat_alter_data_type_as_destructive_with_ignore_policies( + make_snapshot, mocker: MockerFixture +): + overrides: t.Dict[t.Optional[str], t.Dict[str, t.Any]] = { + None: {"treat_alter_data_type_as_destructive": True} + } + + # 1. on_destructive_change=IGNORE + on_additive_change=ERROR must NOT log or raise + kind_ignore_dest = IncrementalByTimeRangeKind( + time_column="ds", + forward_only=True, + on_destructive_change=OnDestructiveChange.IGNORE, + on_additive_change=OnAdditiveChange.ERROR, + ) + m_old_1 = SqlModel( + name="db.m", + dialect="postgres", + kind=kind_ignore_dest, + query=parse_one( + "SELECT 'x'::VARCHAR(10) AS val, '2022-01-01'::DATE AS ds", read="postgres" + ), + ) + m_new_1 = SqlModel( + name="db.m", + dialect="postgres", + kind=kind_ignore_dest, + query=parse_one( + "SELECT 'x'::VARCHAR(50) AS val, '2022-01-01'::DATE AS ds", read="postgres" + ), + ) + _, _, diff_ignore_dest = _build_snapshots_and_diff(make_snapshot, [m_old_1], [m_new_1]) + console = TerminalConsole() + log_dest_spy = mocker.spy(console, "log_destructive_change") + log_add_spy = mocker.spy(console, "log_additive_change") + assert PlanBuilder(diff_ignore_dest, console=console, schema_differ_overrides=overrides).build() + assert log_dest_spy.call_count == 0 + assert log_add_spy.call_count == 0 + + # 2. on_destructive_change=ERROR + on_additive_change=IGNORE MUST still raise PlanError + kind_ignore_add = IncrementalByTimeRangeKind( + time_column="ds", + forward_only=True, + on_destructive_change=OnDestructiveChange.ERROR, + on_additive_change=OnAdditiveChange.IGNORE, + ) + m_old_2 = SqlModel( + name="db.m", + dialect="postgres", + kind=kind_ignore_add, + query=parse_one( + "SELECT 'x'::VARCHAR(10) AS val, '2022-01-01'::DATE AS ds", read="postgres" + ), + ) + m_new_2 = SqlModel( + name="db.m", + dialect="postgres", + kind=kind_ignore_add, + query=parse_one( + "SELECT 'x'::VARCHAR(50) AS val, '2022-01-01'::DATE AS ds", read="postgres" + ), + ) + _, _, diff_ignore_add = _build_snapshots_and_diff(make_snapshot, [m_old_2], [m_new_2]) + with pytest.raises( + PlanError, match=r"Plan requires a destructive change to a forward-only model\." + ): + PlanBuilder(diff_ignore_add, schema_differ_overrides=overrides).build() + + +def test_scd_type_2_by_time_updated_at_type_follows_time_data_type(make_snapshot): + """The physical `updated_at` column is always created with `time_data_type`, so changing the + projected type of that column is not a physical schema change.""" + + def _scd_model(updated_at_type: str, time_data_type: str = "TIMESTAMP") -> SqlModel: + return SqlModel( + name="db.scd", + dialect="duckdb", + kind=SCDType2ByTimeKind( + unique_key=["id"], updated_at_name="updated_at", time_data_type=time_data_type + ), + query=parse_one( + f"SELECT 1::INT AS id, 'a'::TEXT AS val, '2022-01-01'::{updated_at_type} AS updated_at", + read="duckdb", + ), + ) + + _, _, diff = _build_snapshots_and_diff( + make_snapshot, [_scd_model("DATE")], [_scd_model("TIMESTAMP")] + ) + assert PlanBuilder(diff).build() + + # A change of `time_data_type` itself still changes the physical columns and is rejected. + _, _, diff = _build_snapshots_and_diff( + make_snapshot, + [_scd_model("TIMESTAMP", "TIMESTAMP")], + [_scd_model("TIMESTAMP", "DATE")], + ) + with pytest.raises( + PlanError, match=r"Plan requires a destructive change to a forward-only model\." + ): + PlanBuilder(diff).build() + + +@pytest.mark.parametrize( + "old_policies, expect_error", + [ + # The previous definition ignored additions, so its physical table may not contain the + # columns that its definition lists. The drop is deferred to physical validation. + ({"on_additive_change": OnAdditiveChange.IGNORE}, False), + # The previous schema establishes the drop, keep the static error. + ({}, True), + ], +) +def test_forward_only_static_check_defers_when_previous_policy_ignored_changes( + make_snapshot, old_policies, expect_error +): + def _model(query: str, **policies: t.Any) -> SqlModel: + return SqlModel( + name="db.down", + dialect="duckdb", + kind=IncrementalByTimeRangeKind( + time_column="ds", + forward_only=True, + **{ + "on_destructive_change": OnDestructiveChange.ERROR, + "on_additive_change": OnAdditiveChange.ERROR, + **policies, + }, + ), + query=parse_one(query, read="duckdb"), + ) + + old = _model("SELECT 1::INT AS id, 'x'::TEXT AS b, '2022-01-01'::DATE AS ds", **old_policies) + new = _model("SELECT 1::INT AS id, '2022-01-01'::DATE AS ds") + _, _, diff = _build_snapshots_and_diff(make_snapshot, [old], [new]) + + if expect_error: + with pytest.raises( + PlanError, match=r"Plan requires a destructive change to a forward-only model\." + ): + PlanBuilder(diff).build() + else: + assert PlanBuilder(diff).build() diff --git a/tests/core/test_schema_diff.py b/tests/core/test_schema_diff.py index 52bd6bb606..6d9881c364 100644 --- a/tests/core/test_schema_diff.py +++ b/tests/core/test_schema_diff.py @@ -10,6 +10,9 @@ TableAlterColumnPosition, TableAlterOperation, get_schema_differ, + has_additive_alteration, + has_drop_alteration, + prepare_columns_for_static_diff, TableAlterAddColumnOperation, TableAlterDropColumnOperation, TableAlterChangeColumnTypeOperation, @@ -2366,3 +2369,180 @@ def test_drop_operation_missing_column_error(): str(error_message.value) == "Cannot drop column 'missing_column' from table 'test_table' - column not found. This may indicate a mismatch between the expected and actual table schemas." ) + + +def test_prepare_columns_for_static_diff_nested_partial_unknown_types(): + differ = get_schema_differ("databricks") + + # 1. Partially-unknown STRUCT: dropping nested field 'b' (or unknown field 'u') is detected + old_struct = exp.DataType.build("STRUCT", dialect="databricks") + new_struct_drop_b = exp.DataType.build("STRUCT", dialect="databricks") + comp = prepare_columns_for_static_diff({"info": old_struct}, {"info": new_struct_drop_b}) + assert comp is not None + ops_drop_b = differ.compare_columns("tbl", comp[0], comp[1]) + assert has_drop_alteration(ops_drop_b) + assert [op.expression.sql("databricks") for op in ops_drop_b] == [ + "ALTER TABLE tbl DROP COLUMN info.b" + ] + + new_struct_drop_u = exp.DataType.build("STRUCT", dialect="databricks") + comp_drop_u = prepare_columns_for_static_diff({"info": old_struct}, {"info": new_struct_drop_u}) + assert comp_drop_u is not None + ops_drop_u = differ.compare_columns("tbl", comp_drop_u[0], comp_drop_u[1]) + assert has_drop_alteration(ops_drop_u) + assert [op.expression.sql("databricks") for op in ops_drop_u] == [ + "ALTER TABLE tbl DROP COLUMN info.u" + ] + + # 2. Partially-unknown STRUCT: adding nested field 'c' in the middle preserves order & quoting + old_quoted = exp.DataType.build("STRUCT<`MyA` INT, u NULL>", dialect="databricks") + new_quoted_add = exp.DataType.build( + "STRUCT<`MyA` INT, `MyB` DOUBLE, u NULL>", dialect="databricks" + ) + comp_add = prepare_columns_for_static_diff({"info": old_quoted}, {"info": new_quoted_add}) + assert comp_add is not None + ops_add = differ.compare_columns("tbl", comp_add[0], comp_add[1]) + assert has_additive_alteration(ops_add) + assert not has_drop_alteration(ops_add) + assert [op.expression.sql("databricks") for op in ops_add] == [ + "ALTER TABLE tbl ADD COLUMN info.`MyB` DOUBLE AFTER `MyA`" + ] + + # 3. Partially-unknown STRUCT: incompatible type change on known sibling field 'a' is detected + old_retype = exp.DataType.build("STRUCT", dialect="databricks") + new_retype = exp.DataType.build("STRUCT", dialect="databricks") + comp_retype = prepare_columns_for_static_diff({"info": old_retype}, {"info": new_retype}) + assert comp_retype is not None + ops_retype = differ.compare_columns("tbl", comp_retype[0], comp_retype[1]) + assert has_drop_alteration(ops_retype) + + # 4. Partially-unknown STRUCT: resolving unknown field (NULL -> STRING) does NOT false-positive + new_resolved = exp.DataType.build("STRUCT", dialect="databricks") + comp_resolved = prepare_columns_for_static_diff({"info": old_retype}, {"info": new_resolved}) + assert comp_resolved is not None + assert differ.compare_columns("tbl", comp_resolved[0], comp_resolved[1]) == [] + + # 5. Partially-unknown ARRAY & ARRAY + old_arr = exp.DataType.build("ARRAY>", dialect="databricks") + new_arr_drop = exp.DataType.build("ARRAY>", dialect="databricks") + comp_arr_drop = prepare_columns_for_static_diff({"items": old_arr}, {"items": new_arr_drop}) + assert comp_arr_drop is not None + ops_arr_drop = differ.compare_columns("tbl", comp_arr_drop[0], comp_arr_drop[1]) + assert has_drop_alteration(ops_arr_drop) + assert [op.expression.sql("databricks") for op in ops_arr_drop] == [ + "ALTER TABLE tbl DROP COLUMN items.element.b" + ] + + comp_arr_unk_elem = prepare_columns_for_static_diff( + {"tags": exp.DataType.build("ARRAY")}, + {"tags": exp.DataType.build("ARRAY")}, + ) + assert comp_arr_unk_elem is not None + assert differ.compare_columns("tbl", comp_arr_unk_elem[0], comp_arr_unk_elem[1]) == [] + + # 6. Container-vs-primitive mismatch with partially-unknown STRUCT / ARRAY + comp_prim_to_struct = prepare_columns_for_static_diff( + {"val": exp.DataType.build("STRING", dialect="databricks")}, + {"val": old_struct}, + ) + assert comp_prim_to_struct is not None + ops_prim_to_struct = differ.compare_columns( + "tbl", comp_prim_to_struct[0], comp_prim_to_struct[1] + ) + assert has_drop_alteration(ops_prim_to_struct) + + comp_prim_to_arr = prepare_columns_for_static_diff( + {"val": exp.DataType.build("STRING", dialect="databricks")}, + {"val": exp.DataType.build("ARRAY")}, + ) + assert comp_prim_to_arr is not None + assert has_drop_alteration( + differ.compare_columns("tbl", comp_prim_to_arr[0], comp_prim_to_arr[1]) + ) + + +def test_prepare_columns_for_static_diff_preserves_positional_column_order(): + old_cols = { + "a": exp.DataType.build("INT", dialect="databricks"), + "c": exp.DataType.build("STRING", dialect="databricks"), + } + new_cols = { + "b_first": exp.DataType.build("DOUBLE", dialect="databricks"), + "a": exp.DataType.build("INT", dialect="databricks"), + "b_mid": exp.DataType.build("BIGINT", dialect="databricks"), + "c": exp.DataType.build("STRING", dialect="databricks"), + } + comp = prepare_columns_for_static_diff(old_cols, new_cols) + assert comp is not None + comparable_old, comparable_new = comp + assert list(comparable_old.keys()) == ["a", "c"] + assert list(comparable_new.keys()) == ["b_first", "a", "b_mid", "c"] + + ops = get_schema_differ("databricks").compare_columns("tbl", comparable_old, comparable_new) + assert [op.expression.sql("databricks") for op in ops] == [ + "ALTER TABLE tbl ADD COLUMN b_first DOUBLE FIRST", + "ALTER TABLE tbl ADD COLUMN b_mid BIGINT AFTER a", + ] + + +def test_treat_alter_data_type_as_destructive_respects_ignore_flags(): + differ = get_schema_differ("postgres", overrides={"treat_alter_data_type_as_destructive": True}) + old_cols = {"val": exp.DataType.build("VARCHAR(10)", dialect="postgres")} + new_cols = {"val": exp.DataType.build("VARCHAR(50)", dialect="postgres")} + + # When treat_alter_data_type_as_destructive=True, compatible type widening is destructive: + # ignore_destructive=True must suppress it, while ignore_additive=True must NOT suppress it. + assert ( + differ.compare_columns( + "tbl", old_cols, new_cols, ignore_destructive=True, ignore_additive=False + ) + == [] + ) + ops_keep_destructive = differ.compare_columns( + "tbl", old_cols, new_cols, ignore_destructive=False, ignore_additive=True + ) + assert len(ops_keep_destructive) == 1 + assert isinstance(ops_keep_destructive[0], TableAlterChangeColumnTypeOperation) + assert ops_keep_destructive[0].is_destructive + assert not ops_keep_destructive[0].is_additive + + +def test_prepare_columns_for_static_diff_does_not_alias_input_types(): + differ = get_schema_differ("databricks") + old_cols = { + "details": exp.DataType.build("STRUCT", dialect="databricks"), + "u": exp.DataType.build("NULL"), + } + new_cols = { + "details": exp.DataType.build("STRUCT", dialect="databricks"), + "u": exp.DataType.build("STRING"), + } + old_sql = {k: v.sql("databricks") for k, v in old_cols.items()} + new_sql = {k: v.sql("databricks") for k, v in new_cols.items()} + + for _ in range(2): + comp = prepare_columns_for_static_diff(old_cols, new_cols) + assert comp is not None + ops = differ.compare_columns("tbl", comp[0], comp[1]) + assert [op.expression.sql("databricks") for op in ops] == [ + "ALTER TABLE tbl DROP COLUMN details.b" + ] + # Neither the diff nor the normalization may mutate the caller's type trees. + assert {k: v.sql("databricks") for k, v in old_cols.items()} == old_sql + assert {k: v.sql("databricks") for k, v in new_cols.items()} == new_sql + + +def test_prepare_columns_for_static_diff_positional_struct_fields(): + # Positional (unnamed) struct fields, e.g. produced by annotating STRUCT(1, NULL). + old_struct = exp.DataType( + this=exp.DataType.Type.STRUCT, + expressions=[exp.DataType.build("INT"), exp.DataType.build("NULL")], + nested=True, + ) + new_struct = exp.DataType( + this=exp.DataType.Type.STRUCT, + expressions=[exp.DataType.build("INT"), exp.DataType.build("VARCHAR")], + nested=True, + ) + # Positional shapes can't be compared by name, so the comparison is deferred (not a KeyError). + assert prepare_columns_for_static_diff({"s": old_struct}, {"s": new_struct}) is None diff --git a/tests/core/test_snapshot_evaluator.py b/tests/core/test_snapshot_evaluator.py index 0955af331d..cdcf186f3c 100644 --- a/tests/core/test_snapshot_evaluator.py +++ b/tests/core/test_snapshot_evaluator.py @@ -6113,3 +6113,379 @@ def observer(phase, facts, error): consumer.reset(token) action = "schema_migration" if operation == "migrate" else operation assert events == [("start", action, None), ("finish", action, None)] + + +def _migrate_with_extra_column_in_source( + mocker: MockerFixture, + make_snapshot, + make_mocked_engine_adapter, + *, + physical_version: t.Optional[str], +) -> Snapshot: + model = SqlModel( + name="test_schema.test_model", + kind=IncrementalByTimeRangeKind(time_column="a", on_additive_change=OnAdditiveChange.ERROR), + query=parse_one("SELECT c, a, b FROM tbl WHERE ds BETWEEN @start_ds and @end_ds"), + physical_version=physical_version, + ) + previous_snapshot = make_snapshot(model.copy(update={"stamp": "prev"}), version="0") + previous_snapshot.categorize_as(SnapshotChangeCategory.BREAKING) + snapshot = make_snapshot(model) + snapshot.previous_versions = previous_snapshot.all_versions + snapshot.categorize_as(SnapshotChangeCategory.BREAKING) + assert not snapshot.is_no_rebuild + assert snapshot.version == (physical_version or snapshot.fingerprint.to_version()) + + adapter = make_mocked_engine_adapter(EngineAdapter) + adapter.with_settings = lambda **kwargs: adapter # type: ignore + current_table = snapshot.table_name() + + def columns(table_name): + existing = {"c": exp.DataType.build("int"), "a": exp.DataType.build("int")} + if table_name == current_table: + return existing + return {**existing, "b": exp.DataType.build("int")} + + adapter.columns = columns # type: ignore + mocker.patch( + "sqlmesh.core.engine_adapter.base.EngineAdapter.get_data_objects", + return_value=[ + DataObject( + schema="sqlmesh__test_schema", + name=f"test_schema__test_model__{snapshot.version}", + type=DataObjectType.TABLE, + ) + ], + ) + + alter_spy = mocker.spy(adapter, "alter_table") + SnapshotEvaluator(adapter).migrate([snapshot], {}) + # Guard against passing vacuously: the extra column must have been added in place. + added = [ + op.expression.sql("duckdb") for call in alter_spy.call_args_list for op in call.args[0] + ] + assert any("ADD COLUMN b" in sql for sql in added), added + return snapshot + + +def test_migrate_skips_additive_check_when_snapshot_rebuilds( + mocker: MockerFixture, + make_snapshot, + make_mocked_engine_adapter, +): + """Bug D1: SnapshotEvaluator.migrate must NOT raise AdditiveChangeError when snapshot has + change_category=BREAKING and forward_only=False and gets a fresh physical table.""" + _migrate_with_extra_column_in_source( + mocker, make_snapshot, make_mocked_engine_adapter, physical_version=None + ) + + +def test_migrate_enforces_additive_check_for_pinned_physical_version( + mocker: MockerFixture, + make_snapshot, + make_mocked_engine_adapter, +): + """A BREAKING change of a model with a pinned physical_version keeps its table, so the + column is added in place and on_additive_change=ERROR must still reject it.""" + with pytest.raises(NodeExecutionFailedError) as exc_info: + _migrate_with_extra_column_in_source( + mocker, make_snapshot, make_mocked_engine_adapter, physical_version="v1" + ) + assert isinstance(exc_info.value.__cause__, AdditiveChangeError) + + +def test_dev_preview_non_cloning_create_checks_schema_and_drops_dev_table( + duck_conn, + make_snapshot, +): + """SnapshotEvaluator.create on non-cloning engines (SUPPORTS_CLONING=False) must validate the + newly created dev preview table against the existing prod table when snapshot.is_no_rebuild and + not deployable, raising DestructiveChangeError / AdditiveChangeError and cleaning up __dev.""" + adapter = create_engine_adapter(lambda: duck_conn, "duckdb") + assert adapter.SUPPORTS_CLONING is False + evaluator = SnapshotEvaluator(adapter) + + duck_conn.execute("CREATE TABLE raw_tbl AS SELECT 1 AS id, 'x' AS b, DATE '2023-01-01' AS ds") + + base_model = SqlModel( + name="test_schema.test_model", + dialect="duckdb", + kind=IncrementalByUniqueKeyKind( + unique_key="id", + forward_only=True, + on_destructive_change=OnDestructiveChange.ERROR, + on_additive_change=OnAdditiveChange.ERROR, + ), + query=parse_one("SELECT * FROM raw_tbl WHERE ds BETWEEN @start_ds AND @end_ds"), + ) + prod_snapshot = make_snapshot(base_model, version="1") + prod_snapshot.categorize_as(SnapshotChangeCategory.BREAKING) + + evaluator.create_physical_schemas([prod_snapshot], DeployabilityIndex.all_deployable()) + evaluator.create( + [prod_snapshot], + {prod_snapshot.snapshot_id: prod_snapshot}, + deployability_index=DeployabilityIndex.all_deployable(), + ) + prod_table = prod_snapshot.table_name(is_deployable=True) + assert set(adapter.columns(prod_table)) == {"id", "b", "ds"} + + dev_snapshot = make_snapshot(base_model.copy(update={"stamp": "dev_v2"})) + dev_snapshot.previous_versions = prod_snapshot.all_versions + dev_snapshot.categorize_as(SnapshotChangeCategory.INDIRECT_BREAKING, forward_only=True) + assert dev_snapshot.is_no_rebuild + dev_table = dev_snapshot.table_name(is_deployable=False) + + # 1. Destructive drop in raw_tbl -> raises SnapshotCreationFailedError(DestructiveChangeError) and drops __dev + duck_conn.execute("ALTER TABLE raw_tbl DROP COLUMN b") + with pytest.raises(SnapshotCreationFailedError) as exc_info_drop: + evaluator.create( + [dev_snapshot], + {dev_snapshot.snapshot_id: dev_snapshot}, + deployability_index=DeployabilityIndex.none_deployable(), + ) + assert isinstance(exc_info_drop.value.errors[0].__cause__, DestructiveChangeError) + assert not adapter.table_exists(dev_table) + assert set(adapter.columns(prod_table)) == {"id", "b", "ds"} + + # Allowing destructive snapshot succeeds + evaluator.create( + [dev_snapshot], + {dev_snapshot.snapshot_id: dev_snapshot}, + deployability_index=DeployabilityIndex.none_deployable(), + allow_destructive_snapshots={dev_snapshot.name}, + ) + assert adapter.table_exists(dev_table) + adapter.drop_table(dev_table) + + # 2. Additive column in raw_tbl with on_additive_change=ERROR -> raises SnapshotCreationFailedError(AdditiveChangeError) and drops __dev + duck_conn.execute( + "CREATE OR REPLACE TABLE raw_tbl AS SELECT 1 AS id, 'x' AS b, 2 AS c, DATE '2023-01-01' AS ds" + ) + with pytest.raises(SnapshotCreationFailedError) as exc_info_add: + evaluator.create( + [dev_snapshot], + {dev_snapshot.snapshot_id: dev_snapshot}, + deployability_index=DeployabilityIndex.none_deployable(), + ) + assert isinstance(exc_info_add.value.errors[0].__cause__, AdditiveChangeError) + assert not adapter.table_exists(dev_table) + assert set(adapter.columns(prod_table)) == {"id", "b", "ds"} + + +def test_dev_preview_non_cloning_evaluate_checks_schema_and_drops_dev_table( + duck_conn, + make_snapshot, +): + """SnapshotEvaluator.evaluate with target_table_exists=False on non-cloning engines must + validate the lazily created dev preview table against the existing prod table and drop __dev on + DestructiveChangeError / AdditiveChangeError.""" + adapter = create_engine_adapter(lambda: duck_conn, "duckdb") + assert adapter.SUPPORTS_CLONING is False + evaluator = SnapshotEvaluator(adapter) + + duck_conn.execute("CREATE TABLE raw_tbl AS SELECT 1 AS id, 'x' AS b, DATE '2023-01-01' AS ds") + + base_model = SqlModel( + name="test_schema.test_model", + dialect="duckdb", + kind=IncrementalByUniqueKeyKind( + unique_key="id", + forward_only=True, + on_destructive_change=OnDestructiveChange.ERROR, + on_additive_change=OnAdditiveChange.ERROR, + ), + query=parse_one("SELECT * FROM raw_tbl WHERE ds BETWEEN @start_ds AND @end_ds"), + ) + prod_snapshot = make_snapshot(base_model, version="1") + prod_snapshot.categorize_as(SnapshotChangeCategory.BREAKING) + + evaluator.create_physical_schemas([prod_snapshot], DeployabilityIndex.all_deployable()) + evaluator.evaluate( + prod_snapshot, + start="2023-01-01", + end="2023-01-02", + execution_time="2023-01-02", + snapshots={prod_snapshot.name: prod_snapshot}, + deployability_index=DeployabilityIndex.all_deployable(), + target_table_exists=False, + ) + prod_table = prod_snapshot.table_name(is_deployable=True) + assert set(adapter.columns(prod_table)) == {"id", "b", "ds"} + + dev_snapshot = make_snapshot(base_model.copy(update={"stamp": "dev_v2"})) + dev_snapshot.previous_versions = prod_snapshot.all_versions + dev_snapshot.categorize_as(SnapshotChangeCategory.BREAKING, forward_only=True) + assert dev_snapshot.is_no_rebuild + dev_table = dev_snapshot.table_name(is_deployable=False) + + # 1. Destructive drop -> raises DestructiveChangeError and drops __dev + duck_conn.execute("ALTER TABLE raw_tbl DROP COLUMN b") + with pytest.raises(DestructiveChangeError): + evaluator.evaluate( + dev_snapshot, + start="2023-01-01", + end="2023-01-02", + execution_time="2023-01-02", + snapshots={dev_snapshot.name: dev_snapshot}, + deployability_index=DeployabilityIndex.none_deployable(), + target_table_exists=False, + ) + assert not adapter.table_exists(dev_table) + assert set(adapter.columns(prod_table)) == {"id", "b", "ds"} + + # 2. Additive column -> raises AdditiveChangeError and drops __dev + duck_conn.execute( + "CREATE OR REPLACE TABLE raw_tbl AS SELECT 1 AS id, 'x' AS b, 2 AS c, DATE '2023-01-01' AS ds" + ) + with pytest.raises(AdditiveChangeError): + evaluator.evaluate( + dev_snapshot, + start="2023-01-01", + end="2023-01-02", + execution_time="2023-01-02", + snapshots={dev_snapshot.name: dev_snapshot}, + deployability_index=DeployabilityIndex.none_deployable(), + target_table_exists=False, + ) + assert not adapter.table_exists(dev_table) + assert set(adapter.columns(prod_table)) == {"id", "b", "ds"} + + +def test_check_destructive_schema_change_ignores_when_on_destructive_is_ignore(make_snapshot): + from sqlmesh.core.schema_diff import ( + TableAlterChangeColumnTypeOperation, + TableAlterColumn, + ) + from sqlmesh.core.snapshot.evaluator import _check_destructive_schema_change + + model_ignore = SqlModel( + name="db.m", + dialect="postgres", + kind=IncrementalByTimeRangeKind( + time_column="ds", + forward_only=True, + on_destructive_change=OnDestructiveChange.IGNORE, + ), + query=parse_one( + "SELECT 'x'::VARCHAR(50) AS val, '2023-01-01'::DATE AS ds", read="postgres" + ), + ) + snapshot = make_snapshot(model_ignore) + snapshot.categorize_as(SnapshotChangeCategory.BREAKING, forward_only=True) + assert snapshot.is_no_rebuild + assert snapshot.needs_destructive_check(set()) + + destructive_type_op = TableAlterChangeColumnTypeOperation( + target_table=exp.to_table("db.m"), + column_parts=[TableAlterColumn.primitive("val")], + column_type=exp.DataType.build("VARCHAR(50)", dialect="postgres"), + current_type=exp.DataType.build("VARCHAR(10)", dialect="postgres"), + expected_table_struct=exp.DataType.build("STRUCT"), + array_element_selector="", + is_part_of_destructive_change=True, + ) + assert destructive_type_op.is_destructive + # Must NOT raise DestructiveChangeError when on_destructive_change=IGNORE + _check_destructive_schema_change(snapshot, [destructive_type_op], set()) + + +def test_dev_preview_schema_introspection_failure_drops_dev_table_and_retry_validates( + duck_conn, + make_snapshot, + mocker: MockerFixture, +): + """If introspecting the schemas of a freshly created dev preview fails, the preview must not be + left behind unvalidated, otherwise a retry would skip creation and never validate it.""" + adapter = create_engine_adapter(lambda: duck_conn, "duckdb") + evaluator = SnapshotEvaluator(adapter) + + duck_conn.execute("CREATE TABLE raw_tbl AS SELECT 1 AS id, 'x' AS b, DATE '2023-01-01' AS ds") + + base_model = SqlModel( + name="test_schema.test_model", + dialect="duckdb", + kind=IncrementalByUniqueKeyKind( + unique_key="id", + forward_only=True, + on_destructive_change=OnDestructiveChange.ERROR, + on_additive_change=OnAdditiveChange.ERROR, + ), + query=parse_one("SELECT * FROM raw_tbl WHERE ds BETWEEN @start_ds AND @end_ds"), + ) + prod_snapshot = make_snapshot(base_model, version="1") + prod_snapshot.categorize_as(SnapshotChangeCategory.BREAKING) + evaluator.create_physical_schemas([prod_snapshot], DeployabilityIndex.all_deployable()) + evaluator.create( + [prod_snapshot], + {prod_snapshot.snapshot_id: prod_snapshot}, + deployability_index=DeployabilityIndex.all_deployable(), + ) + + dev_snapshot = make_snapshot(base_model.copy(update={"stamp": "dev_v2"})) + dev_snapshot.previous_versions = prod_snapshot.all_versions + dev_snapshot.categorize_as(SnapshotChangeCategory.INDIRECT_BREAKING, forward_only=True) + dev_table = dev_snapshot.table_name(is_deployable=False) + + duck_conn.execute("ALTER TABLE raw_tbl DROP COLUMN b") + + failing = mocker.patch.object( + type(adapter), "get_alter_operations", side_effect=RuntimeError("transient") + ) + with pytest.raises(SnapshotCreationFailedError) as exc_info: + evaluator.create( + [dev_snapshot], + {dev_snapshot.snapshot_id: dev_snapshot}, + deployability_index=DeployabilityIndex.none_deployable(), + ) + mocker.stop(failing) + assert isinstance(exc_info.value.errors[0].__cause__, RuntimeError) + assert not adapter.table_exists(dev_table) + + # A failing existence check on the prod table (after the preview was created) is handled alike. + prod_table = prod_snapshot.table_name(is_deployable=True) + original_table_exists = type(adapter).table_exists + + def flaky_table_exists(self, table_name, *args, **kwargs): + if str(table_name) == prod_table and original_table_exists(self, dev_table): + raise RuntimeError("transient exists") + return original_table_exists(self, table_name, *args, **kwargs) + + flaky = mocker.patch.object(type(adapter), "table_exists", flaky_table_exists) + with pytest.raises(SnapshotCreationFailedError) as exists_info: + evaluator.create( + [dev_snapshot], + {dev_snapshot.snapshot_id: dev_snapshot}, + deployability_index=DeployabilityIndex.none_deployable(), + ) + mocker.stop(flaky) + assert "transient exists" in str(exists_info.value.errors[0].__cause__) + assert not adapter.table_exists(dev_table) + + # A failing cleanup must not mask the original error. + failing = mocker.patch.object( + type(adapter), "get_alter_operations", side_effect=RuntimeError("transient") + ) + failing_drop = mocker.patch.object( + type(adapter), "drop_table", side_effect=RuntimeError("drop failed") + ) + with pytest.raises(SnapshotCreationFailedError) as drop_info: + evaluator.create( + [dev_snapshot], + {dev_snapshot.snapshot_id: dev_snapshot}, + deployability_index=DeployabilityIndex.none_deployable(), + ) + mocker.stop(failing) + mocker.stop(failing_drop) + assert str(drop_info.value.errors[0].__cause__) == "transient" + adapter.drop_table(dev_table) + + # The retry creates the preview again and now validates it. + with pytest.raises(SnapshotCreationFailedError) as retry_info: + evaluator.create( + [dev_snapshot], + {dev_snapshot.snapshot_id: dev_snapshot}, + deployability_index=DeployabilityIndex.none_deployable(), + ) + assert isinstance(retry_info.value.errors[0].__cause__, DestructiveChangeError) + assert not adapter.table_exists(dev_table)