diff --git a/sqlmesh/core/config/common.py b/sqlmesh/core/config/common.py index dca472d7a9..cd502d7859 100644 --- a/sqlmesh/core/config/common.py +++ b/sqlmesh/core/config/common.py @@ -4,6 +4,8 @@ from enum import Enum import re +from sqlglot import exp + from sqlmesh.utils import classproperty from sqlmesh.utils.errors import ConfigError from sqlmesh.utils.pydantic import field_validator @@ -173,6 +175,25 @@ def _validate_type(v: t.Any) -> None: )(_variables_validator) +def _virtual_environment_mode_validator(cls: t.Type, v: t.Any) -> t.Any: + if v is None: + return cls.model_fields["virtual_environment_mode"].default + if isinstance(v, (exp.Identifier, exp.Var)): + return v.name.lower() + if isinstance(v, exp.Literal) and v.is_string: + return v.this.lower() + if isinstance(v, str) and not isinstance(v, VirtualEnvironmentMode): + return v.lower() + return v + + +virtual_environment_mode_validator = field_validator( + "virtual_environment_mode", + mode="before", + check_fields=False, +)(_virtual_environment_mode_validator) + + def compile_regex_mapping(value: t.Dict[str | re.Pattern, t.Any]) -> t.Dict[re.Pattern, t.Any]: """ Utility function to compile a dict of { "string regex pattern" : "string value" } into { "": "string value" } diff --git a/sqlmesh/core/config/model.py b/sqlmesh/core/config/model.py index e7af572cc7..282635f9b1 100644 --- a/sqlmesh/core/config/model.py +++ b/sqlmesh/core/config/model.py @@ -5,6 +5,7 @@ from sqlglot import exp from sqlmesh.core.dialect import parse_one, extract_func_call from sqlmesh.core.config.base import BaseConfig +from sqlmesh.core.config.common import VirtualEnvironmentMode, virtual_environment_mode_validator from sqlmesh.core.model.kind import ( ModelKind, OnDestructiveChange, @@ -51,6 +52,7 @@ class ModelDefaultsConfig(BaseConfig): post_statements: The list of SQL statements that get executed before a model runs. on_virtual_update: The list of SQL statements to be executed after the virtual update. gateway: The gateway used by models that do not specify one explicitly. + virtual_environment_mode: Indicates how environments should be handled for models that do not specify one explicitly. """ @@ -78,11 +80,13 @@ class ModelDefaultsConfig(BaseConfig): post_statements: t.Optional[t.List[t.Union[str, exp.Expr]]] = None on_virtual_update: t.Optional[t.List[t.Union[str, exp.Expr]]] = None gateway: t.Optional[str] = None + virtual_environment_mode: t.Optional[VirtualEnvironmentMode] = None _model_kind_validator = model_kind_validator _on_destructive_change_validator = on_destructive_change_validator _on_additive_change_validator = on_additive_change_validator _cron_tz_validator = cron_tz_validator + _virtual_environment_mode_validator = virtual_environment_mode_validator @field_validator("audits", mode="before") def _audits_validator(cls, v: t.Any) -> t.Any: diff --git a/sqlmesh/core/config/root.py b/sqlmesh/core/config/root.py index b36b7dadc1..0a8b023756 100644 --- a/sqlmesh/core/config/root.py +++ b/sqlmesh/core/config/root.py @@ -20,7 +20,11 @@ VirtualEnvironmentMode, ) from sqlmesh.core.config.base import BaseConfig, UpdateStrategy -from sqlmesh.core.config.common import variables_validator, compile_regex_mapping +from sqlmesh.core.config.common import ( + compile_regex_mapping, + variables_validator, + virtual_environment_mode_validator, +) from sqlmesh.core.config.connection import ( ConnectionConfig, DuckDBConnectionConfig, @@ -217,6 +221,7 @@ class Config(BaseConfig): _connection_config_validator = connection_config_validator _scheduler_config_validator = scheduler_config_validator # type: ignore _variables_validator = variables_validator + _virtual_environment_mode_validator = virtual_environment_mode_validator @model_validator(mode="before") def _normalize_and_validate_fields(cls, data: t.Any) -> t.Any: @@ -273,7 +278,13 @@ def _normalize_identifiers(key: str) -> None: "Please specify one or the other" ) - if self.plan.use_finalized_state and not self.virtual_environment_mode.is_full: + if self.plan.use_finalized_state and ( + not self.virtual_environment_mode.is_full + or ( + self.model_defaults.virtual_environment_mode is not None + and not self.model_defaults.virtual_environment_mode.is_full + ) + ): raise ConfigError( "Using the finalized state is only supported when `virtual_environment_mode` is set to `full`." ) diff --git a/sqlmesh/core/context.py b/sqlmesh/core/context.py index fd7f8ad071..046bde7e45 100644 --- a/sqlmesh/core/context.py +++ b/sqlmesh/core/context.py @@ -62,6 +62,7 @@ Config, load_configs, ) +from sqlmesh.core.config.common import VirtualEnvironmentMode from sqlmesh.core.config.connection import ConnectionConfig from sqlmesh.core.config.loader import C from sqlmesh.core.config.root import RegexKeyDict @@ -550,8 +551,22 @@ def upsert_model(self, model: t.Union[str, Model], **kwargs: t.Any) -> Model: raise SQLMeshError(f"The disabled model '{model.name}' cannot be upserted") path = model._path + if kwargs.get("virtual_environment_mode") is not None: + # model_copy(update=...) does not validate, so normalize strings to the enum here. + raw_mode = kwargs["virtual_environment_mode"] + try: + kwargs["virtual_environment_mode"] = VirtualEnvironmentMode( + str(getattr(raw_mode, "value", raw_mode)).lower() + ) + except ValueError: + raise ConfigError( + f"Invalid virtual_environment_mode '{raw_mode}'. " + f"Expected one of: {', '.join(m.value for m in VirtualEnvironmentMode)}." + ) + model = model.copy(update=kwargs) model._path = path + self._validate_model_virtual_environment_mode(model) self.dag.add(model.fqn, model.depends_on) @@ -780,6 +795,7 @@ def load( self._uncached_model_names = uncached for model in self._models.values(): + self._validate_model_virtual_environment_mode(model) self.dag.add(model.fqn, model.depends_on) if update_schemas: @@ -856,8 +872,23 @@ def _update_model_schemas_and_validate(self, model_fqns: t.Optional[t.Set[str]] for model in models.values(): # The model definition can be validated correctly only after the schema is set. + self._validate_model_virtual_environment_mode(model) model.validate_definition() + def _validate_model_virtual_environment_mode(self, model: Model) -> None: + if ( + # Plan selection reads the root flag for every model, so check it alongside the model's own project flag. + ( + self.config.plan.use_finalized_state + or self.config_for_node(model).plan.use_finalized_state + ) + and not model.kind.is_symbolic + and not model.virtual_environment_mode.is_full + ): + raise ConfigError( + "Using the finalized state is only supported when `virtual_environment_mode` is set to `full`." + ) + @python_api_analytics def run( self, @@ -1875,19 +1906,43 @@ def plan_builder( max_interval_end_per_model, ) - if not self.config.virtual_environment_mode.is_full: + model_snapshots = [ + s for s in context_diff.snapshots.values() if s.is_model and not s.is_symbolic + ] + if model_snapshots and all(not s.virtual_environment_mode.is_full for s in model_snapshots): + forward_only = True + elif not model_snapshots and not self.config.virtual_environment_mode.is_full: forward_only = True elif forward_only is None: forward_only = self.config.plan.forward_only # When handling prod restatements, only clear intervals from other model versions if we are using full virtual environments # If we are not, then there is no point, because none of the data in dev environments can be promoted by definition + has_full_vde_snapshots = ( + any(s.virtual_environment_mode.is_full for s in model_snapshots) + if model_snapshots + else self.config.virtual_environment_mode.is_full + ) restate_all_snapshots = ( - expanded_restate_models is not None - and not is_dev - and self.config.virtual_environment_mode.is_full + expanded_restate_models is not None and not is_dev and has_full_vde_snapshots ) + if is_dev and context_diff.prod_snapshots: + mismatched_modes = { + (prod_info.virtual_environment_mode.value, s.virtual_environment_mode.value) + for s in context_diff.snapshots.values() + if s.is_model + and not s.is_symbolic + and (prod_info := context_diff.prod_snapshots.get(s.name)) is not None + and prod_info.virtual_environment_mode != s.virtual_environment_mode + } + for prod_mode, local_mode in sorted(mismatched_modes): + self.console.log_warning( + f"Local virtual_environment_mode ('{local_mode}') differs from the active `prod` " + f"environment state ('{prod_mode}'). Affected models will be rebuilt in preview " + f"and will require a full rebuild when applied to `prod`." + ) + return self.PLAN_BUILDER_TYPE( context_diff=context_diff, start=start, diff --git a/sqlmesh/core/context_diff.py b/sqlmesh/core/context_diff.py index 047e58609a..c45b65a0fd 100644 --- a/sqlmesh/core/context_diff.py +++ b/sqlmesh/core/context_diff.py @@ -88,6 +88,8 @@ class ContextDiff(PydanticModel): """Environment statements.""" diff_rendered: bool = False """Whether the diff should compare raw vs rendered models""" + prod_snapshots: t.Dict[str, SnapshotTableInfo] = {} + """Promoted snapshot table infos currently active in the `prod` environment.""" @classmethod def create( @@ -231,6 +233,22 @@ def create( else: previous_plan_id = env.plan_id if env and not is_new_environment else None + if environment == c.PROD: + prod_env = existing_env + elif is_new_environment and create_from.lower() == c.PROD: + prod_env = env + else: + prod_env = state_reader.get_environment(c.PROD) + + prod_snapshots: t.Dict[str, SnapshotTableInfo] = {} + if prod_env and not prod_env.expired: + prod_snapshot_infos = ( + prod_env.promoted_snapshots + if not ensure_finalized_snapshots + else prod_env.finalized_or_current_snapshots + ) + prod_snapshots = {s.name: s for s in prod_snapshot_infos} + return ContextDiff( environment=environment, is_new_environment=is_new_environment, @@ -253,6 +271,7 @@ def create( environment_statements=environment_statements, previous_gateway_managed_virtual_layer=env.gateway_managed if env else False, gateway_managed_virtual_layer=gateway_managed_virtual_layer, + prod_snapshots=prod_snapshots, ) @classmethod @@ -272,6 +291,12 @@ def create_no_diff(cls, environment: str, state_reader: StateReader) -> ContextD environment_statements = state_reader.get_environment_statements(environment) snapshots = state_reader.get_snapshots(env.snapshots) + prod_env = env if env.name == c.PROD else state_reader.get_environment(c.PROD) + prod_snapshots = ( + {s.name: s for s in prod_env.promoted_snapshots} + if prod_env and not prod_env.expired + else {} + ) return ContextDiff( environment=env.name, @@ -294,6 +319,7 @@ def create_no_diff(cls, environment: str, state_reader: StateReader) -> ContextD environment_statements=environment_statements, previous_gateway_managed_virtual_layer=env.gateway_managed, gateway_managed_virtual_layer=env.gateway_managed, + prod_snapshots=prod_snapshots, ) @property diff --git a/sqlmesh/core/loader.py b/sqlmesh/core/loader.py index 5a1ce8b17d..3d68db9fad 100644 --- a/sqlmesh/core/loader.py +++ b/sqlmesh/core/loader.py @@ -772,7 +772,10 @@ def _load_sql_models( infer_names=self.config.model_naming.infer_names, signal_definitions=signals, default_catalog_per_gateway=self.context.default_catalog_per_gateway, - virtual_environment_mode=self.config.virtual_environment_mode, + virtual_environment_mode=( + self.config.model_defaults.virtual_environment_mode + or self.config.virtual_environment_mode + ), ) with create_process_pool_executor( @@ -855,7 +858,10 @@ def _load_python_models( audit_definitions=audits, signal_definitions=signals, default_catalog_per_gateway=self.context.default_catalog_per_gateway, - virtual_environment_mode=self.config.virtual_environment_mode, + virtual_environment_mode=( + self.config.model_defaults.virtual_environment_mode + or self.config.virtual_environment_mode + ), ): if model.enabled: models[model.fqn] = model diff --git a/sqlmesh/core/model/decorator.py b/sqlmesh/core/model/decorator.py index 304c07276c..4e2db37db7 100644 --- a/sqlmesh/core/model/decorator.py +++ b/sqlmesh/core/model/decorator.py @@ -159,7 +159,7 @@ def model( variables: t.Optional[t.Dict[str, t.Any]] = None, infer_names: t.Optional[bool] = False, blueprint_variables: t.Optional[t.Dict[str, t.Any]] = None, - virtual_environment_mode: VirtualEnvironmentMode = VirtualEnvironmentMode.default, + virtual_environment_mode: t.Optional[VirtualEnvironmentMode] = None, ) -> Model: """Get the model registered by this function.""" env: t.Dict[str, t.Tuple[t.Any, t.Optional[bool]]] = {} @@ -218,6 +218,11 @@ def model( ) rendered_defaults = parse_defaults_properties(rendered_defaults, dialect=dialect) + if ( + virtual_environment_mode is not None + and rendered_defaults.get("virtual_environment_mode") is None + ): + rendered_defaults["virtual_environment_mode"] = virtual_environment_mode common_kwargs = { "defaults": rendered_defaults, @@ -236,7 +241,6 @@ def model( "audit_definitions": audit_definitions, "signal_definitions": signal_definitions, "blueprint_variables": blueprint_variables, - "virtual_environment_mode": virtual_environment_mode, **rendered_fields, } diff --git a/sqlmesh/core/model/definition.py b/sqlmesh/core/model/definition.py index e8e122dece..4cad480fe1 100644 --- a/sqlmesh/core/model/definition.py +++ b/sqlmesh/core/model/definition.py @@ -239,7 +239,7 @@ def render_definition( expressions.append( exp.Property( this=field_name, - value=exp.to_table(field_value, dialect=self.dialect), + value=d.parse_one(field_value, into=exp.Table, dialect=self.dialect), ) ) elif field_name not in ("default_catalog", "enabled", "ignored_rules_"): @@ -2296,6 +2296,13 @@ def load_sql_based_model( rendered_defaults = parse_defaults_properties(rendered_defaults, dialect=dialect) + default_virtual_environment_mode = kwargs.pop("virtual_environment_mode", None) + if ( + default_virtual_environment_mode is not None + and rendered_defaults.get("virtual_environment_mode") is None + ): + rendered_defaults["virtual_environment_mode"] = default_virtual_environment_mode + # Extract the query and any pre/post statements query_or_seed_insert, pre_statements, post_statements, on_virtual_update, inline_audits = ( _split_sql_model_statements(expressions[1:], path, dialect=dialect) @@ -2308,8 +2315,8 @@ def load_sql_based_model( if rendered_meta.comments else None ), - **{prop.name.lower(): prop.args.get("value") for prop in rendered_meta.expressions}, **kwargs, + **{prop.name.lower(): prop.args.get("value") for prop in rendered_meta.expressions}, } # Discard the potentially half-rendered versions of these properties and replace them with the @@ -2609,11 +2616,14 @@ def _create_model( kwargs["kind"] = create_model_kind(raw_kind, dialect, defaults or {}) defaults = {k: v for k, v in (defaults or {}).items() if k in klass.all_fields()} + if defaults.get("virtual_environment_mode") is None: + defaults.pop("virtual_environment_mode", None) if issubclass(klass, ExternalModel): # An external model's gateway selects a gateway-specific source definition in # external_models.yaml, so it must remain explicit rather than inheriting the # gateway used to execute managed models in the project. defaults.pop("gateway", None) + defaults.pop("virtual_environment_mode", None) if not issubclass(klass, SqlModel): defaults.pop("optimize_query", None) diff --git a/sqlmesh/core/model/meta.py b/sqlmesh/core/model/meta.py index 67ea32f9d9..06e219f3d2 100644 --- a/sqlmesh/core/model/meta.py +++ b/sqlmesh/core/model/meta.py @@ -11,7 +11,7 @@ from sqlglot.optimizer.normalize_identifiers import normalize_identifiers from sqlmesh.core import dialect as d -from sqlmesh.core.config.common import VirtualEnvironmentMode +from sqlmesh.core.config.common import VirtualEnvironmentMode, virtual_environment_mode_validator from sqlmesh.core.constants import LIQUID_CLUSTERING_KEYWORDS from sqlmesh.core.config.linter import LinterConfig from sqlmesh.core.dialect import normalize_model_name @@ -125,6 +125,7 @@ class ModelMeta(_Node): _bool_validator = bool_validator _model_kind_validator = model_kind_validator + _virtual_environment_mode_validator = virtual_environment_mode_validator _properties_validator = properties_validator _default_catalog_validator = default_catalog_validator _depends_on_validator = depends_on_validator diff --git a/sqlmesh/core/node.py b/sqlmesh/core/node.py index 73e4a31940..8d3b65e519 100644 --- a/sqlmesh/core/node.py +++ b/sqlmesh/core/node.py @@ -15,7 +15,9 @@ from sqlmesh.utils.pydantic import ( PydanticModel, SQLGlotCron, + ValidationInfo, field_validator, + get_dialect, model_validator, PRIVATE_FIELDS, ) @@ -348,11 +350,12 @@ def copy(self, **kwargs: t.Any) -> Self: @field_validator("name", mode="before") @classmethod - def _name_validator(cls, v: t.Any) -> t.Optional[str]: + def _name_validator(cls, v: t.Any, info: ValidationInfo) -> t.Optional[str]: if v is None: return None if isinstance(v, exp.Expr): - return v.meta["sql"] + dialect = v.meta.get("dialect") or get_dialect(info) + return v.meta.get("sql") or v.sql(dialect=dialect) return str(v) @field_validator("start", "end", mode="before") diff --git a/sqlmesh/core/plan/builder.py b/sqlmesh/core/plan/builder.py index a6307a9ffd..7323133429 100644 --- a/sqlmesh/core/plan/builder.py +++ b/sqlmesh/core/plan/builder.py @@ -397,9 +397,8 @@ def _build_restatements( return {} restatements: t.Dict[SnapshotId, Interval] = {} - forward_only_preview_needed = self._forward_only_preview_needed is_preview = False - if not restate_models and forward_only_preview_needed: + if not restate_models and self._forward_only_preview_needed: # Add model names for new forward-only snapshots to the restatement list # in order to compute previews. restate_models = { @@ -445,7 +444,7 @@ def _build_restatements( if not restating_parents and snapshot.name not in restate_models: continue - if not forward_only_preview_needed: + if not self._restatement_guard_bypassed: if self._is_dev and not snapshot.is_paused: self._console.log_warning( f"Cannot restate model '{snapshot.name}' because the current version is used in production. " @@ -567,23 +566,72 @@ def _build_models_to_backfill( ).sorted } + def _is_breaking_prod_kind_change(self, snapshot: Snapshot) -> bool: + if not snapshot.is_model: + return False + prod_info = self._context_diff.prod_snapshots.get(snapshot.name) + if prod_info is None: + return False + if ( + prod_info.virtual_environment_mode.is_full and snapshot.virtual_environment_mode.is_full + ) and ( + snapshot.name in self._context_diff.modified_snapshots + and self._context_diff.modified_snapshots[snapshot.name][1].snapshot_id + == prod_info.snapshot_id + ): + return is_breaking_kind_change( + self._context_diff.modified_snapshots[snapshot.name][1], snapshot + ) + # DEV_ONLY incremental kind changes are deliberately treated conservatively as breaking. + # For DEV_ONLY models always compare against the prod table info, which can't tell whether + # the partitioning is unchanged. Comparing the full snapshots when prod is the dev baseline + # would classify the same prod transition differently across repeated dev plans. + return is_breaking_kind_change(prod_info, snapshot) + + def _is_snapshot_forward_only(self, s_id: SnapshotId) -> bool: + snapshot = self._context_diff.snapshots[s_id] + is_snapshot_dev_only = snapshot.is_model and not snapshot.virtual_environment_mode.is_full + forward_only = ( + self._forward_only or is_snapshot_dev_only or self._is_forward_only_change(s_id) + ) + if forward_only and (self._is_breaking_prod_kind_change(snapshot) or snapshot.is_seed): + # Breaking kind changes relative to prod and seed changes can't be forward-only. + forward_only = False + if forward_only and s_id.name in self._context_diff.modified_snapshots: + new, old = self._context_diff.modified_snapshots[s_id.name] + if is_breaking_kind_change(old, new): + # Breaking kind changes can't be forward-only. + forward_only = False + return forward_only + def _adjust_snapshot_intervals(self) -> None: for new, old in self._context_diff.modified_snapshots.values(): if not new.is_model or not old.is_model: continue is_same_version = old.version_get_or_generate() == new.version_get_or_generate() - if is_same_version and should_force_rebuild(old, new): + is_breaking_prod_change = self._is_breaking_prod_kind_change(new) + if (is_same_version or not new.virtual_environment_mode.is_full) and ( + should_force_rebuild(old, new) or is_breaking_prod_change + ): # If the difference between 2 snapshots requires a full rebuild, # then clear the intervals for the new snapshot. self._context_diff.snapshots[new.snapshot_id].intervals = [] elif new.snapshot_id in self._context_diff.new_snapshots: new.intervals = [] new.dev_intervals = [] - if is_same_version: + if is_same_version and not is_breaking_prod_change: new.merge_intervals(old) if new.is_forward_only: new.dev_intervals = new.intervals.copy() + for snapshot in self._context_diff.snapshots.values(): + if ( + snapshot.is_model + and not snapshot.virtual_environment_mode.is_full + and self._is_breaking_prod_kind_change(snapshot) + ): + snapshot.intervals = [] + def _check_destructive_additive_changes(self, directly_modified: t.Set[SnapshotId]) -> None: for s_id in sorted(directly_modified): if s_id.name not in self._context_diff.modified_snapshots: @@ -595,7 +643,7 @@ def _check_destructive_additive_changes(self, directly_modified: t.Set[SnapshotI ) needs_additive_check = snapshot.needs_additive_check(self._allow_additive_models) # should we raise/warn if this snapshot has/inherits a destructive change? - should_raise_or_warn = (self._is_forward_only_change(s_id) or self._forward_only) and ( + should_raise_or_warn = self._is_snapshot_forward_only(s_id) and ( needs_destructive_check or needs_additive_check ) @@ -663,12 +711,7 @@ def _categorize_snapshots( if not snapshot or not self._is_new_snapshot(snapshot): continue - forward_only = self._forward_only or self._is_forward_only_change(s_id) - if forward_only and s_id.name in self._context_diff.modified_snapshots: - new, old = self._context_diff.modified_snapshots[s_id.name] - if is_breaking_kind_change(old, new) or snapshot.is_seed: - # Breaking kind changes and seed changes can't be forward-only. - forward_only = False + forward_only = self._is_snapshot_forward_only(s_id) if s_id in self._choices: snapshot.categorize_as(self._choices[s_id], forward_only) @@ -691,7 +734,7 @@ def _categorize_snapshot( if self._context_diff.directly_modified(s_id.name): if self._auto_categorization_enabled: new, old = self._context_diff.modified_snapshots[s_id.name] - if is_breaking_kind_change(old, new): + if is_breaking_kind_change(old, new) or self._is_breaking_prod_kind_change(new): snapshot.categorize_as(SnapshotChangeCategory.BREAKING, False) return @@ -738,8 +781,14 @@ def _categorize_snapshot( if p_id in snapshot.parents: direct_parent_categories.add(parent.change_category) - if all_upstream_forward_only == {True} or ( - snapshot.is_model and snapshot.model.forward_only + _, old = self._context_diff.modified_snapshots[snapshot.name] + if ( + all_upstream_forward_only == {True} + or (snapshot.is_model and snapshot.model.forward_only) + ) and not ( + is_breaking_kind_change(old, snapshot) + or self._is_breaking_prod_kind_change(snapshot) + or snapshot.is_seed ): forward_only = True @@ -848,6 +897,8 @@ def _is_forward_only_change(self, s_id: SnapshotId) -> bool: ) and not self._context_diff.indirectly_modified(s_id.name): return False snapshot = self._context_diff.snapshots[s_id] + if snapshot.is_model and self._is_breaking_prod_kind_change(snapshot): + return False if snapshot.name in self._context_diff.modified_snapshots: _, old = self._context_diff.modified_snapshots[snapshot.name] # If the model kind has changed in a breaking way, then we can't consider this to be a forward-only change. @@ -922,11 +973,35 @@ def _ensure_new_env_with_changes(self) -> None: f"Creating a new environment requires a change, but project files match the `{self._context_diff.create_from}` environment. Make a change or use the --include-unmodified flag to create a new environment without changes." ) + @cached_property + def _restatement_guard_bypassed(self) -> bool: + """Whether explicit restatements skip the guard against restating models that are live in prod. + + This intentionally excludes the DEV_ONLY-driven preview condition of `_forward_only_preview_needed`, + since an unrelated DEV_ONLY change must not allow restating active production tables. + """ + return self._is_dev and ( + self._forward_only + or ( + self._enable_preview + and any( + snapshot.model.forward_only + for snapshot in self._modified_and_added_snapshots + if snapshot.is_model + ) + ) + ) + @cached_property def _forward_only_preview_needed(self) -> bool: """Determines whether the plan should compute previews for forward-only changes (if there are any).""" return self._is_dev and ( self._forward_only + or any( + not snapshot.virtual_environment_mode.is_full + for snapshot in self._modified_and_added_snapshots + if snapshot.is_model + ) or ( self._enable_preview and any( diff --git a/sqlmesh/core/plan/common.py b/sqlmesh/core/plan/common.py index bece17639c..72eff732e6 100644 --- a/sqlmesh/core/plan/common.py +++ b/sqlmesh/core/plan/common.py @@ -4,7 +4,13 @@ from dataclasses import dataclass, field from sqlmesh.core.state_sync import StateReader -from sqlmesh.core.snapshot import Snapshot, SnapshotId, SnapshotIdAndVersion, SnapshotNameVersion +from sqlmesh.core.snapshot import ( + Snapshot, + SnapshotId, + SnapshotIdAndVersion, + SnapshotInfoLike, + SnapshotNameVersion, +) from sqlmesh.core.snapshot.definition import Interval from sqlmesh.utils.dag import DAG from sqlmesh.utils.date import now_timestamp @@ -16,18 +22,23 @@ def should_force_rebuild(old: Snapshot, new: Snapshot) -> bool: if new.is_view and new.is_indirect_non_breaking and not new.is_forward_only: # View models always need to be rebuilt to reflect updated upstream dependencies return True - if new.is_seed and not ( - new.is_metadata - and new.previous_version - and new.previous_version.snapshot_id(new.name) == old.snapshot_id - ): + if new.is_seed and not is_seed_metadata_only_change(old.snapshot_id, new): # Seed models always need to be rebuilt to reflect changes in the seed file # Unless only their metadata has been updated (eg description added) and the seed file has not been touched return True return is_breaking_kind_change(old, new) -def is_breaking_kind_change(old: Snapshot, new: Snapshot) -> bool: +def is_seed_metadata_only_change(old_snapshot_id: SnapshotId, new: Snapshot) -> bool: + """Whether `new` only differs from the snapshot `old_snapshot_id` in its metadata (eg a description).""" + return bool( + new.is_metadata + and new.previous_version + and new.previous_version.snapshot_id(new.name) == old_snapshot_id + ) + + +def is_breaking_kind_change(old: SnapshotInfoLike, new: SnapshotInfoLike) -> bool: if new.is_model != old.is_model: # If one is a model and the other isn't, then we need to rebuild return True @@ -38,15 +49,15 @@ def is_breaking_kind_change(old: Snapshot, new: Snapshot) -> bool: if old.virtual_environment_mode != new.virtual_environment_mode: # If the virtual environment mode has changed, then we need to rebuild return True - if old.model.kind.name == new.model.kind.name: + if old.model_kind_name == new.model_kind_name: # If the kind hasn't changed, then we don't need to rebuild return False if not old.is_incremental or not new.is_incremental: # If either is not incremental, then we need to rebuild return True - if old.model.partitioned_by == new.model.partitioned_by: + if isinstance(old, Snapshot) and isinstance(new, Snapshot): # If the partitioning hasn't changed, then we don't need to rebuild - return False + return old.model.partitioned_by != new.model.partitioned_by return True diff --git a/sqlmesh/core/plan/evaluator.py b/sqlmesh/core/plan/evaluator.py index 71e5285d63..fff8010b1c 100644 --- a/sqlmesh/core/plan/evaluator.py +++ b/sqlmesh/core/plan/evaluator.py @@ -21,7 +21,11 @@ from sqlmesh.core.execution_observation import action, console_observer_scope from sqlmesh.core import constants as c from sqlmesh.core.console import Console, get_console -from sqlmesh.core.environment import EnvironmentNamingInfo, execute_environment_statements +from sqlmesh.core.environment import ( + Environment, + EnvironmentNamingInfo, + execute_environment_statements, +) from sqlmesh.core.macros import RuntimeStage from sqlmesh.core.snapshot.definition import to_view_mapping, SnapshotTableInfo from sqlmesh.core.plan import stages @@ -38,7 +42,11 @@ ) from sqlmesh.utils import to_snake_case from sqlmesh.core.state_sync import StateSync -from sqlmesh.core.plan.common import identify_restatement_intervals_across_snapshot_versions +from sqlmesh.core.plan.common import ( + identify_restatement_intervals_across_snapshot_versions, + is_breaking_kind_change, + is_seed_metadata_only_change, +) from sqlmesh.utils import CorrelationId from sqlmesh.utils.concurrency import NodeExecutionFailedError from sqlmesh.utils.errors import PlanError, ConflictingPlanError, SQLMeshError @@ -117,6 +125,7 @@ def evaluate( def _evaluate_stages( self, plan_stages: t.List[stages.PlanStage], plan: EvaluatablePlan ) -> None: + self._clear_stale_dev_only_intervals_in_prod(plan) for stage in plan_stages: stage_name = stage.__class__.__name__ handler_name = f"visit_{to_snake_case(stage_name)}" @@ -127,6 +136,115 @@ def _evaluate_stages( with action("stage", stage_name, native_plan_id=plan.plan_id): handler(stage, plan) + def _clear_stale_dev_only_intervals_in_prod(self, plan: EvaluatablePlan) -> None: + if not isinstance(plan, EvaluatablePlan) or not plan.is_prod: + return + + dev_only_infos = [ + s + for s in plan.environment.snapshots + if s.is_model and not s.virtual_environment_mode.is_full + ] + if not dev_only_infos: + return + + existing_prod = self.state_sync.get_environment(c.PROD) + if not isinstance(existing_prod, Environment) or existing_prod.expired: + return + + existing_prod_by_name = {s.name: s for s in existing_prod.promoted_snapshots} + new_snapshots_by_id = {s.snapshot_id: s for s in plan.new_snapshots} + + incremental_kind_change_ids: t.Set[SnapshotId] = set() + for s in dev_only_infos: + old_prod = existing_prod_by_name.get(s.name) + if old_prod is None or old_prod.snapshot_id == s.snapshot_id: + continue + if ( + s.is_forward_only + and old_prod.is_incremental + and s.is_incremental + and old_prod.virtual_environment_mode == s.virtual_environment_mode + and old_prod.model_kind_name != s.model_kind_name + ): + incremental_kind_change_ids.add(old_prod.snapshot_id) + if s.snapshot_id not in new_snapshots_by_id: + incremental_kind_change_ids.add(s.snapshot_id) + + # Seeds that aren't new in this plan (eg created by an earlier dev plan) are fetched too, to tell + # metadata-only changes from seed data changes. + seed_ids_to_fetch = { + s.snapshot_id + for s in dev_only_infos + if s.is_seed + and not s.is_forward_only + and s.snapshot_id not in new_snapshots_by_id + and (old_prod := existing_prod_by_name.get(s.name)) is not None + and old_prod.snapshot_id != s.snapshot_id + } + ids_to_fetch = incremental_kind_change_ids | seed_ids_to_fetch + + fetched_snapshots = self.state_sync.get_snapshots(ids_to_fetch) if ids_to_fetch else {} + + rebuild_infos: t.List[SnapshotTableInfo] = [] + for s in dev_only_infos: + old_prod = existing_prod_by_name.get(s.name) + if old_prod is None: + if not s.is_forward_only: + rebuild_infos.append(s) + continue + if old_prod.snapshot_id == s.snapshot_id: + continue + if not s.is_forward_only: + new_full = new_snapshots_by_id.get(s.snapshot_id) or fetched_snapshots.get( + s.snapshot_id + ) + if not ( + new_full is not None + and new_full.is_seed + and old_prod.virtual_environment_mode == s.virtual_environment_mode + and is_seed_metadata_only_change(old_prod.snapshot_id, new_full) + ): + # Metadata-only seed changes keep their data, so their intervals must be preserved. + rebuild_infos.append(s) + continue + old_target: SnapshotInfoLike = fetched_snapshots.get(old_prod.snapshot_id, old_prod) + new_target: SnapshotInfoLike = new_snapshots_by_id.get( + s.snapshot_id, fetched_snapshots.get(s.snapshot_id, s) + ) + if is_breaking_kind_change(old_target, new_target): + rebuild_infos.append(s) + + if not rebuild_infos: + return + + missing_rebuild_ids = [ + s.snapshot_id + for s in rebuild_infos + if s.snapshot_id not in new_snapshots_by_id and s.snapshot_id not in fetched_snapshots + ] + if missing_rebuild_ids: + fetched_snapshots.update(self.state_sync.get_snapshots(missing_rebuild_ids)) + + rebuild_snapshots = [ + s_full.copy(update={"effective_from": None, "intervals": [], "dev_intervals": []}) + for s in rebuild_infos + if ( + s_full := new_snapshots_by_id.get(s.snapshot_id) + or fetched_snapshots.get(s.snapshot_id) + ) + is not None + ] + if not rebuild_snapshots: + return + + refreshed = self.state_sync.refresh_snapshot_intervals(rebuild_snapshots) + intervals_to_remove = [ + (s, (s.intervals[0][0], s.intervals[-1][1])) for s in refreshed if s.intervals + ] + if intervals_to_remove: + self.state_sync.remove_intervals(intervals_to_remove, remove_shared_versions=True) + def visit_before_all_stage(self, stage: stages.BeforeAllStage, plan: EvaluatablePlan) -> None: execute_environment_statements( adapter=self.snapshot_evaluator.adapter, diff --git a/sqlmesh/core/plan/stages.py b/sqlmesh/core/plan/stages.py index 729e1705b4..a11914fbd7 100644 --- a/sqlmesh/core/plan/stages.py +++ b/sqlmesh/core/plan/stages.py @@ -3,7 +3,7 @@ from dataclasses import dataclass from sqlmesh.core import constants as c from sqlmesh.core.environment import EnvironmentStatements, EnvironmentNamingInfo, Environment -from sqlmesh.core.plan.common import should_force_rebuild +from sqlmesh.core.plan.common import is_breaking_kind_change, should_force_rebuild from sqlmesh.core.plan.definition import EvaluatablePlan from sqlmesh.core.state_sync import StateReader from sqlmesh.core.scheduler import merged_missing_intervals, SnapshotToIntervals @@ -658,8 +658,18 @@ def _adjust_intervals( # Make sure the intervals are up to date and restatements are reflected self.state_reader.refresh_snapshot_intervals(snapshots_by_name.values()) + has_dev_only_models = any( + s.is_model and not s.virtual_environment_mode.is_full + for s in snapshots_by_name.values() + ) + prod_environment: t.Optional[Environment] = None + if existing_environment and existing_environment.name == c.PROD: + prod_environment = existing_environment + elif not existing_environment or has_dev_only_models: + prod_environment = self.state_reader.get_environment(c.PROD) + if not existing_environment: - existing_environment = self.state_reader.get_environment(c.PROD) + existing_environment = prod_environment if existing_environment: new_snapshot_ids = set() @@ -687,6 +697,57 @@ def _adjust_intervals( # then clear the intervals for the new snapshot. new.intervals = [] + if has_dev_only_models: + candidate_envs = [ + env + for env in (existing_environment, prod_environment) + if isinstance(env, Environment) and not env.expired + ] + if candidate_envs: + incremental_kind_change_ids: t.Set[SnapshotId] = set() + env_promoted_maps = [ + {s.name: s for s in env.promoted_snapshots} for env in candidate_envs + ] + for new in snapshots_by_name.values(): + if ( + not new.is_model + or new.virtual_environment_mode.is_full + or not new.intervals + ): + continue + for promoted_by_name in env_promoted_maps: + old_info = promoted_by_name.get(new.name) + if old_info is None or old_info.snapshot_id == new.snapshot_id: + continue + if ( + old_info.is_incremental + and new.is_incremental + and old_info.virtual_environment_mode == new.virtual_environment_mode + and old_info.model_kind_name != new.model_kind_name + ): + incremental_kind_change_ids.add(old_info.snapshot_id) + + full_old_snapshots = ( + self.state_reader.get_snapshots(incremental_kind_change_ids) + if incremental_kind_change_ids + else {} + ) + for new in snapshots_by_name.values(): + if ( + not new.is_model + or new.virtual_environment_mode.is_full + or not new.intervals + ): + continue + for promoted_by_name in env_promoted_maps: + old_info = promoted_by_name.get(new.name) + if old_info is None or old_info.snapshot_id == new.snapshot_id: + continue + old_target = full_old_snapshots.get(old_info.snapshot_id, old_info) + if is_breaking_kind_change(old_target, new): + new.intervals = [] + break + for new_snapshot in plan.new_snapshots: if new_snapshot.is_forward_only: # Forward-only snapshots inherit intervals in dev because of cloning diff --git a/sqlmesh/core/snapshot/evaluator.py b/sqlmesh/core/snapshot/evaluator.py index 345f8816e6..7789edfbc3 100644 --- a/sqlmesh/core/snapshot/evaluator.py +++ b/sqlmesh/core/snapshot/evaluator.py @@ -23,6 +23,7 @@ from __future__ import annotations import abc +import itertools import logging import typing as t import sys @@ -1134,7 +1135,7 @@ def _clone_snapshot_in_dev( logger.info(f"Cloning table '{source_table_name}' into '{target_table_name}'") adapter.clone_table( target_table_name, - snapshot.table_name(), + source_table_name, rendered_physical_properties=rendered_physical_properties, table_format=snapshot.model.table_format, ) @@ -1389,7 +1390,7 @@ def _cleanup_snapshot( snapshot = snapshot.table_info table_names = [(False, snapshot.table_name(is_deployable=False))] - if not dev_table_only: + if not dev_table_only and snapshot.virtual_environment_mode.is_full: table_names.append((True, snapshot.table_name(is_deployable=True))) evaluation_strategy = _evaluation_strategy(snapshot, adapter) @@ -1573,7 +1574,7 @@ def _execute_create( def _can_clone(self, snapshot: Snapshot, deployability_index: DeployabilityIndex) -> bool: adapter = self.get_adapter(snapshot.model.gateway) - return ( + if not ( snapshot.is_forward_only and snapshot.is_materialized and bool(snapshot.previous_versions) @@ -1582,8 +1583,12 @@ def _can_clone(self, snapshot: Snapshot, deployability_index: DeployabilityIndex and not snapshot.is_managed and not snapshot.is_dbt_custom and not deployability_index.is_deployable(snapshot) - # If the deployable table is missing we can't clone it - and adapter.table_exists(snapshot.table_name()) + ): + return False + # If the deployable object is missing or is not a physical table (e.g. a virtual VIEW in prod), we can't clone it + source_data_object = adapter.get_data_object(snapshot.table_name(), safe_to_cache=True) + return source_data_object is not None and bool( + source_data_object.type.is_table or source_data_object.type.is_managed_table ) def _get_physical_data_objects( @@ -2533,15 +2538,28 @@ def create( ) -> None: model = t.cast(SeedModel, model) if not model.is_hydrated and self.adapter.table_exists(table_name): - # This likely means that the table was created and populated previously, but the evaluation stage - # failed before the interval could be added for this model. + # There is no seed content to rebuild from, so whatever exists at the target is left untouched. + # Usually this is the table created and populated previously, before the evaluation stage failed + # and the interval could be added. It may also be an object of the wrong type (eg a VIEW). logger.warning( - "Seed model '%s' is not hydrated, but the table '%s' exists. Skipping creation", + "Seed model '%s' is not hydrated, but the object '%s' exists. Leaving it untouched " + "because there is no seed content to rebuild it from", model.name, table_name, ) return + # Render the first chunk before touching the target so that a failure to produce the seed + # content doesn't remove an existing object that can't be restored. + seed_chunks = model.render_seed() + first_chunk = next(seed_chunks, None) + + if is_table_deployable and model.virtual_environment_mode.is_dev_only: + self.adapter.drop_data_object_on_type_mismatch( + self.adapter.get_data_object(table_name), + DataObjectType.TABLE, + ) + super().create( table_name, model, @@ -2552,20 +2570,21 @@ def create( ) # For seeds we insert data at the time of table creation. try: - for index, df in enumerate(model.render_seed()): - if index == 0: - self._replace_query_for_model( - model, - table_name, - df, - render_kwargs, - skip_grants=True, # Skip grants; they're applied after data insertion - **kwargs, - ) - else: - self.adapter.insert_append( - table_name, df, target_columns_to_types=model.columns_to_types - ) + if first_chunk is not None: + for index, df in enumerate(itertools.chain([first_chunk], seed_chunks)): + if index == 0: + self._replace_query_for_model( + model, + table_name, + df, + render_kwargs, + skip_grants=True, # Skip grants; they're applied after data insertion + **kwargs, + ) + else: + self.adapter.insert_append( + table_name, df, target_columns_to_types=model.columns_to_types + ) if not skip_grants: # Apply grants after seed table creation and data insertion diff --git a/sqlmesh/core/state_sync/cache.py b/sqlmesh/core/state_sync/cache.py index 77f3fc6ba5..d87afa628c 100644 --- a/sqlmesh/core/state_sync/cache.py +++ b/sqlmesh/core/state_sync/cache.py @@ -139,8 +139,16 @@ def remove_intervals( snapshot_intervals: t.Sequence[t.Tuple[SnapshotIdAndVersionLike, Interval]], remove_shared_versions: bool = False, ) -> None: - for s, _ in snapshot_intervals: - self.snapshot_cache.pop(s.snapshot_id, None) + if remove_shared_versions: + removed_names = {s.name for s, _ in snapshot_intervals} + self.snapshot_cache = { + snapshot_id: value + for snapshot_id, value in self.snapshot_cache.items() + if snapshot_id.name not in removed_names + } + else: + for s, _ in snapshot_intervals: + self.snapshot_cache.pop(s.snapshot_id, None) self.state_sync.remove_intervals(snapshot_intervals, remove_shared_versions) def unpause_snapshots( diff --git a/sqlmesh/dbt/basemodel.py b/sqlmesh/dbt/basemodel.py index 32a76aba13..0fb6dd4852 100644 --- a/sqlmesh/dbt/basemodel.py +++ b/sqlmesh/dbt/basemodel.py @@ -11,7 +11,7 @@ from sqlmesh.core import dialect as d from sqlmesh.core.config.base import UpdateStrategy -from sqlmesh.core.config.common import VirtualEnvironmentMode +from sqlmesh.core.config.common import VirtualEnvironmentMode, virtual_environment_mode_validator from sqlmesh.core.model import Model from sqlmesh.core.model.common import ParsableSql from sqlmesh.core.node import DbtNodeInfo @@ -126,6 +126,7 @@ class BaseModelConfig(GeneralConfig): tests: t.List[TestConfig] = [] dialect_: t.Optional[str] = Field(None, alias="dialect") grain: t.Union[str, t.List[str]] = [] + virtual_environment_mode: t.Optional[VirtualEnvironmentMode] = None # DBT configuration fields unique_id: str = "" @@ -148,6 +149,8 @@ class BaseModelConfig(GeneralConfig): _canonical_name: t.Optional[str] = None + _virtual_environment_mode_validator = virtual_environment_mode_validator + @field_validator("pre_hook", "post_hook", mode="before") @classmethod def _validate_hooks(cls, v: t.Union[str, t.List[t.Union[SqlStr, str]]]) -> t.List[Hook]: @@ -288,7 +291,13 @@ def fqn(self) -> str: @property def sqlmesh_config_fields(self) -> t.Set[str]: - return {"description", "owner", "stamp", "storage_format"} + return { + "description", + "owner", + "stamp", + "storage_format", + "virtual_environment_mode", + } @property def node_info(self) -> DbtNodeInfo: diff --git a/sqlmesh/dbt/loader.py b/sqlmesh/dbt/loader.py index ac4764f2af..059e8603b1 100644 --- a/sqlmesh/dbt/loader.py +++ b/sqlmesh/dbt/loader.py @@ -173,7 +173,10 @@ def _to_sqlmesh(config: BMC, context: DbtContext) -> Model: return config.to_sqlmesh( context, audit_definitions=audits, - virtual_environment_mode=self.config.virtual_environment_mode, + virtual_environment_mode=( + self.config.model_defaults.virtual_environment_mode + or self.config.virtual_environment_mode + ), ) for project in self._load_projects(): diff --git a/sqlmesh/dbt/model.py b/sqlmesh/dbt/model.py index 55994abf85..0b7e02d7b6 100644 --- a/sqlmesh/dbt/model.py +++ b/sqlmesh/dbt/model.py @@ -726,6 +726,14 @@ def to_sqlmesh( # Set allow_partials to True for dbt models to preserve the original semantics. allow_partials = True + raw_vde_mode = model_kwargs.pop("virtual_environment_mode", None) + resolved_vde_mode = ( + VirtualEnvironmentMode(raw_vde_mode.lower()) + if isinstance(raw_vde_mode, str) + and not isinstance(raw_vde_mode, VirtualEnvironmentMode) + else raw_vde_mode + ) or virtual_environment_mode + # pop begin for all models so we don't pass it through for non-incremental materializations # (happens if model config is microbatch but project config overrides) begin = model_kwargs.pop("begin", None) @@ -766,7 +774,7 @@ def to_sqlmesh( # Note: any table dependencies that are not referenced using the `ref` macro will not be included. extract_dependencies_from_query=False, allow_partials=allow_partials, - virtual_environment_mode=virtual_environment_mode, + virtual_environment_mode=resolved_vde_mode, dbt_node_info=self.node_info, **optional_kwargs, **model_kwargs, diff --git a/sqlmesh/dbt/seed.py b/sqlmesh/dbt/seed.py index c0c8186f29..ea85e839d3 100644 --- a/sqlmesh/dbt/seed.py +++ b/sqlmesh/dbt/seed.py @@ -78,6 +78,14 @@ def to_sqlmesh( kwargs["columns"] = new_columns + raw_vde_mode = kwargs.pop("virtual_environment_mode", None) + resolved_vde_mode = ( + VirtualEnvironmentMode(raw_vde_mode.lower()) + if isinstance(raw_vde_mode, str) + and not isinstance(raw_vde_mode, VirtualEnvironmentMode) + else raw_vde_mode + ) or virtual_environment_mode + # dbt treats single whitespace as a null value csv_settings = CsvSettings( delimiter=self.delimiter, @@ -90,7 +98,7 @@ def to_sqlmesh( SeedKind(path=seed_path, csv_settings=csv_settings), dialect=self.dialect(context), audit_definitions=audit_definitions, - virtual_environment_mode=virtual_environment_mode, + virtual_environment_mode=resolved_vde_mode, start=self.start or context.sqlmesh_config.model_defaults.start, dbt_node_info=self.node_info, **kwargs, diff --git a/tests/core/integration/test_dev_only_vde.py b/tests/core/integration/test_dev_only_vde.py index 611e207771..d0d696d806 100644 --- a/tests/core/integration/test_dev_only_vde.py +++ b/tests/core/integration/test_dev_only_vde.py @@ -1,12 +1,26 @@ from __future__ import annotations +from contextlib import ExitStack +from pathlib import Path import typing as t import pytest +from pytest_mock.plugin import MockerFixture +from sqlglot import exp from sqlmesh.core.model.common import ParsableSql import time_machine from sqlmesh.core import dialect as d +from sqlmesh.core.config import ( + Config, + DuckDBConnectionConfig, + GatewayConfig, + ModelDefaultsConfig, +) from sqlmesh.core.config.common import VirtualEnvironmentMode +from sqlmesh.core.context import Context +from sqlmesh.core.engine_adapter.duckdb import DuckDBEngineAdapter +from sqlmesh.core.engine_adapter.shared import DataObjectType +from sqlmesh.core.janitor import delete_expired_snapshots from sqlmesh.core.model import ( FullKind, IncrementalUnmanagedKind, @@ -18,7 +32,8 @@ from sqlmesh.core.snapshot import ( SnapshotChangeCategory, ) -from sqlmesh.utils.date import to_date, to_timestamp +from sqlmesh.utils.date import now_timestamp, to_date, to_timestamp +from sqlmesh.utils.errors import PlanError, SQLMeshError from tests.core.integration.utils import add_projection_to_model pytestmark = pytest.mark.slow @@ -475,3 +490,978 @@ def test_virtual_environment_mode_dev_only_seed_model_change_schema( context.clear_caches() context.plan("prod", auto_apply=True, no_prompts=True, skip_tests=True) assert "new_column" in context.engine_adapter.columns("sushi.waiter_as_customer_by_day") + + +class CloningDuckDBEngineAdapter(DuckDBEngineAdapter): + SUPPORTS_CLONING = True + + def __init__(self, *args: t.Any, **kwargs: t.Any) -> None: + cloned_tables = kwargs.get("cloned_tables") + if cloned_tables is None: + cloned_tables = [] + kwargs["cloned_tables"] = cloned_tables + super().__init__(*args, **kwargs) + self.cloned_tables: t.List[t.Tuple[t.Any, t.Any]] = cloned_tables + + def clone_table( + self, + target_table_name: t.Union[str, exp.Table], + source_table_name: t.Union[str, exp.Table], + replace: bool = False, + exists: bool = True, + clone_kwargs: t.Optional[t.Dict[str, t.Any]] = None, + table_format: t.Optional[str] = None, + table_kind: t.Optional[str] = None, + **kwargs: t.Any, + ) -> None: + self._clear_data_object_cache(source_table_name) + source_obj = self.get_data_object(source_table_name) + if source_obj is None or not (source_obj.type.is_table or source_obj.type.is_managed_table): + fmt = source_obj.type.value.upper() if source_obj else "MISSING" + raise SQLMeshError( + f"[DELTA_CLONE_UNSUPPORTED_SOURCE] Unsupported clone source '{source_table_name}', whose format is {fmt}" + ) + + self.cloned_tables.append((target_table_name, source_table_name)) + target_sql = exp.to_table(target_table_name, dialect=self.dialect).sql(dialect=self.dialect) + source_sql = exp.to_table(source_table_name, dialect=self.dialect).sql(dialect=self.dialect) + if replace: + self.execute(f"CREATE OR REPLACE TABLE {target_sql} AS SELECT * FROM {source_sql}") + elif exists: + self.execute(f"CREATE TABLE IF NOT EXISTS {target_sql} AS SELECT * FROM {source_sql}") + else: + self.execute(f"CREATE TABLE {target_sql} AS SELECT * FROM {source_sql}") + self._clear_data_object_cache(target_table_name) + + +def _snap_id(ctx: Context, name: str) -> t.Any: + snap = ctx.get_snapshot(name) + assert snap is not None + return snap.snapshot_id + + +def _scalar(ctx: Context, sql: str) -> t.Any: + row = ctx.engine_adapter.fetchone(sql) + assert row is not None + return row[0] + + +def _obj_type(ctx: Context, name: str) -> DataObjectType: + obj = ctx.engine_adapter.get_data_object(name) + assert obj is not None + return obj.type + + +@time_machine.travel("2023-01-08 15:00:00 UTC") +def test_vde_mode_transition_full_to_dev_only_with_cloning( + tmp_path: Path, mocker: MockerFixture +) -> None: + mocker.patch("sqlmesh.core.engine_adapter.DuckDBEngineAdapter", CloningDuckDBEngineAdapter) + + seeds_dir = tmp_path / "seeds" + models_dir = tmp_path / "models" + seeds_dir.mkdir() + models_dir.mkdir() + db_path = tmp_path / "warehouse.db" + + (seeds_dir / "seed_data.csv").write_text("id,label\n1,alpha\n2,beta\n", encoding="utf-8") + (models_dir / "seed_m.sql").write_text( + """ + MODEL ( + name db.seed_m, + kind SEED ( + path '../seeds/seed_data.csv' + ) + ); + """, + encoding="utf-8", + ) + (models_dir / "full_m.sql").write_text( + """ + MODEL ( + name db.full_m, + kind FULL + ); + SELECT id, label FROM db.seed_m; + """, + encoding="utf-8", + ) + + def write_upstream(id_val: int) -> None: + (models_dir / "upstream.sql").write_text( + f""" + MODEL ( + name db.upstream, + kind INCREMENTAL_BY_TIME_RANGE ( + time_column ds, + batch_size 1 + ), + start '2023-01-05' + ); + SELECT {id_val}::INT AS id, @start_ds::DATE AS ds; + """, + encoding="utf-8", + ) + + def write_downstream(val: int) -> None: + (models_dir / "downstream.sql").write_text( + f""" + MODEL ( + name db.downstream, + kind INCREMENTAL_BY_TIME_RANGE ( + time_column ds + ), + start '2023-01-05' + ); + SELECT id, {val}::INT AS val, ds FROM db.upstream WHERE ds BETWEEN @start_ds AND @end_ds; + """, + encoding="utf-8", + ) + + (models_dir / "v_model.sql").write_text( + """ + MODEL ( + name db.v_model, + kind VIEW + ); + SELECT id, val, ds FROM db.downstream; + """, + encoding="utf-8", + ) + + write_upstream(1) + write_downstream(10) + + with ExitStack() as stack: + + def make_ctx(mode: VirtualEnvironmentMode) -> Context: + ctx = Context( + paths=tmp_path, + config=Config( + gateways={ + "": GatewayConfig(connection=DuckDBConnectionConfig(database=str(db_path))) + }, + model_defaults=ModelDefaultsConfig(dialect="duckdb"), + virtual_environment_mode=mode, + ), + ) + stack.callback(ctx.close) + return ctx + + # 1. Deploy prod in FULL VDE mode + ctx_full = make_ctx(VirtualEnvironmentMode.FULL) + assert isinstance(ctx_full.engine_adapter, CloningDuckDBEngineAdapter) + ctx_full.plan("prod", auto_apply=True, no_prompts=True) + prod_objs_initial = {o.name: o.type for o in ctx_full.engine_adapter.get_data_objects("db")} + assert prod_objs_initial == { + "seed_m": DataObjectType.VIEW, + "full_m": DataObjectType.VIEW, + "upstream": DataObjectType.VIEW, + "downstream": DataObjectType.VIEW, + "v_model": DataObjectType.VIEW, + } + ctx_full.close() + + # 2. Switch config to DEV_ONLY, edit downstream & apply plan("dev") #1 + write_downstream(20) + ctx_dev = make_ctx(VirtualEnvironmentMode.DEV_ONLY) + assert isinstance(ctx_dev.engine_adapter, CloningDuckDBEngineAdapter) + plan_dev1 = ctx_dev.plan_builder("dev").build() + ctx_dev.apply(plan_dev1) + assert ( + plan_dev1.context_diff.snapshots[_snap_id(ctx_dev, "db.downstream")].is_forward_only + is False + ) + assert ctx_dev.engine_adapter.cloned_tables == [] + + # 3. Edit downstream again & apply plan("dev") #2 (reproduces #5887 [DELTA_CLONE_UNSUPPORTED_SOURCE] crash on HEAD) + write_downstream(30) + ctx_dev.load() + plan_dev2 = ctx_dev.plan_builder("dev").build() + ctx_dev.apply(plan_dev2) + assert ( + plan_dev2.context_diff.snapshots[_snap_id(ctx_dev, "db.downstream")].is_forward_only + is False + ) + assert ctx_dev.engine_adapter.cloned_tables == [] + + # 4. Edit upstream (making downstream indirectly_modified) & apply plan("dev") #3 + write_upstream(2) + ctx_dev.load() + plan_dev3 = ctx_dev.plan_builder("dev").build() + ctx_dev.apply(plan_dev3) + upstream_sid = _snap_id(ctx_dev, "db.upstream") + downstream_sid = _snap_id(ctx_dev, "db.downstream") + assert downstream_sid in plan_dev3.indirectly_modified[upstream_sid] + assert plan_dev3.context_diff.snapshots[upstream_sid].is_forward_only is False + assert plan_dev3.context_diff.snapshots[downstream_sid].is_forward_only is False + assert ctx_dev.engine_adapter.cloned_tables == [] + + # 5. Edit downstream again & apply plan("dev2", create_from="dev") + write_downstream(40) + ctx_dev.load() + plan_dev2_branch = ctx_dev.plan_builder("dev2", create_from="dev").build() + ctx_dev.apply(plan_dev2_branch) + assert ( + plan_dev2_branch.context_diff.snapshots[ + _snap_id(ctx_dev, "db.downstream") + ].is_forward_only + is False + ) + assert ctx_dev.engine_adapter.cloned_tables == [] + assert ctx_dev.engine_adapter.fetchall("SELECT DISTINCT id, val FROM db__dev2.v_model") == [ + (2, 40) + ] + + # 6. plan("dev_sel", select_models=["db.downstream"]) keeps unselected FULL snapshots from prod as non-forward-only + write_downstream(50) + ctx_dev.load() + plan_dev_sel = ctx_dev.plan_builder("dev_sel", select_models=["db.downstream"]).build() + ctx_dev.apply(plan_dev_sel) + unselected_models = [ + s + for s in plan_dev_sel.context_diff.snapshots.values() + if s.is_model and s.name != '"warehouse"."db"."downstream"' + ] + assert unselected_models + for unselected_snap in unselected_models: + assert unselected_snap.virtual_environment_mode == VirtualEnvironmentMode.FULL + assert unselected_snap.is_forward_only is False + + # 7. Apply plan("prod") to migrate prod from FULL -> DEV_ONLY + plan_prod = ctx_dev.plan_builder("prod").build() + assert plan_prod.requires_backfill + ctx_dev.apply(plan_prod) + + prod_objs_migrated = {o.name: o.type for o in ctx_dev.engine_adapter.get_data_objects("db")} + assert prod_objs_migrated == { + "seed_m": DataObjectType.TABLE, + "full_m": DataObjectType.TABLE, + "upstream": DataObjectType.TABLE, + "downstream": DataObjectType.TABLE, + "v_model": DataObjectType.VIEW, + } + assert _scalar(ctx_dev, "SELECT COUNT(*) FROM db.seed_m") == 2 + assert _scalar(ctx_dev, "SELECT COUNT(*) FROM db.full_m") == 2 + assert ctx_dev.engine_adapter.fetchall( + "SELECT id, val, ds FROM db.v_model ORDER BY ds" + ) == [ + (2, 50, to_date("2023-01-05")), + (2, 50, to_date("2023-01-06")), + (2, 50, to_date("2023-01-07")), + ] + + # 8. Edit downstream & apply plan("dev") #4 -> now clones prod's physical table db.downstream! + ctx_dev.engine_adapter.cloned_tables.clear() + write_downstream(60) + ctx_dev.load() + plan_dev4 = ctx_dev.plan_builder("dev").build() + assert ( + plan_dev4.context_diff.snapshots[_snap_id(ctx_dev, "db.downstream")].is_forward_only + is True + ) + ctx_dev.apply(plan_dev4) + assert len(ctx_dev.engine_adapter.cloned_tables) == 1 + cloned_target, cloned_source = ctx_dev.engine_adapter.cloned_tables[0] + assert exp.to_table(cloned_source).name == "downstream" + assert exp.to_table(cloned_source).db == "db" + + # 9. Invalidate dev environments & run delete_expired_snapshots(ignore_ttl=True) + for env_name in ("dev", "dev2", "dev_sel"): + ctx_dev.invalidate_environment(env_name, sync=True) + failures = delete_expired_snapshots( + ctx_dev.state_sync, + ctx_dev.snapshot_evaluator, + current_ts=now_timestamp(), + ignore_ttl=True, + ) + assert failures == [] + assert [ + o + for o in ctx_dev.engine_adapter.get_data_objects("sqlmesh__db") + if not o.name.endswith("__dev") + ] == [] + assert ctx_dev.engine_adapter.get_data_object(cloned_target) is None + assert _scalar(ctx_dev, "SELECT COUNT(*) FROM db.v_model") == 3 + assert _scalar(ctx_dev, "SELECT COUNT(*) FROM db.full_m") == 2 + assert _scalar(ctx_dev, "SELECT COUNT(*) FROM db.seed_m") == 2 + + +@time_machine.travel("2023-01-08 15:00:00 UTC") +def test_vde_mode_transition_dev_only_to_full_with_cloning_and_janitor( + tmp_path: Path, mocker: MockerFixture +) -> None: + mocker.patch("sqlmesh.core.engine_adapter.DuckDBEngineAdapter", CloningDuckDBEngineAdapter) + + seeds_dir = tmp_path / "seeds" + models_dir = tmp_path / "models" + seeds_dir.mkdir() + models_dir.mkdir() + db_path = tmp_path / "warehouse.db" + + (seeds_dir / "seed_data.csv").write_text("id,label\n1,alpha\n2,beta\n", encoding="utf-8") + (models_dir / "seed_m.sql").write_text( + """ + MODEL ( + name db.seed_m, + kind SEED ( + path '../seeds/seed_data.csv' + ) + ); + """, + encoding="utf-8", + ) + (models_dir / "full_m.sql").write_text( + """ + MODEL ( + name db.full_m, + kind FULL + ); + SELECT id, label FROM db.seed_m; + """, + encoding="utf-8", + ) + (models_dir / "upstream.sql").write_text( + """ + MODEL ( + name db.upstream, + kind INCREMENTAL_BY_TIME_RANGE ( + time_column ds, + batch_size 1 + ), + start '2023-01-05' + ); + SELECT 1::INT AS id, @start_ds::DATE AS ds; + """, + encoding="utf-8", + ) + + def write_downstream(val: int) -> None: + (models_dir / "downstream.sql").write_text( + f""" + MODEL ( + name db.downstream, + kind INCREMENTAL_BY_TIME_RANGE ( + time_column ds, + forward_only true + ), + start '2023-01-05' + ); + SELECT id, {val}::INT AS val, ds FROM db.upstream WHERE ds BETWEEN @start_ds AND @end_ds; + """, + encoding="utf-8", + ) + + def write_view(tag: int) -> None: + (models_dir / "v_model.sql").write_text( + f""" + MODEL ( + name db.v_model, + kind VIEW + ); + SELECT id, val, {tag}::INT AS tag, ds FROM db.downstream; + """, + encoding="utf-8", + ) + + write_downstream(10) + write_view(1) + + with ExitStack() as stack: + + def make_ctx(mode: VirtualEnvironmentMode) -> Context: + ctx = Context( + paths=tmp_path, + config=Config( + gateways={ + "": GatewayConfig(connection=DuckDBConnectionConfig(database=str(db_path))) + }, + model_defaults=ModelDefaultsConfig(dialect="duckdb"), + virtual_environment_mode=mode, + ), + ) + stack.callback(ctx.close) + return ctx + + # 1. Start in DEV_ONLY mode and deploy prod + ctx_dev_only = make_ctx(VirtualEnvironmentMode.DEV_ONLY) + ctx_dev_only.plan("prod", auto_apply=True, no_prompts=True) + assert {o.name: o.type for o in ctx_dev_only.engine_adapter.get_data_objects("db")} == { + "seed_m": DataObjectType.TABLE, + "full_m": DataObjectType.TABLE, + "upstream": DataObjectType.TABLE, + "downstream": DataObjectType.TABLE, + "v_model": DataObjectType.VIEW, + } + ctx_dev_only.close() + + # 2. Switch config to FULL, edit models, and run plan("dev") #1 and plan("dev") #2 + write_downstream(20) + write_view(2) + ctx_full = make_ctx(VirtualEnvironmentMode.FULL) + plan_dev1 = ctx_full.plan_builder("dev").build() + dev1_downstream_snap = plan_dev1.context_diff.snapshots[_snap_id(ctx_full, "db.downstream")] + assert dev1_downstream_snap.is_forward_only is False + dev1_downstream_dev_table = dev1_downstream_snap.table_name(is_deployable=False) + ctx_full.apply(plan_dev1) + + write_downstream(30) + ctx_full.load() + plan_dev2 = ctx_full.plan_builder("dev").build() + assert ( + plan_dev2.context_diff.snapshots[_snap_id(ctx_full, "db.downstream")].is_forward_only + is False + ) + ctx_full.apply(plan_dev2) + + # 3. Apply plan("prod") to migrate prod from DEV_ONLY -> FULL + plan_prod = ctx_full.plan_builder("prod").build() + assert plan_prod.requires_backfill + ctx_full.apply(plan_prod) + + assert {o.name: o.type for o in ctx_full.engine_adapter.get_data_objects("db")} == { + "seed_m": DataObjectType.VIEW, + "full_m": DataObjectType.VIEW, + "upstream": DataObjectType.VIEW, + "downstream": DataObjectType.VIEW, + "v_model": DataObjectType.VIEW, + } + assert ( + len( + [ + o + for o in ctx_full.engine_adapter.get_data_objects("sqlmesh__db") + if not o.name.endswith("__dev") + ] + ) + == 5 + ) + + # 4. Invalidate dev and run delete_expired_snapshots(ignore_ttl=True) + ctx_full.invalidate_environment("dev", sync=True) + failures = delete_expired_snapshots( + ctx_full.state_sync, + ctx_full.snapshot_evaluator, + current_ts=now_timestamp(), + ignore_ttl=True, + ) + assert failures == [] + assert ctx_full.engine_adapter.get_data_object(dev1_downstream_dev_table) is None + assert ( + len( + [ + o + for o in ctx_full.engine_adapter.get_data_objects("sqlmesh__db") + if not o.name.endswith("__dev") + ] + ) + == 5 + ) + assert ctx_full.engine_adapter.fetchall( + "SELECT id, val, tag, ds FROM db.v_model ORDER BY ds" + ) == [ + (1, 30, 2, to_date("2023-01-05")), + (1, 30, 2, to_date("2023-01-06")), + (1, 30, 2, to_date("2023-01-07")), + ] + assert _scalar(ctx_full, "SELECT COUNT(*) FROM db.downstream") == 3 + + +@time_machine.travel("2023-01-08 15:00:00 UTC") +def test_dev_only_view_to_incremental_with_cloning(tmp_path: Path, mocker: MockerFixture) -> None: + mocker.patch("sqlmesh.core.engine_adapter.DuckDBEngineAdapter", CloningDuckDBEngineAdapter) + + models_dir = tmp_path / "models" + models_dir.mkdir() + db_path = tmp_path / "warehouse.db" + + (models_dir / "m.sql").write_text( + """ + MODEL ( + name db.m, + kind VIEW + ); + SELECT 1::INT AS id, DATE '2023-01-05' AS ds; + """, + encoding="utf-8", + ) + + with ExitStack() as stack: + ctx = Context( + paths=tmp_path, + config=Config( + gateways={ + "": GatewayConfig(connection=DuckDBConnectionConfig(database=str(db_path))) + }, + model_defaults=ModelDefaultsConfig(dialect="duckdb"), + virtual_environment_mode=VirtualEnvironmentMode.DEV_ONLY, + ), + ) + stack.callback(ctx.close) + assert isinstance(ctx.engine_adapter, CloningDuckDBEngineAdapter) + + # 1. Deploy db.m as VIEW in prod + ctx.plan("prod", auto_apply=True, no_prompts=True) + assert _obj_type(ctx, "db.m") == DataObjectType.VIEW + + # 2. Change db.m to INCREMENTAL_BY_TIME_RANGE in plan("dev") #1 + (models_dir / "m.sql").write_text( + """ + MODEL ( + name db.m, + kind INCREMENTAL_BY_TIME_RANGE ( + time_column ds, + batch_size 1 + ), + start '2023-01-05' + ); + SELECT 1::INT AS id, @start_ds::DATE AS ds; + """, + encoding="utf-8", + ) + ctx.load() + plan_dev1 = ctx.plan_builder("dev").build() + ctx.apply(plan_dev1) + assert ctx.engine_adapter.cloned_tables == [] + + # 3. Make follow-up SQL edit to db.m in plan("dev") #2 -> must NOT clone prod's VIEW! + (models_dir / "m.sql").write_text( + """ + MODEL ( + name db.m, + kind INCREMENTAL_BY_TIME_RANGE ( + time_column ds, + batch_size 1 + ), + start '2023-01-05' + ); + SELECT 2::INT AS id, @start_ds::DATE AS ds; + """, + encoding="utf-8", + ) + ctx.load() + plan_dev2 = ctx.plan_builder("dev").build() + ctx.apply(plan_dev2) + assert ctx.engine_adapter.cloned_tables == [] + assert plan_dev2.context_diff.snapshots[_snap_id(ctx, "db.m")].is_forward_only is False + + # 4. Deploy to prod -> migrates db.m cleanly from VIEW to TABLE + plan_prod = ctx.plan_builder("prod").build() + assert plan_prod.requires_backfill + ctx.apply(plan_prod) + assert _obj_type(ctx, "db.m") == DataObjectType.TABLE + assert ctx.engine_adapter.fetchall("SELECT id, ds FROM db.m ORDER BY ds") == [ + (2, to_date("2023-01-05")), + (2, to_date("2023-01-06")), + (2, to_date("2023-01-07")), + ] + + +@time_machine.travel("2023-01-08 15:00:00 UTC") +def test_vde_mode_round_trip_dev_only_to_full_to_dev_only_without_janitor( + tmp_path: Path, mocker: MockerFixture +) -> None: + mocker.patch("sqlmesh.core.engine_adapter.DuckDBEngineAdapter", CloningDuckDBEngineAdapter) + + seeds_dir = tmp_path / "seeds" + models_dir = tmp_path / "models" + seeds_dir.mkdir() + models_dir.mkdir() + db_path = tmp_path / "warehouse.db" + + (seeds_dir / "seed_data.csv").write_text("id,label\n1,alpha\n2,beta\n", encoding="utf-8") + (models_dir / "seed_m.sql").write_text( + """ + MODEL ( + name db.seed_m, + kind SEED ( + path '../seeds/seed_data.csv' + ) + ); + """, + encoding="utf-8", + ) + (models_dir / "full_m.sql").write_text( + """ + MODEL ( + name db.full_m, + kind FULL + ); + SELECT id, label FROM db.seed_m; + """, + encoding="utf-8", + ) + (models_dir / "upstream.sql").write_text( + """ + MODEL ( + name db.upstream, + kind INCREMENTAL_BY_TIME_RANGE ( + time_column ds, + batch_size 1 + ), + start '2023-01-05' + ); + SELECT 1::INT AS id, @start_ds::DATE AS ds; + """, + encoding="utf-8", + ) + + def write_downstream(val: int) -> None: + (models_dir / "downstream.sql").write_text( + f""" + MODEL ( + name db.downstream, + kind INCREMENTAL_BY_TIME_RANGE ( + time_column ds, + batch_size 1 + ), + start '2023-01-05' + ); + SELECT id, {val}::INT AS val, ds FROM db.upstream WHERE ds BETWEEN @start_ds AND @end_ds; + """, + encoding="utf-8", + ) + + write_downstream(10) + + with ExitStack() as stack: + + def make_ctx(mode: VirtualEnvironmentMode) -> Context: + ctx = Context( + paths=tmp_path, + config=Config( + gateways={ + "": GatewayConfig(connection=DuckDBConnectionConfig(database=str(db_path))) + }, + model_defaults=ModelDefaultsConfig(dialect="duckdb"), + virtual_environment_mode=mode, + ), + ) + stack.callback(ctx.close) + return ctx + + # 1. Deploy prod in DEV_ONLY mode -> populates version='novde' intervals in _intervals + ctx_dev_only_1 = make_ctx(VirtualEnvironmentMode.DEV_ONLY) + ctx_dev_only_1.plan("prod", auto_apply=True, no_prompts=True) + assert _scalar(ctx_dev_only_1, "SELECT COUNT(*) FROM db.upstream") == 3 + assert _scalar(ctx_dev_only_1, "SELECT COUNT(*) FROM db.downstream") == 3 + assert _scalar(ctx_dev_only_1, "SELECT COUNT(*) FROM db.full_m") == 2 + ctx_dev_only_1.close() + + # 2. Migrate prod to FULL WITHOUT running Janitor -> drops physical db.* tables & creates VIEWs + ctx_full_1 = make_ctx(VirtualEnvironmentMode.FULL) + ctx_full_1.plan("prod", auto_apply=True, no_prompts=True) + assert {o.name: o.type for o in ctx_full_1.engine_adapter.get_data_objects("db")} == { + "seed_m": DataObjectType.VIEW, + "full_m": DataObjectType.VIEW, + "upstream": DataObjectType.VIEW, + "downstream": DataObjectType.VIEW, + } + ctx_full_1.close() + + # 3. Migrate back to DEV_ONLY: + # - Edit downstream SQL (new snapshot s_3, previewed in dev first so s_3 is in _snapshots) + # - Leave upstream, full_m, and seed_m unedited (reverts to original s_1 snapshot IDs!) + write_downstream(99) + ctx_dev_only_2 = make_ctx(VirtualEnvironmentMode.DEV_ONLY) + plan_dev = ctx_dev_only_2.plan_builder("dev").build() + ctx_dev_only_2.apply(plan_dev) + + plan_prod = ctx_dev_only_2.plan_builder("prod").build() + assert plan_prod.requires_backfill + ctx_dev_only_2.apply(plan_prod) + + assert {o.name: o.type for o in ctx_dev_only_2.engine_adapter.get_data_objects("db")} == { + "seed_m": DataObjectType.TABLE, + "full_m": DataObjectType.TABLE, + "upstream": DataObjectType.TABLE, + "downstream": DataObjectType.TABLE, + } + assert _scalar(ctx_dev_only_2, "SELECT COUNT(*) FROM db.seed_m") == 2 + assert _scalar(ctx_dev_only_2, "SELECT COUNT(*) FROM db.full_m") == 2 + assert _scalar(ctx_dev_only_2, "SELECT COUNT(*) FROM db.upstream") == 3 + assert ctx_dev_only_2.engine_adapter.fetchall( + "SELECT id, val, ds FROM db.downstream ORDER BY ds" + ) == [ + (1, 99, to_date("2023-01-05")), + (1, 99, to_date("2023-01-06")), + (1, 99, to_date("2023-01-07")), + ] + ctx_dev_only_2.close() + + # 4. Migrate prod to FULL again, then migrate back to DEV_ONLY with skip_backfill=True + # followed by ctx.run() -> stale 'novde' intervals must be purged so run() backfills! + ctx_full_2 = make_ctx(VirtualEnvironmentMode.FULL) + ctx_full_2.plan("prod", auto_apply=True, no_prompts=True) + ctx_full_2.close() + + ctx_dev_only_3 = make_ctx(VirtualEnvironmentMode.DEV_ONLY) + plan_skip_backfill = ctx_dev_only_3.plan_builder("prod", skip_backfill=True).build() + ctx_dev_only_3.apply(plan_skip_backfill) + assert _scalar(ctx_dev_only_3, "SELECT COUNT(*) FROM db.upstream") == 0 + assert _scalar(ctx_dev_only_3, "SELECT COUNT(*) FROM db.downstream") == 0 + + ctx_dev_only_3.run(skip_janitor=True) + assert _scalar(ctx_dev_only_3, "SELECT COUNT(*) FROM db.full_m") == 2 + assert _scalar(ctx_dev_only_3, "SELECT COUNT(*) FROM db.upstream") == 3 + assert _scalar(ctx_dev_only_3, "SELECT COUNT(*) FROM db.downstream") == 3 + + +@time_machine.travel("2023-01-08 15:00:00 UTC") +def test_destructive_change_checks_with_vde_mode_transitions_and_per_model_overrides( + tmp_path: Path, +) -> None: + with ExitStack() as stack: + + def make_ctx(proj_dir: Path, db_file: Path, mode: VirtualEnvironmentMode) -> Context: + ctx = Context( + paths=proj_dir, + config=Config( + gateways={ + "": GatewayConfig(connection=DuckDBConnectionConfig(database=str(db_file))) + }, + model_defaults=ModelDefaultsConfig(dialect="duckdb"), + virtual_environment_mode=mode, + ), + ) + stack.callback(ctx.close) + return ctx + + # 1. FULL -> DEV_ONLY transition while dropping a column on an INCREMENTAL model with + # on_destructive_change=error is a full rebuild (not forward-only) -> must NOT raise PlanError! + proj_transition = tmp_path / "proj_transition" + models_transition = proj_transition / "models" + models_transition.mkdir(parents=True) + db_transition = proj_transition / "warehouse.db" + + (models_transition / "inc_m.sql").write_text( + """ + MODEL ( + name db.inc_m, + kind INCREMENTAL_BY_TIME_RANGE ( + time_column ds, + batch_size 1, + on_destructive_change error + ), + start '2023-01-05' + ); + SELECT 1::INT AS id, 2::INT AS dropped_col, @start_ds::DATE AS ds; + """, + encoding="utf-8", + ) + + ctx_full = make_ctx(proj_transition, db_transition, VirtualEnvironmentMode.FULL) + ctx_full.plan("prod", auto_apply=True, no_prompts=True) + ctx_full.close() + + (models_transition / "inc_m.sql").write_text( + """ + MODEL ( + name db.inc_m, + kind INCREMENTAL_BY_TIME_RANGE ( + time_column ds, + batch_size 1, + on_destructive_change error + ), + start '2023-01-05' + ); + SELECT 1::INT AS id, @start_ds::DATE AS ds; + """, + encoding="utf-8", + ) + + ctx_dev_only = make_ctx(proj_transition, db_transition, VirtualEnvironmentMode.DEV_ONLY) + plan_dev = ctx_dev_only.plan_builder("dev").build() + ctx_dev_only.apply(plan_dev) + plan_prod = ctx_dev_only.plan_builder("prod").build() + ctx_dev_only.apply(plan_prod) + assert list(ctx_dev_only.engine_adapter.columns("db.inc_m")) == ["id", "ds"] + ctx_dev_only.close() + + # 2. Per-model DEV_ONLY incremental model inside a FULL project MUST enforce on_destructive_change=error! + proj_per_model = tmp_path / "proj_per_model" + models_per_model = proj_per_model / "models" + models_per_model.mkdir(parents=True) + db_per_model = proj_per_model / "warehouse.db" + + (models_per_model / "per_model_dev_only.sql").write_text( + """ + MODEL ( + name db.per_model_dev_only, + kind INCREMENTAL_BY_TIME_RANGE ( + time_column ds, + batch_size 1, + on_destructive_change error + ), + virtual_environment_mode 'dev_only', + start '2023-01-05' + ); + SELECT 1::INT AS id, 2::INT AS dropped_col, @start_ds::DATE AS ds; + """, + encoding="utf-8", + ) + + ctx_mixed = make_ctx(proj_per_model, db_per_model, VirtualEnvironmentMode.FULL) + ctx_mixed.plan("prod", auto_apply=True, no_prompts=True) + + (models_per_model / "per_model_dev_only.sql").write_text( + """ + MODEL ( + name db.per_model_dev_only, + kind INCREMENTAL_BY_TIME_RANGE ( + time_column ds, + batch_size 1, + on_destructive_change error + ), + virtual_environment_mode 'dev_only', + start '2023-01-05' + ); + SELECT 1::INT AS id, @start_ds::DATE AS ds; + """, + encoding="utf-8", + ) + ctx_mixed.load() + + with pytest.raises( + PlanError, match="Plan requires a destructive change to a forward-only model" + ): + ctx_mixed.plan_builder("dev").build() + + with pytest.raises( + PlanError, match="Plan requires a destructive change to a forward-only model" + ): + ctx_mixed.plan_builder("prod").build() + + +@time_machine.travel("2023-01-08 15:00:00 UTC") +def test_dev_only_prod_kind_change_classification_is_stable_across_dev_plans( + init_and_plan_context: t.Callable, +): + context, _ = init_and_plan_context( + "examples/sushi", config="test_config_virtual_environment_mode_dev_only" + ) + + model_name = "memory.sushi.test_kind_change" + unmanaged = load_sql_based_model( + d.parse( + f""" + MODEL (name {model_name}, kind INCREMENTAL_UNMANAGED); + SELECT 1 AS id, 'a' AS value; + """ + ) + ).copy(update={"virtual_environment_mode": VirtualEnvironmentMode.DEV_ONLY}) + context.upsert_model(unmanaged) + context.plan("prod", auto_apply=True, no_prompts=True) + + unique_key = load_sql_based_model( + d.parse( + f""" + MODEL (name {model_name}, kind INCREMENTAL_BY_UNIQUE_KEY (unique_key id)); + SELECT 1 AS id, 'a' AS value; + """ + ) + ).copy(update={"virtual_environment_mode": VirtualEnvironmentMode.DEV_ONLY}) + context.upsert_model(unique_key) + + # First dev plan: the baseline is the prod snapshot. + plan_1 = context.plan_builder("dev", skip_tests=True).build() + snapshot_1 = plan_1.context_diff.snapshots[context.get_snapshot(model_name).snapshot_id] + assert not snapshot_1.is_forward_only + context.apply(plan_1) + + # Follow-up dev plan: the baseline is the earlier dev snapshot, but prod is still the unmanaged version. + follow_up = add_projection_to_model(t.cast(SqlModel, unique_key)) + context.upsert_model(follow_up) + plan_2 = context.plan_builder("dev", skip_tests=True).build() + snapshot_2 = plan_2.context_diff.snapshots[context.get_snapshot(model_name).snapshot_id] + assert not snapshot_2.is_forward_only + assert snapshot_2.change_category == snapshot_1.change_category + assert not snapshot_2.intervals + + +@time_machine.travel("2023-01-08 15:00:00 UTC") +def test_dev_restatement_guard_not_disabled_by_unrelated_dev_only_change( + init_and_plan_context: t.Callable, mocker: MockerFixture +): + context, _ = init_and_plan_context("examples/sushi") + context.plan("prod", auto_apply=True, no_prompts=True) + + # In an otherwise FULL project, modify one model as DEV_ONLY locally and explicitly + # restate an unrelated model that is live in prod. + modified = add_projection_to_model( + t.cast(SqlModel, context.get_model("sushi.waiter_revenue_by_day")) + ).copy(update={"virtual_environment_mode": VirtualEnvironmentMode.DEV_ONLY}) + context.upsert_model(modified) + log_warning = mocker.patch.object(context.console, "log_warning") + + plan = context.plan_builder( + "dev", + skip_tests=True, + restate_models=["sushi.customer_revenue_lifetime"], + always_include_local_changes=True, + ).build() + + restated_names = {s_id.name for s_id in plan.restatements} + assert '"memory"."sushi"."customer_revenue_lifetime"' not in restated_names + assert any( + "used in production" in call.args[0] and "customer_revenue_lifetime" in call.args[0] + for call in log_warning.call_args_list + ) + + +@time_machine.travel("2023-01-08 15:00:00 UTC") +def test_dev_only_seed_metadata_only_change_keeps_prod_intervals( + init_and_plan_context: t.Callable, +): + context, _ = init_and_plan_context( + "examples/sushi", config="test_config_virtual_environment_mode_dev_only" + ) + context.plan("prod", auto_apply=True, no_prompts=True) + + seed_name = "sushi.waiter_names" + old_snapshot = context.get_snapshot(seed_name) + seed_model = context.get_model(seed_name) + context.upsert_model(seed_model.copy(update={"description": "Updated seed description"})) + + plan = context.plan_builder("prod", skip_tests=True).build() + new_snapshot = plan.context_diff.snapshots[context.get_snapshot(seed_name).snapshot_id] + assert new_snapshot.snapshot_id != old_snapshot.snapshot_id + assert new_snapshot.is_metadata + assert not plan.missing_intervals + context.apply(plan) + + # The already loaded seed data must not be marked as missing after the metadata-only change. + refreshed = context.state_sync.get_snapshots([new_snapshot.snapshot_id])[ + new_snapshot.snapshot_id + ] + context.state_sync.refresh_snapshot_intervals([refreshed]) + assert refreshed.intervals + assert not context.plan_builder("prod", skip_tests=True).build().missing_intervals + + +@time_machine.travel("2023-01-08 15:00:00 UTC") +def test_dev_only_seed_metadata_only_change_dev_then_prod_keeps_prod_intervals( + init_and_plan_context: t.Callable, +): + context, _ = init_and_plan_context( + "examples/sushi", config="test_config_virtual_environment_mode_dev_only" + ) + context.plan("prod", auto_apply=True, no_prompts=True) + + seed_name = "sushi.waiter_names" + seed_model = context.get_model(seed_name) + context.upsert_model(seed_model.copy(update={"description": "Updated seed description"})) + + # The snapshot is created by the dev plan, so the prod plan doesn't see it as new. + context.plan("dev", auto_apply=True, no_prompts=True, skip_tests=True) + new_snapshot_id = context.get_snapshot(seed_name).snapshot_id + + plan = context.plan_builder("prod", skip_tests=True).build() + assert new_snapshot_id not in {s.snapshot_id for s in plan.new_snapshots} + assert not plan.missing_intervals + context.apply(plan) + + refreshed = context.state_sync.get_snapshots([new_snapshot_id])[new_snapshot_id] + context.state_sync.refresh_snapshot_intervals([refreshed]) + assert refreshed.intervals + assert not context.plan_builder("prod", skip_tests=True).build().missing_intervals diff --git a/tests/core/test_model.py b/tests/core/test_model.py index c5d1a71bee..f0b22b0274 100644 --- a/tests/core/test_model.py +++ b/tests/core/test_model.py @@ -13630,3 +13630,405 @@ def test_blueprint_catalog_not_cross_contaminated(): assert db_model.catalog == "example_catalog", ( f"Catalog lost for DuckDB blueprint after ClickHouse iteration. Got: {db_model.catalog}" ) + + +def test_virtual_environment_mode_model_ddl_and_defaults_precedence(tmp_path: Path) -> None: + from sqlmesh.core.config import PlanConfig + from sqlmesh.core.config.common import VirtualEnvironmentMode + + # 1. SQL MODEL DDL: string literal, identifier, and case-insensitive values + for raw_mode, expected_mode in ( + ("'dev_only'", VirtualEnvironmentMode.DEV_ONLY), + ("dev_only", VirtualEnvironmentMode.DEV_ONLY), + ("'DEV_ONLY'", VirtualEnvironmentMode.DEV_ONLY), + ("DEV_ONLY", VirtualEnvironmentMode.DEV_ONLY), + ("'full'", VirtualEnvironmentMode.FULL), + ("full", VirtualEnvironmentMode.FULL), + ): + sql_model = load_sql_based_model( + d.parse( + f"MODEL (name db.m, kind FULL, virtual_environment_mode {raw_mode}); SELECT 1 AS a" + ) + ) + assert sql_model.virtual_environment_mode == expected_mode + + # Round-trip fidelity via render_definition(include_defaults=True) + reloaded = load_sql_based_model(sql_model.render_definition(include_defaults=True)) + assert reloaded.virtual_environment_mode == expected_mode + + # Dialect fallback in _Node._name_validator when exp.Table lacks meta["sql"] (e.g. BigQuery backticks) + previous_model_dialect = model._dialect + try: + model._dialect = "bigquery" + bq_model = SqlModel( + name=exp.to_table("`my-proj.my_ds.my_tbl`", dialect="bigquery"), + kind="FULL", + dialect="bigquery", + virtual_environment_mode="dev_only", + query=parse_one("SELECT 1 AS a"), + ) + assert bq_model.name == "`my-proj.my_ds.my_tbl`" + assert bq_model.fqn == '"my-proj"."my_ds"."my_tbl"' + bq_reloaded = load_sql_based_model(bq_model.render_definition(include_defaults=True)) + assert bq_reloaded.name == bq_model.name + assert bq_reloaded.fqn == bq_model.fqn + assert bq_reloaded.virtual_environment_mode == VirtualEnvironmentMode.DEV_ONLY + finally: + model._dialect = previous_model_dialect + + # 2. Python @model decorator + from sqlmesh.utils import UniqueKeyDict + + previous_registry = model.get_registry() + try: + model.set_registry(UniqueKeyDict("models")) + + @model("db.py_m", kind="FULL", columns={"a": "int"}, virtual_environment_mode="dev_only") + def py_m(context: ExecutionContext, **kwargs: t.Any) -> pd.DataFrame: + return pd.DataFrame([{"a": 1}]) + + py_model = model.get_registry()["db.py_m"].model( + module_path=Path("."), + path=Path("."), + ) + assert py_model.virtual_environment_mode == VirtualEnvironmentMode.DEV_ONLY + finally: + model.set_registry(previous_registry) + + # 3. ModelDefaultsConfig accepts both string and VirtualEnvironmentMode enum + assert ModelDefaultsConfig().virtual_environment_mode is None + assert ( + ModelDefaultsConfig(virtual_environment_mode="dev_only").virtual_environment_mode + == VirtualEnvironmentMode.DEV_ONLY + ) + assert ( + ModelDefaultsConfig( + virtual_environment_mode=VirtualEnvironmentMode.DEV_ONLY + ).virtual_environment_mode + == VirtualEnvironmentMode.DEV_ONLY + ) + assert ( + create_external_model( + "source_schema.default_source", + defaults=ModelDefaultsConfig(virtual_environment_mode="dev_only").dict(), + ).virtual_environment_mode + == VirtualEnvironmentMode.FULL + ) + + # 4. Config validation: use_finalized_state=True + model_defaults.virtual_environment_mode=DEV_ONLY raises ConfigError + with pytest.raises(ConfigError, match="virtual_environment_mode"): + Config( + plan=PlanConfig(use_finalized_state=True), + model_defaults=ModelDefaultsConfig( + virtual_environment_mode=VirtualEnvironmentMode.DEV_ONLY + ), + ) + + # 5. Full 3-tier precedence in Context: model-level > model_defaults > root config + models_dir = tmp_path / "models" + models_dir.mkdir(parents=True, exist_ok=True) + (models_dir / "sql_default.sql").write_text( + "MODEL (name db.sql_default, kind FULL); SELECT 1 AS a;\n", + encoding="utf-8", + ) + (models_dir / "sql_full.sql").write_text( + "MODEL (name db.sql_full, kind FULL, virtual_environment_mode 'full'); SELECT 1 AS a;\n", + encoding="utf-8", + ) + (models_dir / "sql_dev_only.sql").write_text( + "MODEL (name db.sql_dev_only, kind FULL, virtual_environment_mode dev_only); SELECT 1 AS a;\n", + encoding="utf-8", + ) + (models_dir / "py_models.py").write_text( + """import typing as t +import pandas as pd +from sqlmesh import ExecutionContext, model + + +@model("db.py_default", kind="FULL", columns={"a": "int"}) +def py_default(context: ExecutionContext, **kwargs: t.Any) -> pd.DataFrame: + return pd.DataFrame([{"a": 1}]) + + +@model("db.py_full", kind="FULL", columns={"a": "int"}, virtual_environment_mode="full") +def py_full(context: ExecutionContext, **kwargs: t.Any) -> pd.DataFrame: + return pd.DataFrame([{"a": 1}]) + + +@model("db.py_dev_only", kind="FULL", columns={"a": "int"}, virtual_environment_mode="dev_only") +def py_dev_only(context: ExecutionContext, **kwargs: t.Any) -> pd.DataFrame: + return pd.DataFrame([{"a": 1}]) +""", + encoding="utf-8", + ) + + # Root FULL + model_defaults DEV_ONLY -> default models get DEV_ONLY, explicit 'full' keeps FULL + ctx_defaults_dev_only = Context( + paths=tmp_path, + config=Config( + gateways={"": GatewayConfig(connection=DuckDBConnectionConfig())}, + virtual_environment_mode=VirtualEnvironmentMode.FULL, + model_defaults=ModelDefaultsConfig( + dialect="duckdb", + virtual_environment_mode=VirtualEnvironmentMode.DEV_ONLY, + ), + ), + ) + assert ( + ctx_defaults_dev_only.get_model("db.sql_default").virtual_environment_mode + == VirtualEnvironmentMode.DEV_ONLY + ) + assert ( + ctx_defaults_dev_only.get_model("db.py_default").virtual_environment_mode + == VirtualEnvironmentMode.DEV_ONLY + ) + assert ( + ctx_defaults_dev_only.get_model("db.sql_full").virtual_environment_mode + == VirtualEnvironmentMode.FULL + ) + assert ( + ctx_defaults_dev_only.get_model("db.py_full").virtual_environment_mode + == VirtualEnvironmentMode.FULL + ) + assert ( + ctx_defaults_dev_only.get_model("db.sql_dev_only").virtual_environment_mode + == VirtualEnvironmentMode.DEV_ONLY + ) + assert ( + ctx_defaults_dev_only.get_model("db.py_dev_only").virtual_environment_mode + == VirtualEnvironmentMode.DEV_ONLY + ) + ctx_defaults_dev_only.close() + + # Root DEV_ONLY + model_defaults FULL -> default models get FULL, explicit dev_only keeps DEV_ONLY + ctx_defaults_full = Context( + paths=tmp_path, + config=Config( + gateways={"": GatewayConfig(connection=DuckDBConnectionConfig())}, + virtual_environment_mode=VirtualEnvironmentMode.DEV_ONLY, + model_defaults=ModelDefaultsConfig( + dialect="duckdb", + virtual_environment_mode=VirtualEnvironmentMode.FULL, + ), + ), + ) + assert ( + ctx_defaults_full.get_model("db.sql_default").virtual_environment_mode + == VirtualEnvironmentMode.FULL + ) + assert ( + ctx_defaults_full.get_model("db.py_default").virtual_environment_mode + == VirtualEnvironmentMode.FULL + ) + assert ( + ctx_defaults_full.get_model("db.sql_dev_only").virtual_environment_mode + == VirtualEnvironmentMode.DEV_ONLY + ) + assert ( + ctx_defaults_full.get_model("db.py_dev_only").virtual_environment_mode + == VirtualEnvironmentMode.DEV_ONLY + ) + ctx_defaults_full.close() + + # Root DEV_ONLY + unset model_defaults -> default models inherit root DEV_ONLY, explicit 'full' keeps FULL + ctx_root_dev_only = Context( + paths=tmp_path, + config=Config( + gateways={"": GatewayConfig(connection=DuckDBConnectionConfig())}, + virtual_environment_mode=VirtualEnvironmentMode.DEV_ONLY, + model_defaults=ModelDefaultsConfig(dialect="duckdb"), + ), + ) + assert ( + ctx_root_dev_only.get_model("db.sql_default").virtual_environment_mode + == VirtualEnvironmentMode.DEV_ONLY + ) + assert ( + ctx_root_dev_only.get_model("db.py_default").virtual_environment_mode + == VirtualEnvironmentMode.DEV_ONLY + ) + assert ( + ctx_root_dev_only.get_model("db.sql_full").virtual_environment_mode + == VirtualEnvironmentMode.FULL + ) + assert ( + ctx_root_dev_only.get_model("db.py_full").virtual_environment_mode + == VirtualEnvironmentMode.FULL + ) + ctx_root_dev_only.close() + + +def test_per_model_dev_only_with_use_finalized_state_raises_config_error(tmp_path: Path) -> None: + from contextlib import ExitStack + from sqlmesh.core.config import PlanConfig + from sqlmesh.core.config.common import VirtualEnvironmentMode + + expected_msg = "Using the finalized state is only supported when `virtual_environment_mode` is set to `full`." + + with ExitStack() as stack: + # 1. Per-model virtual_environment_mode 'dev_only' in MODEL (...) raises ConfigError on Context load + proj_bad = tmp_path / "proj_bad" + (proj_bad / "models").mkdir(parents=True) + (proj_bad / "models" / "m.sql").write_text( + "MODEL (name db.m, kind FULL, virtual_environment_mode 'dev_only'); SELECT 1 AS a;\n", + encoding="utf-8", + ) + + finalized_cfg = Config( + gateways={"": GatewayConfig(connection=DuckDBConnectionConfig())}, + virtual_environment_mode=VirtualEnvironmentMode.FULL, + model_defaults=ModelDefaultsConfig(dialect="duckdb"), + plan=PlanConfig(use_finalized_state=True), + ) + + with pytest.raises(ConfigError, match=expected_msg): + ctx_bad = Context(paths=proj_bad, config=finalized_cfg) + stack.callback(ctx_bad.close) + + ctx_no_load = Context(paths=proj_bad, config=finalized_cfg, load=False) + stack.callback(ctx_no_load.close) + with pytest.raises(ConfigError, match=expected_msg): + ctx_no_load.load(update_schemas=False) + ctx_no_load.close() + + # 2. Symbolic (EMBEDDED) model does NOT fail, while upsert_model(..., virtual_environment_mode=DEV_ONLY) DOES fail + proj_ok = tmp_path / "proj_ok" + (proj_ok / "models").mkdir(parents=True) + (proj_ok / "models" / "embedded_m.sql").write_text( + "MODEL (name db.embedded_m, kind EMBEDDED, virtual_environment_mode 'dev_only'); SELECT 1 AS a;\n", + encoding="utf-8", + ) + (proj_ok / "models" / "full_m.sql").write_text( + "MODEL (name db.full_m, kind FULL); SELECT a FROM db.embedded_m;\n", + encoding="utf-8", + ) + + ctx_ok = Context(paths=proj_ok, config=finalized_cfg) + stack.callback(ctx_ok.close) + + with pytest.raises(ConfigError, match=expected_msg): + ctx_ok.upsert_model( + "db.full_m", virtual_environment_mode=VirtualEnvironmentMode.DEV_ONLY + ) + + +def test_upsert_model_normalizes_string_virtual_environment_mode(tmp_path: Path) -> None: + from sqlmesh.core.config import PlanConfig + from sqlmesh.core.config.common import VirtualEnvironmentMode + + proj = tmp_path / "proj" + (proj / "models").mkdir(parents=True) + (proj / "models" / "m.sql").write_text( + "MODEL (name db.m, kind FULL); SELECT 1 AS a;\n", encoding="utf-8" + ) + + ctx = Context( + paths=proj, + config=Config( + gateways={"": GatewayConfig(connection=DuckDBConnectionConfig())}, + model_defaults=ModelDefaultsConfig(dialect="duckdb"), + plan=PlanConfig(use_finalized_state=True), + ), + ) + try: + # A plain string must be normalized to the enum instead of failing with AttributeError + updated = ctx.upsert_model("db.m", virtual_environment_mode="full") + assert updated.virtual_environment_mode == VirtualEnvironmentMode.FULL + + updated = ctx.upsert_model("db.m", virtual_environment_mode=VirtualEnvironmentMode.FULL) + assert updated.virtual_environment_mode == VirtualEnvironmentMode.FULL + + # dev_only is rejected with ConfigError, in both string and enum form, without mutating state + for dev_only in ("dev_only", "DEV_ONLY", VirtualEnvironmentMode.DEV_ONLY): + with pytest.raises(ConfigError, match="finalized state"): + ctx.upsert_model("db.m", virtual_environment_mode=dev_only) + assert ctx.get_model("db.m").virtual_environment_mode == VirtualEnvironmentMode.FULL + finally: + ctx.close() + + +def test_virtual_environment_mode_none_default_falls_back_to_explicit_mode() -> None: + from sqlmesh.core.config.common import VirtualEnvironmentMode + + # defaults with an explicit None must not shadow the mode passed alongside them + sql_model = load_sql_based_model( + d.parse("MODEL (name db.m, kind FULL); SELECT 1 AS a"), + defaults={"virtual_environment_mode": None}, + virtual_environment_mode=VirtualEnvironmentMode.DEV_ONLY, + ) + assert sql_model.virtual_environment_mode == VirtualEnvironmentMode.DEV_ONLY + + # a concrete default still wins over the fallback + sql_model = load_sql_based_model( + d.parse("MODEL (name db.m, kind FULL); SELECT 1 AS a"), + defaults={"virtual_environment_mode": "full"}, + virtual_environment_mode=VirtualEnvironmentMode.DEV_ONLY, + ) + assert sql_model.virtual_environment_mode == VirtualEnvironmentMode.FULL + + from sqlmesh.utils import UniqueKeyDict + + previous_registry = model.get_registry() + try: + model.set_registry(UniqueKeyDict("models")) + + @model("db.py_m1", kind="FULL", columns={"a": "int"}) + def py_m1(context: ExecutionContext, **kwargs: t.Any) -> pd.DataFrame: + return pd.DataFrame([{"a": 1}]) + + py_model = model.get_registry()["db.py_m1"].model( + module_path=Path("."), + path=Path("."), + defaults={"virtual_environment_mode": None}, + virtual_environment_mode=VirtualEnvironmentMode.DEV_ONLY, + ) + assert py_model.virtual_environment_mode == VirtualEnvironmentMode.DEV_ONLY + + py_model = model.get_registry()["db.py_m1"].model( + module_path=Path("."), + path=Path("."), + defaults={"virtual_environment_mode": "full"}, + virtual_environment_mode=VirtualEnvironmentMode.DEV_ONLY, + ) + assert py_model.virtual_environment_mode == VirtualEnvironmentMode.FULL + finally: + model.set_registry(previous_registry) + + +def test_finalized_state_check_uses_root_flag_in_multi_project_context(tmp_path: Path) -> None: + from sqlmesh.core.config import PlanConfig + from sqlmesh.core.config.common import VirtualEnvironmentMode + + root_proj = tmp_path / "root" + child_proj = tmp_path / "child" + for proj in (root_proj, child_proj): + (proj / "models").mkdir(parents=True) + (root_proj / "models" / "a.sql").write_text( + "MODEL (name db.a, kind FULL); SELECT 1 AS a;\n", encoding="utf-8" + ) + (child_proj / "models" / "b.sql").write_text( + "MODEL (name db.b, kind FULL, virtual_environment_mode dev_only); SELECT 1 AS b;\n", + encoding="utf-8", + ) + + gateways = {"": GatewayConfig(connection=DuckDBConnectionConfig())} + defaults = ModelDefaultsConfig(dialect="duckdb") + # The plan reads the root config's flag for every model, including the child project's. + root_config = Config( + gateways=gateways, + model_defaults=defaults, + plan=PlanConfig(use_finalized_state=True), + ) + child_config = Config( + gateways=gateways, + model_defaults=defaults, + virtual_environment_mode=VirtualEnvironmentMode.FULL, + plan=PlanConfig(use_finalized_state=False), + ) + + with pytest.raises(ConfigError, match="finalized state"): + ctx = Context( + paths=[root_proj, child_proj], + config={root_proj: root_config, child_proj: child_config}, # type: ignore + ) + ctx.close() diff --git a/tests/core/test_plan.py b/tests/core/test_plan.py index add85d2eec..5c930c6488 100644 --- a/tests/core/test_plan.py +++ b/tests/core/test_plan.py @@ -1,6 +1,8 @@ import logging import typing as t +from contextlib import ExitStack from datetime import timedelta +from pathlib import Path from unittest.mock import patch import pytest @@ -4510,3 +4512,180 @@ def test_forward_only_indirect_change_to_materialized_view(make_snapshot): # Forward-only indirect changes to MVs should not always be classified as indirect breaking. # Instead, we want to preserve the standard categorization. assert snapshot_b_new.change_category == SnapshotChangeCategory.INDIRECT_NON_BREAKING + + +@time_machine.travel("2023-01-08 15:00:00 UTC") +def test_restate_prod_uses_state_virtual_environment_mode(tmp_path: Path) -> None: + from sqlmesh.core.config import ( + Config, + DuckDBConnectionConfig, + GatewayConfig, + ModelDefaultsConfig, + ) + from sqlmesh.core.config.common import VirtualEnvironmentMode + + model_sql = """ + MODEL ( + name db.m, + kind INCREMENTAL_BY_TIME_RANGE ( + time_column ds + ), + start '2023-01-05' + ); + SELECT 1 AS id, @start_ds AS ds; + """ + + def make_config(db_file: Path, mode: VirtualEnvironmentMode) -> Config: + return Config( + gateways={"": GatewayConfig(connection=DuckDBConnectionConfig(database=str(db_file)))}, + model_defaults=ModelDefaultsConfig(dialect="duckdb"), + virtual_environment_mode=mode, + ) + + with ExitStack() as stack: + # 1. Deploy prod in DEV_ONLY mode, then restate prod with local FULL config -> restate_all_snapshots is False + proj_dev_only = tmp_path / "proj_dev_only" + (proj_dev_only / "models").mkdir(parents=True) + (proj_dev_only / "models" / "m.sql").write_text(model_sql, encoding="utf-8") + db_dev_only = proj_dev_only / "warehouse.db" + + ctx_deploy_dev_only = Context( + paths=proj_dev_only, + config=make_config(db_dev_only, VirtualEnvironmentMode.DEV_ONLY), + ) + stack.callback(ctx_deploy_dev_only.close) + ctx_deploy_dev_only.plan("prod", auto_apply=True, no_prompts=True) + ctx_deploy_dev_only.close() + + ctx_restate_with_full_cfg = Context( + paths=proj_dev_only, + config=make_config(db_dev_only, VirtualEnvironmentMode.FULL), + ) + stack.callback(ctx_restate_with_full_cfg.close) + plan_from_dev_only_state = ctx_restate_with_full_cfg.plan_builder( + "prod", restate_models=["db.m"] + ).build() + assert plan_from_dev_only_state.to_evaluatable().restate_all_snapshots is False + ctx_restate_with_full_cfg.close() + + # 2. Deploy prod in FULL mode, then restate prod with local DEV_ONLY config -> restate_all_snapshots is True + proj_full = tmp_path / "proj_full" + (proj_full / "models").mkdir(parents=True) + (proj_full / "models" / "m.sql").write_text(model_sql, encoding="utf-8") + db_full = proj_full / "warehouse.db" + + ctx_deploy_full = Context( + paths=proj_full, + config=make_config(db_full, VirtualEnvironmentMode.FULL), + ) + stack.callback(ctx_deploy_full.close) + ctx_deploy_full.plan("prod", auto_apply=True, no_prompts=True) + ctx_deploy_full.close() + + ctx_restate_with_dev_only_cfg = Context( + paths=proj_full, + config=make_config(db_full, VirtualEnvironmentMode.DEV_ONLY), + ) + stack.callback(ctx_restate_with_dev_only_cfg.close) + plan_from_full_state = ctx_restate_with_dev_only_cfg.plan_builder( + "prod", restate_models=["db.m"] + ).build() + assert plan_from_full_state.to_evaluatable().restate_all_snapshots is True + ctx_restate_with_dev_only_cfg.close() + + +@time_machine.travel("2023-01-08 15:00:00 UTC") +def test_dev_plan_warns_on_virtual_environment_mode_mismatch_with_prod( + tmp_path: Path, mocker: MockerFixture +) -> None: + from sqlmesh.core.config import ( + Config, + DuckDBConnectionConfig, + GatewayConfig, + ModelDefaultsConfig, + ) + from sqlmesh.core.config.common import VirtualEnvironmentMode + + model_sql = "MODEL (name db.m, kind FULL); SELECT 1 AS a;\n" + + def make_config(db_file: Path, mode: VirtualEnvironmentMode) -> Config: + return Config( + gateways={"": GatewayConfig(connection=DuckDBConnectionConfig(database=str(db_file)))}, + model_defaults=ModelDefaultsConfig(dialect="duckdb"), + virtual_environment_mode=mode, + ) + + with ExitStack() as stack: + # 1. Prod deployed in FULL mode + proj_full = tmp_path / "proj_full" + (proj_full / "models").mkdir(parents=True) + (proj_full / "models" / "m.sql").write_text(model_sql, encoding="utf-8") + db_full = proj_full / "warehouse.db" + + ctx_full = Context( + paths=proj_full, + config=make_config(db_full, VirtualEnvironmentMode.FULL), + ) + stack.callback(ctx_full.close) + ctx_full.plan("prod", auto_apply=True, no_prompts=True) + + # Matching-mode dev plan does NOT warn + matching_warn = mocker.patch.object(ctx_full.console, "log_warning") + ctx_full.plan_builder("dev") + matching_warn.assert_not_called() + ctx_full.close() + + # Switch local config to DEV_ONLY -> dev plan warns once, prod plan does NOT warn + ctx_local_dev_only = Context( + paths=proj_full, + config=make_config(db_full, VirtualEnvironmentMode.DEV_ONLY), + ) + stack.callback(ctx_local_dev_only.close) + mismatch_warn_1 = mocker.patch.object(ctx_local_dev_only.console, "log_warning") + ctx_local_dev_only.plan_builder("dev") + mismatch_warn_1.assert_called_once() + msg_1 = mismatch_warn_1.call_args[0][0] + assert "virtual_environment_mode" in msg_1 + assert "'dev_only'" in msg_1 + assert "'full'" in msg_1 + + mismatch_warn_1.reset_mock() + ctx_local_dev_only.plan_builder("prod") + mismatch_warn_1.assert_not_called() + ctx_local_dev_only.close() + + # 2. Reverse: prod deployed in DEV_ONLY mode, local config switched to FULL + proj_dev_only = tmp_path / "proj_dev_only" + (proj_dev_only / "models").mkdir(parents=True) + (proj_dev_only / "models" / "m.sql").write_text(model_sql, encoding="utf-8") + db_dev_only = proj_dev_only / "warehouse.db" + + ctx_dev_only = Context( + paths=proj_dev_only, + config=make_config(db_dev_only, VirtualEnvironmentMode.DEV_ONLY), + ) + stack.callback(ctx_dev_only.close) + ctx_dev_only.plan("prod", auto_apply=True, no_prompts=True) + + matching_warn_2 = mocker.patch.object(ctx_dev_only.console, "log_warning") + ctx_dev_only.plan_builder("dev") + matching_warn_2.assert_not_called() + ctx_dev_only.close() + + ctx_local_full = Context( + paths=proj_dev_only, + config=make_config(db_dev_only, VirtualEnvironmentMode.FULL), + ) + stack.callback(ctx_local_full.close) + mismatch_warn_2 = mocker.patch.object(ctx_local_full.console, "log_warning") + ctx_local_full.plan_builder("dev") + mismatch_warn_2.assert_called_once() + msg_2 = mismatch_warn_2.call_args[0][0] + assert "virtual_environment_mode" in msg_2 + assert "'full'" in msg_2 + assert "'dev_only'" in msg_2 + + mismatch_warn_2.reset_mock() + ctx_local_full.plan_builder("prod") + mismatch_warn_2.assert_not_called() + ctx_local_full.close() diff --git a/tests/core/test_snapshot_evaluator.py b/tests/core/test_snapshot_evaluator.py index 0955af331d..b5e41d5e62 100644 --- a/tests/core/test_snapshot_evaluator.py +++ b/tests/core/test_snapshot_evaluator.py @@ -25,6 +25,7 @@ DataObjectType, InsertOverwriteStrategy, ) +from sqlmesh.core.config.common import VirtualEnvironmentMode from sqlmesh.core.environment import EnvironmentNamingInfo from sqlmesh.core.macros import RuntimeStage, macro, MacroEvaluator, MacroFunc from sqlmesh.core.model import ( @@ -36,6 +37,8 @@ IncrementalByPartitionKind, IncrementalByUniqueKeyKind, PythonModel, + SeedKind, + SeedModel, SqlModel, TimeColumn, ViewKind, @@ -47,6 +50,7 @@ ) from sqlmesh.core.model.kind import OnDestructiveChange, ExternalKind, OnAdditiveChange from sqlmesh.core.model.meta import GrantsTargetLayer +from sqlmesh.core.model.seed import Seed from sqlmesh.core.node import IntervalUnit from sqlmesh.core.snapshot import ( DeployabilityIndex, @@ -69,6 +73,7 @@ IncrementalUnmanagedStrategy, MaterializableStrategy, SCDType2Strategy, + SeedStrategy, SnapshotCreationFailedError, ViewStrategy, ) @@ -1125,10 +1130,17 @@ def test_pre_hook_forward_only_clone( adapter.with_settings = lambda **kwargs: adapter adapter.table_exists = lambda _: True # type: ignore adapter.SUPPORTS_CLONING = True + source_table_obj = DataObject( + schema="sqlmesh__test_schema", + name=f"test_schema__test_model__{snapshot.version}", + type=DataObjectType.TABLE, + ) mocker.patch.object( adapter, "get_data_objects", - return_value=[], + side_effect=lambda schema_name, object_names=None, **kwargs: ( + [source_table_obj] if object_names and source_table_obj.name in object_names else [] + ), ) mocker.patch.object( adapter, @@ -1140,6 +1152,7 @@ def test_pre_hook_forward_only_clone( evaluator.create([snapshot], {}, deployability_index=DeployabilityIndex.none_deployable()) adapter.cursor.execute.assert_any_call(pre_statement) + assert any("CLONE" in str(call_args) for call_args in adapter.cursor.execute.call_args_list) def test_create_only_dev_table_exists(mocker: MockerFixture, adapter_mock, make_snapshot): @@ -6113,3 +6126,291 @@ def observer(phase, facts, error): consumer.reset(token) action = "schema_migration" if operation == "migrate" else operation assert events == [("start", action, None), ("finish", action, None)] + + +def test_can_clone_skips_when_prod_object_is_view( + adapter_mock: Mock, make_snapshot: t.Callable[..., Snapshot] +) -> None: + adapter_mock.SUPPORTS_CLONING = True + adapter_mock.table_exists.return_value = True + adapter_mock.get_alter_operations.return_value = [] + evaluator = SnapshotEvaluator(adapter_mock) + + model_obj = SqlModel( + name="test_schema.test_model", + kind=IncrementalByTimeRangeKind(time_column="ds"), + query=parse_one("SELECT 1::INT AS a, ds::DATE AS ds FROM src"), + virtual_environment_mode=VirtualEnvironmentMode.DEV_ONLY, + ) + snapshot = make_snapshot(model_obj) + snapshot.categorize_as(SnapshotChangeCategory.BREAKING, forward_only=True) + snapshot.previous_versions = snapshot.all_versions + + deployability_index = DeployabilityIndex.none_deployable() + + for non_table_type in ( + DataObjectType.VIEW, + DataObjectType.MATERIALIZED_VIEW, + DataObjectType.UNKNOWN, + ): + adapter_mock.get_data_object.return_value = DataObject( + schema="test_schema", + name="test_model", + type=non_table_type, + ) + assert evaluator._can_clone(snapshot, deployability_index) is False + + adapter_mock.get_data_object.return_value = None + assert evaluator._can_clone(snapshot, deployability_index) is False + + for table_type in (DataObjectType.TABLE, DataObjectType.MANAGED_TABLE): + adapter_mock.get_data_object.return_value = DataObject( + schema="test_schema", + name="test_model", + type=table_type, + ) + assert evaluator._can_clone(snapshot, deployability_index) is True + + adapter_mock.get_data_object.return_value = DataObject( + schema="test_schema", + name="test_model", + type=DataObjectType.VIEW, + ) + adapter_mock.clone_table.reset_mock() + adapter_mock.create_table.reset_mock() + + evaluator.create([snapshot], {}, deployability_index=deployability_index) + + adapter_mock.clone_table.assert_not_called() + adapter_mock.create_table.assert_called_once() + assert adapter_mock.create_table.call_args[0][0] == snapshot.table_name(is_deployable=False) + + +def test_cleanup_snapshot_dev_only_does_not_drop_prod_table_or_view( + duck_conn: t.Any, mocker: MockerFixture, make_snapshot: t.Callable[..., Snapshot] +) -> None: + from sqlmesh.core.engine_adapter import DuckDBEngineAdapter + + adapter = DuckDBEngineAdapter(lambda: duck_conn) + duck_conn.execute("CREATE SCHEMA test_schema") + duck_conn.execute("CREATE SCHEMA sqlmesh__test_schema") + + dev_only_full_snapshot = make_snapshot( + SqlModel( + name="test_schema.mat_model", + kind=FullKind(), + query=parse_one("SELECT 1 AS a"), + virtual_environment_mode=VirtualEnvironmentMode.DEV_ONLY, + ) + ) + dev_only_full_snapshot.categorize_as(SnapshotChangeCategory.BREAKING) + + dev_only_inc_snapshot = make_snapshot( + SqlModel( + name="test_schema.inc_model", + kind=IncrementalByTimeRangeKind(time_column="ds"), + query=parse_one("SELECT 1 AS a, '2023-01-01'::DATE AS ds"), + virtual_environment_mode=VirtualEnvironmentMode.DEV_ONLY, + ) + ) + dev_only_inc_snapshot.categorize_as(SnapshotChangeCategory.BREAKING) + + dev_only_view_snapshot = make_snapshot( + SqlModel( + name="test_schema.view_model", + kind=ViewKind(), + query=parse_one("SELECT 1 AS a"), + virtual_environment_mode=VirtualEnvironmentMode.DEV_ONLY, + ) + ) + dev_only_view_snapshot.categorize_as(SnapshotChangeCategory.BREAKING) + + full_vde_snapshot = make_snapshot( + SqlModel( + name="test_schema.full_vde_model", + kind=FullKind(), + query=parse_one("SELECT 1 AS a"), + virtual_environment_mode=VirtualEnvironmentMode.FULL, + ) + ) + full_vde_snapshot.categorize_as(SnapshotChangeCategory.BREAKING) + + # Create live prod objects and __dev objects for DEV_ONLY snapshots + duck_conn.execute("CREATE TABLE test_schema.mat_model AS SELECT 1 AS a") + duck_conn.execute( + f"CREATE TABLE {dev_only_full_snapshot.table_name(is_deployable=False)} AS SELECT 1 AS a" + ) + duck_conn.execute( + "CREATE TABLE test_schema.inc_model AS SELECT 1 AS a, DATE '2023-01-01' AS ds" + ) + duck_conn.execute( + f"CREATE TABLE {dev_only_inc_snapshot.table_name(is_deployable=False)} AS SELECT 1 AS a, DATE '2023-01-01' AS ds" + ) + duck_conn.execute("CREATE VIEW test_schema.view_model AS SELECT 1 AS a") + duck_conn.execute( + f"CREATE VIEW {dev_only_view_snapshot.table_name(is_deployable=False)} AS SELECT 1 AS a" + ) + + # Create both deployable and __dev physical tables for FULL VDE snapshot + duck_conn.execute( + f"CREATE TABLE {full_vde_snapshot.table_name(is_deployable=True)} AS SELECT 1 AS a" + ) + duck_conn.execute( + f"CREATE TABLE {full_vde_snapshot.table_name(is_deployable=False)} AS SELECT 1 AS a" + ) + + drop_table_spy = mocker.spy(DuckDBEngineAdapter, "drop_table") + drop_view_spy = mocker.spy(DuckDBEngineAdapter, "drop_view") + + evaluator = SnapshotEvaluator(adapter) + evaluator.cleanup( + [ + SnapshotTableCleanupTask( + snapshot=dev_only_full_snapshot.table_info, dev_table_only=False + ), + SnapshotTableCleanupTask( + snapshot=dev_only_inc_snapshot.table_info, dev_table_only=False + ), + SnapshotTableCleanupTask( + snapshot=dev_only_view_snapshot.table_info, dev_table_only=False + ), + SnapshotTableCleanupTask(snapshot=full_vde_snapshot.table_info, dev_table_only=False), + ] + ) + + # Only __dev objects should be dropped for DEV_ONLY models + assert adapter.get_data_object(dev_only_full_snapshot.table_name(is_deployable=False)) is None + assert adapter.get_data_object(dev_only_inc_snapshot.table_name(is_deployable=False)) is None + assert adapter.get_data_object(dev_only_view_snapshot.table_name(is_deployable=False)) is None + + # Live prod objects in . must remain untouched + assert adapter.get_data_object("test_schema.mat_model") is not None + assert adapter.get_data_object("test_schema.inc_model") is not None + assert adapter.get_data_object("test_schema.view_model") is not None + assert duck_conn.execute("SELECT * FROM test_schema.view_model").fetchall() == [(1,)] + + dropped_tables = {str(call_args.args[1]) for call_args in drop_table_spy.call_args_list} + dropped_views = {str(call_args.args[1]) for call_args in drop_view_spy.call_args_list} + for prod_obj in ( + dev_only_full_snapshot.table_name(is_deployable=True), + dev_only_inc_snapshot.table_name(is_deployable=True), + dev_only_view_snapshot.table_name(is_deployable=True), + ): + assert prod_obj not in dropped_tables + assert prod_obj not in dropped_views + + # Both __dev and deployable sqlmesh__.* tables ARE dropped for FULL VDE models + assert adapter.get_data_object(full_vde_snapshot.table_name(is_deployable=False)) is None + assert adapter.get_data_object(full_vde_snapshot.table_name(is_deployable=True)) is None + + +def test_seed_create_drops_existing_view_in_dev_only_prod(adapter_mock: Mock) -> None: + seed_model = SeedModel( + name="test_schema.seed_model", + kind=SeedKind(path="./seed.csv"), + seed=Seed(content="id,name\n1,alice\n"), + column_hashes={"id": "1", "name": "2"}, + columns={"id": "int", "name": "varchar"}, + virtual_environment_mode=VirtualEnvironmentMode.DEV_ONLY, + ) + dehydrated_seed_model = seed_model.to_dehydrated() + existing_view_obj = DataObject( + schema="test_schema", + name="seed_model", + type=DataObjectType.VIEW, + ) + + # 1. Hydrated SeedModel in DEV_ONLY deployable prod drops mismatched view once content is rendered, BEFORE create_table + events: t.List[str] = [] + adapter_mock.get_data_object.return_value = existing_view_obj + + def _record_drop(obj: t.Any, tpe: t.Any) -> bool: + events.append("drop_mismatch") + return True + + def _record_create_table(*args: t.Any, **kwargs: t.Any) -> None: + events.append("create_table") + + adapter_mock.drop_data_object_on_type_mismatch.side_effect = _record_drop + adapter_mock.create_table.side_effect = _record_create_table + + SeedStrategy(adapter_mock).create( + table_name="test_schema.seed_model", + model=seed_model, + is_table_deployable=True, + render_kwargs={}, + skip_grants=True, + ) + adapter_mock.drop_data_object_on_type_mismatch.assert_called_once_with( + existing_view_obj, DataObjectType.TABLE + ) + assert events == ["drop_mismatch", "create_table"] + + # 2. Dehydrated SeedModel has no content to rebuild the table from, so the existing object must be kept + adapter_mock.drop_data_object_on_type_mismatch.reset_mock() + adapter_mock.create_table.reset_mock() + adapter_mock.table_exists.side_effect = lambda name: True + + SeedStrategy(adapter_mock).create( + table_name="test_schema.seed_model", + model=dehydrated_seed_model, + is_table_deployable=True, + render_kwargs={}, + skip_grants=True, + ) + adapter_mock.drop_data_object_on_type_mismatch.assert_not_called() + adapter_mock.drop_data_object.assert_not_called() + adapter_mock.create_table.assert_not_called() + + # And if nothing exists, a dehydrated seed still fails clearly without touching anything + adapter_mock.table_exists.side_effect = lambda name: False + with pytest.raises(SQLMeshError, match="is not hydrated"): + SeedStrategy(adapter_mock).create( + table_name="test_schema.seed_model", + model=dehydrated_seed_model, + is_table_deployable=True, + render_kwargs={}, + skip_grants=True, + ) + adapter_mock.drop_data_object_on_type_mismatch.assert_not_called() + + # 2b. If the seed content can't be rendered, the existing object is left in place + adapter_mock.table_exists.side_effect = None + adapter_mock.table_exists.return_value = False + adapter_mock.create_table.reset_mock() + with patch.object(SeedModel, "render_seed", side_effect=SQLMeshError("render failed")): + with pytest.raises(SQLMeshError, match="render failed"): + SeedStrategy(adapter_mock).create( + table_name="test_schema.seed_model", + model=seed_model, + is_table_deployable=True, + render_kwargs={}, + skip_grants=True, + ) + adapter_mock.drop_data_object_on_type_mismatch.assert_not_called() + adapter_mock.create_table.assert_not_called() + + # 3. Non-deployable (__dev) seed table creation must NOT call drop_data_object_on_type_mismatch + adapter_mock.drop_data_object_on_type_mismatch.reset_mock() + SeedStrategy(adapter_mock).create( + table_name="sqlmesh__test_schema.test_schema__seed_model__1__dev", + model=seed_model, + is_table_deployable=False, + render_kwargs={}, + skip_grants=True, + ) + adapter_mock.drop_data_object_on_type_mismatch.assert_not_called() + + # 4. FULL VDE mode seed table creation must NOT call drop_data_object_on_type_mismatch + full_seed_model = seed_model.copy( + update={"virtual_environment_mode": VirtualEnvironmentMode.FULL} + ) + adapter_mock.drop_data_object_on_type_mismatch.reset_mock() + SeedStrategy(adapter_mock).create( + table_name="sqlmesh__test_schema.test_schema__seed_model__1", + model=full_seed_model, + is_table_deployable=True, + render_kwargs={}, + skip_grants=True, + ) + adapter_mock.drop_data_object_on_type_mismatch.assert_not_called() diff --git a/tests/dbt/test_transformation.py b/tests/dbt/test_transformation.py index 32249cbef3..ddcc96e1a8 100644 --- a/tests/dbt/test_transformation.py +++ b/tests/dbt/test_transformation.py @@ -3033,3 +3033,176 @@ def test_dbt_hooks_with_transaction_flag_execution(sushi_test_dbt_context: Conte for i in range(len(shared_table) - 1): assert shared_table["created_at"][i] <= shared_table["created_at"][i + 1] + + +def test_dbt_virtual_environment_mode_precedence(tmp_path: Path): + from contextlib import ExitStack + from sqlmesh.utils import yaml + from sqlmesh.core.config.common import VirtualEnvironmentMode + + context = DbtContext() + context.project_name = "Test" + context.target = DuckDbConfig(name="target", schema="foo") + + # 1. ModelConfig.to_sqlmesh precedence: per-model field / meta > passed virtual_environment_mode + default_model_cfg = ModelConfig( + name="m_default", alias="m_default", schema="foo", sql="SELECT 1 AS a" + ) + assert ( + default_model_cfg.to_sqlmesh( + context, virtual_environment_mode=VirtualEnvironmentMode.DEV_ONLY + ).virtual_environment_mode + == VirtualEnvironmentMode.DEV_ONLY + ) + + dev_only_model_cfg = ModelConfig( + name="m_dev_only", + alias="m_dev_only", + schema="foo", + sql="SELECT 1 AS a", + virtual_environment_mode="DEV_ONLY", + ) + assert dev_only_model_cfg.virtual_environment_mode == VirtualEnvironmentMode.DEV_ONLY + assert ( + dev_only_model_cfg.to_sqlmesh( + context, virtual_environment_mode=VirtualEnvironmentMode.FULL + ).virtual_environment_mode + == VirtualEnvironmentMode.DEV_ONLY + ) + + full_model_cfg = ModelConfig( + name="m_full", + alias="m_full", + schema="foo", + sql="SELECT 1 AS a", + virtual_environment_mode="full", + ) + assert ( + full_model_cfg.to_sqlmesh( + context, virtual_environment_mode=VirtualEnvironmentMode.DEV_ONLY + ).virtual_environment_mode + == VirtualEnvironmentMode.FULL + ) + + meta_model_cfg = ModelConfig( + name="m_meta", + alias="m_meta", + schema="foo", + sql="SELECT 1 AS a", + meta={"virtual_environment_mode": "dev_only"}, + ) + assert ( + meta_model_cfg.to_sqlmesh( + context, virtual_environment_mode=VirtualEnvironmentMode.FULL + ).virtual_environment_mode + == VirtualEnvironmentMode.DEV_ONLY + ) + + # 2. SeedConfig.to_sqlmesh parity + seed_csv = Path("examples/sushi_dbt/seeds/waiter_names.csv") + seed_dev_only = SeedConfig( + name="s_dev_only", + package_name="Test", + schema="foo", + path=seed_csv, + virtual_environment_mode="dev_only", + ) + assert ( + seed_dev_only.to_sqlmesh( + context, virtual_environment_mode=VirtualEnvironmentMode.FULL + ).virtual_environment_mode + == VirtualEnvironmentMode.DEV_ONLY + ) + + seed_full = SeedConfig( + name="s_full", + package_name="Test", + schema="foo", + path=seed_csv, + virtual_environment_mode="full", + ) + assert ( + seed_full.to_sqlmesh( + context, virtual_environment_mode=VirtualEnvironmentMode.DEV_ONLY + ).virtual_environment_mode + == VirtualEnvironmentMode.FULL + ) + + # 3. End-to-end dbt project with {{ config(virtual_environment_mode=...) }} and +virtual_environment_mode + dbt_project_dir = tmp_path / "dbt_proj" + models_dir = dbt_project_dir / "models" + seeds_dir = dbt_project_dir / "seeds" + models_dir.mkdir(parents=True) + seeds_dir.mkdir(parents=True) + + (models_dir / "m_default.sql").write_text("SELECT 1 AS id", encoding="utf-8") + (models_dir / "m_jinja_dev_only.sql").write_text( + "{{ config(materialized='table', virtual_environment_mode='dev_only') }}\nSELECT 1 AS id", + encoding="utf-8", + ) + (models_dir / "m_jinja_full.sql").write_text( + "{{ config(materialized='table', virtual_environment_mode='full') }}\nSELECT 1 AS id", + encoding="utf-8", + ) + (seeds_dir / "raw_seed.csv").write_text("id,name\n1,alice\n", encoding="utf-8") + + with open(dbt_project_dir / "dbt_project.yml", "w", encoding="utf-8") as f: + yaml.dump( + { + "name": "test_project", + "version": "1.0.0", + "config-version": 2, + "profile": "test", + "model-paths": ["models"], + "seed-paths": ["seeds"], + "seeds": { + "test_project": { + "+virtual_environment_mode": "dev_only", + } + }, + }, + f, + ) + + with open(dbt_project_dir / "sqlmesh.yaml", "w", encoding="utf-8") as f: + yaml.dump( + { + "virtual_environment_mode": "full", + "model_defaults": { + "start": "2025-01-01", + }, + }, + f, + ) + + db_file = tmp_path / "dbt_local.db" + with open(dbt_project_dir / "profiles.yml", "w", encoding="utf-8") as f: + yaml.dump( + { + "test": { + "outputs": {"duckdb": {"type": "duckdb", "path": str(db_file)}}, + "target": "duckdb", + } + }, + f, + ) + + with ExitStack() as stack: + sqlmesh_ctx = Context(paths=dbt_project_dir) + stack.callback(sqlmesh_ctx.close) + assert ( + sqlmesh_ctx.get_model("main.m_default").virtual_environment_mode + == VirtualEnvironmentMode.FULL + ) + assert ( + sqlmesh_ctx.get_model("main.m_jinja_dev_only").virtual_environment_mode + == VirtualEnvironmentMode.DEV_ONLY + ) + assert ( + sqlmesh_ctx.get_model("main.m_jinja_full").virtual_environment_mode + == VirtualEnvironmentMode.FULL + ) + assert ( + sqlmesh_ctx.get_model("main.raw_seed").virtual_environment_mode + == VirtualEnvironmentMode.DEV_ONLY + )