From 349fc115a1edee8b5a050357833e48fdba784c4f Mon Sep 17 00:00:00 2001 From: Iaroslav Zeigerman Date: Mon, 5 Dec 2022 12:33:46 -0800 Subject: [PATCH 1/3] removes type restriction fixes #333 --- example/models/items.sql | 2 +- sqlmesh/core/engine_adapter.py | 39 +++++++---- sqlmesh/core/model.py | 109 +++++++++++++++++++---------- sqlmesh/core/snapshot.py | 2 +- sqlmesh/core/snapshot_evaluator.py | 4 +- 5 files changed, 104 insertions(+), 52 deletions(-) diff --git a/example/models/items.sql b/example/models/items.sql index 7fcb8a7acc..cc20cf9a3a 100644 --- a/example/models/items.sql +++ b/example/models/items.sql @@ -10,7 +10,7 @@ MODEL ( ); SELECT - id::INT AS id, /* Primary key */ + id AS id, /* Primary key */ name::TEXT AS name, /* Name of the sushi */ price::DOUBLE AS price, /* Price of the sushi */ ds::TEXT AS ds /* Date */ diff --git a/sqlmesh/core/engine_adapter.py b/sqlmesh/core/engine_adapter.py index 405496bec4..2bc03d3736 100644 --- a/sqlmesh/core/engine_adapter.py +++ b/sqlmesh/core/engine_adapter.py @@ -104,29 +104,42 @@ def replace_query(self, table_name: str, query_or_df: QueryOrDF): def create_table( self, table_name: str, - columns: t.Dict[str, exp.DataType], + query_or_columns: Query | t.Dict[str, exp.DataType], exists: bool = True, **kwargs, ) -> None: - """Create a table using a DDL statement + """Create a table using a DDL statement or a CTAS. + + If a query is passed in instead of column type map, CREATE TABLE AS will be used. Args: - table_name: The name of the table to create. Can be fully qualified or just table name - columns: A mapping between the column name and its data type - exists: Indicates if you to include an IF EXISTS check + table_name: The name of the table to create. Can be fully qualified or just table name. + query_columns: A query or mapping between the column name and its data type. + exists: Indicates if you to include an IF EXISTS check. kwargs: Optional create table properties. """ properties = self._create_table_properties(**kwargs) - schema = exp.Schema( - this=exp.to_table(table_name), - expressions=[ - exp.ColumnDef(this=exp.to_identifier(column), kind=kind) - for column, kind in columns.items() - ], - ) + query = None + schema: t.Optional[exp.Schema | exp.Table] = exp.to_table(table_name) + + if isinstance(query_or_columns, dict): + schema = exp.Schema( + this=schema, + expressions=[ + exp.ColumnDef(this=exp.to_identifier(column), kind=kind) + for column, kind in query_or_columns.items() + ], + ) + else: + query = query_or_columns + create_expression = exp.Create( - this=schema, kind="TABLE", exists=exists, properties=properties + this=schema, + kind="TABLE", + exists=exists, + properties=properties, + expression=query, ) self.execute(create_expression) diff --git a/sqlmesh/core/model.py b/sqlmesh/core/model.py index 0781c6f724..15a442b838 100644 --- a/sqlmesh/core/model.py +++ b/sqlmesh/core/model.py @@ -252,6 +252,7 @@ from croniter import croniter from pydantic import Field, root_validator, validator from sqlglot import exp, maybe_parse, parse +from sqlglot.optimizer.annotate_types import annotate_types from sqlglot.optimizer.scope import traverse_scope from sqlglot.optimizer.simplify import simplify from sqlglot.time import format_time @@ -722,7 +723,7 @@ def depends_on(self) -> t.Set[str]: return self.depends_on_ if self._depends_on is None: - self._depends_on = find_tables(self._render_query(self.query)) - {self.name} + self._depends_on = find_tables(self._render_query()) - {self.name} return self._depends_on @property @@ -731,7 +732,7 @@ def column_descriptions(self) -> t.Dict[str, str]: if self._column_descriptions is None: self._column_descriptions = { select.alias: "\n".join(comment.strip() for comment in select.comments) - for select in self._render_query(self.query).expressions + for select in self._render_query().expressions if select.comments } return self._column_descriptions @@ -743,12 +744,22 @@ def columns(self) -> t.Dict[str, exp.DataType]: return self.columns_ if self._columns is None: + query = annotate_types(self._render_query()) + self._columns = { - expression.alias_or_name: expression.find(exp.Cast).to - for expression in self._render_query(self.query).expressions + expression.alias_or_name: exp.DataType.build(expression.type) + for expression in query.expressions } return self._columns + @property + def annotated(self) -> bool: + """Checks if all column projection types of this model are known.""" + return all( + column_type.this != exp.DataType.Type.UNKNOWN + for column_type in self.columns.values() + ) + def render(self) -> t.List[exp.Expression]: """Returns the original list of sql expressions comprising the model.""" expressions = [] @@ -775,11 +786,12 @@ def render(self) -> t.List[exp.Expression]: def _render_query( self, - query: exp.Expression, + query_: t.Optional[exp.Expression] = None, *, start: t.Optional[TimeLike] = None, end: t.Optional[TimeLike] = None, latest: t.Optional[TimeLike] = None, + add_incremental_filter: bool = False, snapshots: t.Optional[t.Dict[str, Snapshot]] = None, mapping: t.Optional[t.Dict[str, str]] = None, expand: t.Iterable[str] = tuple(), @@ -790,10 +802,11 @@ def _render_query( """Renders a query, expanding macros with provided kwargs, and optionally expanding referenced models. Args: - query: The query to render. + query_: The query to render, defaults to self.query. start: The start datetime to render. Defaults to epoch start. end: The end datetime to render. Defaults to epoch start. latest: The latest datetime to use for non-incremental queries. Defaults to epoch start. + add_incremental_filter: Add an incremental filter to the query if the model is incremental. snapshots: All snapshots to use for expansion and mapping of physical locations. If passing snapshots is undesirable, mapping can be used instead to manually map tables. mapping: Mapping to replace table names, if not set, the mapping wil be created from snapshots. @@ -823,14 +836,24 @@ def _render_query( expand = set(expand) | {name for name in snapshots if name not in mapping} if key not in self._query_cache: - macro_evaluator = MacroEvaluator(dialect or self.dialect, self.python_env) - macro_evaluator.locals.update(kwargs) - macro_evaluator.locals.update(date_dict(*dates)) + if self.is_sql: + macro_evaluator = MacroEvaluator( + dialect or self.dialect, self.python_env + ) + macro_evaluator.locals.update(kwargs) + macro_evaluator.locals.update(date_dict(*dates)) - for definition in self.macro_definitions: - macro_evaluator.evaluate(definition) + for definition in self.macro_definitions: + macro_evaluator.evaluate(definition) - self._query_cache[key] = macro_evaluator.transform(query) # type: ignore + self._query_cache[key] = macro_evaluator.transform(query_ or self.query) # type: ignore + else: + self._query_cache[key] = exp.select( + *( + exp.alias_(f"NULL::{column_type}", name) + for name, column_type in self.columns.items() + ) + ) query = self._query_cache[key] @@ -840,7 +863,7 @@ def _expand(node: exp.Expression) -> exp.Expression: if isinstance(node, exp.Table) and snapshots: name = exp.table_name(node) model = snapshots[name].model if name in snapshots else None - if name in expand and model and model.is_sql: + if name in expand and model: return model._render_query( model.query, start=start, @@ -858,6 +881,16 @@ def _expand(node: exp.Expression) -> exp.Expression: query = query.transform(_expand) + # Ensure there is no data leakage in incremental mode by filtering out all + # events that have data outside the time window of interest. + if add_incremental_filter and self.kind == ModelKind.INCREMENTAL: + # expansion copies the query for us. if it doesn't occur, make sure to copy. + if not expand: + query = query.copy() + for node, _, _ in query.walk(prune=lambda n, *_: isinstance(n, exp.Select)): + if isinstance(node, exp.Select): + self._filter_time_column(node, start or EPOCH_DS, end or EPOCH_DS) + if mapping: return exp.replace_tables(query, mapping) @@ -866,6 +899,25 @@ def _expand(node: exp.Expression) -> exp.Expression: return query + def ctas_query(self, snapshots: t.Dict[str, Snapshot]) -> exp.Subqueryable: + """Return a dummy query to do a CTAS. + + If a model's column types are unknown, the only way to create the table is to + run the fully expanded query. This can be expensive so we add a WHERE FALSE to all + SELECTS and hopefully the optimizer is smart enough to not do anything. + + Args: + All upstream snapshots of this model so queries can be expanded. + Return: + The mocked out ctas query. + """ + query = self._render_query(snapshots=snapshots, expand=snapshots) + # the query is expanded so it's been copied, it's safe to mutate. + for select in query.find_all(exp.Select): + select.where("FALSE", copy=False) + + return query + def render_query( self, *, @@ -895,26 +947,16 @@ def render_query( Returns: The rendered expression. """ - query: exp.Subqueryable = self._render_query( - self.query, + return self._render_query( start=start, end=end, latest=latest, + add_incremental_filter=True, snapshots=snapshots, mapping=mapping, expand=expand, **kwargs, - ).copy() - - # Ensure there is no data leakage in incremental mode by filtering out all - # events that have data outside the time window of interest. - if self.kind == ModelKind.INCREMENTAL: - prune = lambda n, p, k: isinstance(n, exp.Select) - for node, _, _ in query.walk(prune=prune): - if isinstance(node, exp.Select): - self._filter_time_column(node, start or EPOCH_DS, end or EPOCH_DS) - - return simplify(query) + ) def exec_python( self, @@ -1067,8 +1109,9 @@ def validate_definition(self) -> None: ConfigError """ name_counts: t.Dict[str, int] = {} + query = self._render_query() - for expression in self._render_query(self.query).expressions: + for expression in query.expressions: if isinstance(expression, exp.Star): _raise_config_error( "SELECT * is not allowed you must explicitly select columns.", @@ -1086,12 +1129,6 @@ def validate_definition(self) -> None: self.path, ) - if not isinstance(expression, exp.Cast): - _raise_config_error( - f"Outer projection `{expression}` must be explicitly cast to a type. You can use the double colon syntax regardless of dialect for convenience (eg: my_column::int)", - self.path, - ) - for name, count in name_counts.items(): if count > 1: _raise_config_error( @@ -1106,9 +1143,7 @@ def validate_definition(self) -> None: self.path, ) - projections = { - p.lower() for p in self._render_query(self.query).named_selects - } + projections = {p.lower() for p in query.named_selects} missing_keys = unique_partition_keys - projections if missing_keys: missing_keys_str = ", ".join(f"'{k}'" for k in sorted(missing_keys)) @@ -1149,6 +1184,8 @@ def _filter_time_column( else: query.having(between, copy=False) + simplify(query) + def __repr__(self): return f"Model" diff --git a/sqlmesh/core/snapshot.py b/sqlmesh/core/snapshot.py index 1c14ee6be6..0f6e2ad6b4 100644 --- a/sqlmesh/core/snapshot.py +++ b/sqlmesh/core/snapshot.py @@ -590,7 +590,7 @@ def fingerprint_from_model( Returns: The fingerprint. """ - cache = cache or {} + cache = {} if cache is None else cache if model.name not in cache: data = [ diff --git a/sqlmesh/core/snapshot_evaluator.py b/sqlmesh/core/snapshot_evaluator.py index d14887e003..9b4c7e3f97 100644 --- a/sqlmesh/core/snapshot_evaluator.py +++ b/sqlmesh/core/snapshot_evaluator.py @@ -182,7 +182,9 @@ def create(self, snapshot: Snapshot, snapshots: t.Dict[str, Snapshot]) -> None: logger.info("Creating table '%s'", table_name) self.adapter.create_table( table_name, - columns=snapshot.model.columns, + query_or_columns=snapshot.model.columns + if snapshot.model.annotated + else snapshot.model.ctas_query(snapshots), storage_format=snapshot.model.storage_format, partitioned_by=snapshot.model.partitioned_by, ) From 822520ba3e95c1dfc9cfe070b4c1a894056cd4e0 Mon Sep 17 00:00:00 2001 From: tobymao Date: Mon, 5 Dec 2022 13:49:03 -0800 Subject: [PATCH 2/3] fix tests --- setup.py | 2 +- sqlmesh/core/model.py | 4 +- tests/core/test_context.py | 129 +++++++++++++++++--------- tests/core/test_integration.py | 1 + tests/core/test_model.py | 12 +-- tests/core/test_snapshot_evaluator.py | 2 +- 6 files changed, 96 insertions(+), 54 deletions(-) diff --git a/setup.py b/setup.py index ed20cf4f62..b048995ba3 100644 --- a/setup.py +++ b/setup.py @@ -32,7 +32,7 @@ "requests", "rich", "ruamel.yaml", - "sqlglot>=10.2.0", + "sqlglot>=10.2.1", ], extras_require={ "dev": [ diff --git a/sqlmesh/core/model.py b/sqlmesh/core/model.py index 15a442b838..55084de5a7 100644 --- a/sqlmesh/core/model.py +++ b/sqlmesh/core/model.py @@ -747,7 +747,7 @@ def columns(self) -> t.Dict[str, exp.DataType]: query = annotate_types(self._render_query()) self._columns = { - expression.alias_or_name: exp.DataType.build(expression.type) + expression.alias_or_name: expression.type for expression in query.expressions } return self._columns @@ -889,7 +889,7 @@ def _expand(node: exp.Expression) -> exp.Expression: query = query.copy() for node, _, _ in query.walk(prune=lambda n, *_: isinstance(n, exp.Select)): if isinstance(node, exp.Select): - self._filter_time_column(node, start or EPOCH_DS, end or EPOCH_DS) + self._filter_time_column(node, *dates[0:2]) if mapping: return exp.replace_tables(query, mapping) diff --git a/tests/core/test_context.py b/tests/core/test_context.py index a6569f6a8d..942611c47a 100644 --- a/tests/core/test_context.py +++ b/tests/core/test_context.py @@ -97,40 +97,61 @@ def test_render(sushi_context, assert_exp_eq, mock_file_cache): ), f""" SELECT - CAST(o.waiter_id AS INT) AS waiter_id, -- Waiter id - CAST(SUM(oi.quantity * i.price) AS DOUBLE) AS revenue, -- Revenue from orders taken by this waiter - CAST(o.ds AS TEXT) AS ds, -- Date + CAST(o.waiter_id AS INT) AS waiter_id, /* Waiter id */ + CAST(SUM(oi.quantity * i.price) AS DOUBLE) AS revenue, /* Revenue from orders taken by this waiter */ + CAST(o.ds AS TEXT) AS ds /* Date */ FROM ( SELECT - CAST(id AS INT) AS id, -- Primary key - CAST(customer_id AS INT) AS customer_id, -- Id of customer who made the order - CAST(waiter_id AS INT) AS waiter_id, -- Id of waiter who took the order - CAST(start_ts AS TEXT) AS start_ts, -- Start timestamp - CAST(end_ts AS TEXT) AS end_ts, -- End timestamp - CAST(ds AS TEXT) AS ds, -- Date of order - FROM {sushi_context.snapshots["raw.orders"].table_name} + CAST(id AS INT) AS id, /* Primary key */ + CAST(customer_id AS INT) AS customer_id, /* Id of customer who made the order */ + CAST(waiter_id AS INT) AS waiter_id, /* Id of waiter who took the order */ + CAST(start_ts AS TEXT) AS start_ts, /* Start timestamp */ + CAST(end_ts AS TEXT) AS end_ts, /* End timestamp */ + CAST(ds AS TEXT) AS ds /* Date of order */ + FROM ( + SELECT + CAST(NULL AS INT) AS id, + CAST(NULL AS INT) AS customer_id, + CAST(NULL AS INT) AS waiter_id, + CAST(NULL AS INT) AS start_ts, + CAST(NULL AS INT) AS end_ts, + CAST(NULL AS TEXT) AS ds + ) AS orders WHERE ds <= '2021-01-01' AND ds >= '2021-01-01' ) AS o LEFT JOIN ( SELECT - CAST(id AS INT) AS id, -- Primary key - CAST(order_id AS INT) AS order_id, -- Order id - CAST(item_id AS INT) AS item_id, -- Item id - CAST(quantity AS INT) AS quantity, -- Quantity of items ordered - CAST(ds AS TEXT) AS ds, -- Date of order - FROM {sushi_context.snapshots["raw.order_items"].table_name} + CAST(id AS INT) AS id, /* Primary key */ + CAST(order_id AS INT) AS order_id, /* Order id */ + CAST(item_id AS INT) AS item_id, /* Item id */ + CAST(quantity AS INT) AS quantity, /* Quantity of items ordered */ + CAST(ds AS TEXT) AS ds /* Date of order */ + FROM ( + SELECT + CAST(NULL AS INT) AS id, + CAST(NULL AS INT) AS order_id, + CAST(NULL AS INT) AS item_id, + CAST(NULL AS INT) AS quantity, + CAST(NULL AS TEXT) AS ds + ) AS order_items WHERE ds <= '2021-01-01' AND ds >= '2021-01-01' ) AS oi ON o.ds = oi.ds AND o.id = oi.order_id LEFT JOIN ( SELECT - CAST(id AS INT) AS id, -- Primary key - CAST(name AS TEXT) AS name, -- Name of the sushi - CAST(price AS DOUBLE) AS price, -- Price of the sushi - CAST(ds AS TEXT) AS ds, -- Date - FROM {sushi_context.snapshots["raw.items"].table_name} + id AS id, /* Primary key */ + CAST(name AS TEXT) AS name, /* Name of the sushi */ + CAST(price AS DOUBLE) AS price, /* Price of the sushi */ + CAST(ds AS TEXT) AS ds /* Date */ + FROM ( + SELECT + CAST(NULL AS INT) AS id, + CAST(NULL AS TEXT) AS name, + CAST(NULL AS DOUBLE) AS price, + CAST(NULL AS TEXT) AS ds + ) AS items WHERE ds <= '2021-01-01' AND ds >= '2021-01-01' ) AS i @@ -178,39 +199,61 @@ def test_render(sushi_context, assert_exp_eq, mock_file_cache): unpushed.render(snapshot.name), f""" SELECT - CAST(o.waiter_id AS INT) AS waiter_id, -- Waiter id - CAST(SUM(oi.quantity * i.price) AS DOUBLE) AS revenue, -- Revenue from orders taken by this waiter - CAST(o.ds AS TEXT) AS ds, -- Date + CAST(o.waiter_id AS INT) AS waiter_id, /* Waiter id */ + CAST(SUM(oi.quantity * i.price) AS DOUBLE) AS revenue, /* Revenue from orders taken by this waiter */ + CAST(o.ds AS TEXT) AS ds /* Date */ FROM ( SELECT - CAST(id AS INT) AS id, -- Primary key - CAST(customer_id AS INT) AS customer_id, -- Id of customer who made the order - CAST(waiter_id AS INT) AS waiter_id, -- Id of waiter who took the order - CAST(start_ts AS TEXT) AS start_ts, -- Start timestamp - CAST(end_ts AS TEXT) AS end_ts, -- End timestamp - CAST(ds AS TEXT) AS ds, -- Date of order - FROM raw.orders + CAST(id AS INT) AS id, /* Primary key */ + CAST(customer_id AS INT) AS customer_id, /* Id of customer who made the order */ + CAST(waiter_id AS INT) AS waiter_id, /* Id of waiter who took the order */ + CAST(start_ts AS TEXT) AS start_ts, /* Start timestamp */ + CAST(end_ts AS TEXT) AS end_ts, /* End timestamp */ + CAST(ds AS TEXT) AS ds /* Date of order */ + FROM ( + SELECT + CAST(NULL AS INT) AS id, + CAST(NULL AS INT) AS customer_id, + CAST(NULL AS INT) AS waiter_id, + CAST(NULL AS INT) AS start_ts, + CAST(NULL AS INT) AS end_ts, + CAST(NULL AS TEXT) AS ds + ) AS orders WHERE ds <= '1970-01-01' AND ds >= '1970-01-01' ) AS o LEFT JOIN ( SELECT - CAST(id AS INT) AS id, -- Primary key - CAST(order_id AS INT) AS order_id, -- Order id - CAST(item_id AS INT) AS item_id, -- Item id - CAST(quantity AS INT) AS quantity, -- Quantity of items ordered - CAST(ds AS TEXT) AS ds, -- Date of order - FROM raw.order_items - WHERE ds <= '1970-01-01' AND ds >= '1970-01-01' + CAST(id AS INT) AS id, /* Primary key */ + CAST(order_id AS INT) AS order_id, /* Order id */ + CAST(item_id AS INT) AS item_id, /* Item id */ + CAST(quantity AS INT) AS quantity, /* Quantity of items ordered */ + CAST(ds AS TEXT) AS ds /* Date of order */ + FROM ( + SELECT + CAST(NULL AS INT) AS id, + CAST(NULL AS INT) AS order_id, + CAST(NULL AS INT) AS item_id, + CAST(NULL AS INT) AS quantity, + CAST(NULL AS TEXT) AS ds + ) AS order_items + WHERE + ds <= '1970-01-01' AND ds >= '1970-01-01' ) AS oi ON o.ds = oi.ds AND o.id = oi.order_id LEFT JOIN ( SELECT - CAST(id AS INT) AS id, -- Primary key - CAST(name AS TEXT) AS name, -- Name of the sushi - CAST(price AS DOUBLE) AS price, -- Price of the sushi - CAST(ds AS TEXT) AS ds, -- Date - FROM raw.items + id AS id, /* Primary key */ + CAST(name AS TEXT) AS name, /* Name of the sushi */ + CAST(price AS DOUBLE) AS price, /* Price of the sushi */ + CAST(ds AS TEXT) AS ds /* Date */ + FROM ( + SELECT + CAST(NULL AS INT) AS id, + CAST(NULL AS TEXT) AS name, + CAST(NULL AS DOUBLE) AS price, + CAST(NULL AS TEXT) AS ds + ) AS items WHERE ds <= '1970-01-01' AND ds >= '1970-01-01' ) AS i diff --git a/tests/core/test_integration.py b/tests/core/test_integration.py index b65e506810..90bbc0888d 100644 --- a/tests/core/test_integration.py +++ b/tests/core/test_integration.py @@ -395,6 +395,7 @@ def test_no_override(sushi_context: Context) -> None: plan.set_choice(items, SnapshotChangeCategory.BREAKING) plan.set_choice(order_items, SnapshotChangeCategory.NON_BREAKING) assert items.is_new_version + print(waiter_revenue.fingerprint, waiter_revenue.version) assert waiter_revenue.is_new_version plan.set_choice(items, SnapshotChangeCategory.NON_BREAKING) assert not waiter_revenue.is_new_version diff --git a/tests/core/test_model.py b/tests/core/test_model.py index 4a4f624678..38f70b17ba 100644 --- a/tests/core/test_model.py +++ b/tests/core/test_model.py @@ -4,7 +4,7 @@ from sqlglot import exp, parse, parse_one from sqlmesh.core.dialect import format_model_expressions -from sqlmesh.core.model import Model, ModelMeta +from sqlmesh.core.model import Model, ModelMeta, TimeColumn from sqlmesh.utils.date import to_date, to_datetime, to_timestamp from sqlmesh.utils.errors import ConfigError @@ -88,12 +88,9 @@ def test_load(assert_exp_eq): @pytest.mark.parametrize( "query, error", [ - ("x", "must be explicitly cast to a type"), ("sum(x)::int", "must have inferrable names"), ("CAST(x + 1 AS INT)", "must have inferrable names"), - ("y AS y", "must be explicitly cast to a type"), ("y::int, x::int AS y", "duplicate"), - ("x --annotation", "must be explicitly cast to a type"), ("sum(x)::int -- annotation", "must have inferrable names"), ("*", "explicitly select"), ], @@ -346,9 +343,10 @@ def test_render_query(assert_exp_eq): model = Model( name="test", cron="1 0 * * *", + time_column=TimeColumn(column="y"), query=parse_one( """ - SELECT * + SELECT y FROM x WHERE y BETWEEN @start_date and @end_date AND @@ -359,7 +357,7 @@ def test_render_query(assert_exp_eq): assert_exp_eq( model.render_query(start="2020-10-28", end="2020-10-28"), """ - SELECT * + SELECT y FROM x WHERE y <= '2020-10-28' @@ -371,7 +369,7 @@ def test_render_query(assert_exp_eq): assert_exp_eq( model.render_query(start="2020-10-28", end=to_datetime("2020-10-29")), """ - SELECT * + SELECT y FROM x WHERE y <= '2020-10-28' diff --git a/tests/core/test_snapshot_evaluator.py b/tests/core/test_snapshot_evaluator.py index 0c828fb220..eeae5a3a8e 100644 --- a/tests/core/test_snapshot_evaluator.py +++ b/tests/core/test_snapshot_evaluator.py @@ -66,7 +66,7 @@ def test_evaluate(mocker: MockerFixture, make_snapshot): adapter_mock.create_table.assert_called_once_with( "physical_schema.test_schema__test_model__1", - columns={"a": exp.DataType.build("int")}, + query_or_columns={"a": exp.DataType.build("int")}, storage_format="parquet", partitioned_by=["a"], ) From 06ed0d5c15fbf57edd883ebfd5ca55ad0c6f7f51 Mon Sep 17 00:00:00 2001 From: tobymao Date: Mon, 5 Dec 2022 15:24:27 -0800 Subject: [PATCH 3/3] simplify example --- example/models/customer_revenue_by_day.sql | 2 +- example/models/customers.sql | 2 +- example/models/{raw_items.py => items.py} | 2 +- example/models/items.sql | 19 --- .../{raw_order_items.py => order_items.py} | 6 +- example/models/order_items.sql | 18 --- example/models/{raw_orders.py => orders.py} | 3 +- example/models/orders.sql | 19 --- example/models/top_waiters.sql | 2 +- example/models/waiter_revenue_by_day.sql | 2 +- example/models/waiters.sql | 2 +- example/tests/test_orders.yaml | 91 ------------- sqlmesh/core/engine_adapter.py | 2 + sqlmesh/core/model.py | 14 +- sqlmesh/core/snapshot.py | 5 +- tests/core/test_context.py | 123 +++++------------- tests/core/test_dag.py | 2 +- tests/core/test_integration.py | 26 ++-- tests/core/test_model.py | 9 -- tests/core/test_snapshot.py | 2 +- 20 files changed, 72 insertions(+), 279 deletions(-) rename example/models/{raw_items.py => items.py} (98%) delete mode 100644 example/models/items.sql rename example/models/{raw_order_items.py => order_items.py} (92%) delete mode 100644 example/models/order_items.sql rename example/models/{raw_orders.py => orders.py} (96%) delete mode 100644 example/models/orders.sql delete mode 100644 example/tests/test_orders.yaml diff --git a/example/models/customer_revenue_by_day.sql b/example/models/customer_revenue_by_day.sql index 6a86037ace..a762e0e4a3 100644 --- a/example/models/customer_revenue_by_day.sql +++ b/example/models/customer_revenue_by_day.sql @@ -32,4 +32,4 @@ WHERE o.ds BETWEEN @start_ds AND @end_ds GROUP BY o.customer_id, - o.ds \ No newline at end of file + o.ds diff --git a/example/models/customers.sql b/example/models/customers.sql index 12512212b1..534e856fb4 100644 --- a/example/models/customers.sql +++ b/example/models/customers.sql @@ -7,4 +7,4 @@ MODEL ( SELECT DISTINCT customer_id::INT AS customer_id -FROM sushi.orders AS o \ No newline at end of file +FROM sushi.orders AS o diff --git a/example/models/raw_items.py b/example/models/items.py similarity index 98% rename from example/models/raw_items.py rename to example/models/items.py index 9f8c7201e9..6fa432d333 100644 --- a/example/models/raw_items.py +++ b/example/models/items.py @@ -47,7 +47,7 @@ @model( """ MODEL( - name raw.items, + name sushi.items, kind incremental, time_column ds, start 'Jan 1 2022', diff --git a/example/models/items.sql b/example/models/items.sql deleted file mode 100644 index cc20cf9a3a..0000000000 --- a/example/models/items.sql +++ /dev/null @@ -1,19 +0,0 @@ -/* Table of daily sushi prices. */ -MODEL ( - name sushi.items, - kind incremental, - dialect duckdb, - owner jen, - cron '@daily', - batch_size 30, - time_column (ds, '%Y-%m-%d') -); - -SELECT - id AS id, /* Primary key */ - name::TEXT AS name, /* Name of the sushi */ - price::DOUBLE AS price, /* Price of the sushi */ - ds::TEXT AS ds /* Date */ -FROM raw.items -WHERE - ds BETWEEN @start_ds AND @end_ds diff --git a/example/models/raw_order_items.py b/example/models/order_items.py similarity index 92% rename from example/models/raw_order_items.py rename to example/models/order_items.py index 60e5a1ef4b..98c77a0bfc 100644 --- a/example/models/raw_order_items.py +++ b/example/models/order_items.py @@ -12,10 +12,10 @@ @model( """ MODEL( - name raw.order_items, + name sushi.order_items, kind incremental, time_column ds, - depends_on [raw.orders, raw.items], + depends_on [sushi.orders, sushi.items], cron '@daily', batch_size 30, columns ( @@ -40,7 +40,7 @@ def execute( dfs = [] raw_orders = ( - snapshots["raw.orders"].table_name if snapshots else mapping["raw.orders"] + snapshots["sushi.orders"].table_name if snapshots else mapping["sushi.orders"] ) for dt in iter_dates(start, end): diff --git a/example/models/order_items.sql b/example/models/order_items.sql deleted file mode 100644 index eaf9269f03..0000000000 --- a/example/models/order_items.sql +++ /dev/null @@ -1,18 +0,0 @@ -/* Join table of sushi orders and items. */ -MODEL ( - name sushi.order_items, - owner jen, - cron '@daily', - batch_size 30, - time_column ds -); - -SELECT - id::INT AS id, /* Primary key */ - order_id::INT AS order_id, /* Order id */ - item_id::INT AS item_id, /* Item id */ - quantity::INT AS quantity, /* Quantity of items ordered */ - ds::TEXT AS ds /* Date of order */ -FROM raw.order_items -WHERE - ds BETWEEN @start_ds AND @end_ds diff --git a/example/models/raw_orders.py b/example/models/orders.py similarity index 96% rename from example/models/raw_orders.py rename to example/models/orders.py index 1363bf06a3..0156a40810 100644 --- a/example/models/raw_orders.py +++ b/example/models/orders.py @@ -13,8 +13,9 @@ @model( """ + -- Table of sushi orders. MODEL( - name raw.orders, + name sushi.orders, kind incremental, time_column ds, start '2022-01-01', diff --git a/example/models/orders.sql b/example/models/orders.sql deleted file mode 100644 index c8f0239711..0000000000 --- a/example/models/orders.sql +++ /dev/null @@ -1,19 +0,0 @@ -/* Table of sushi orders. */ -MODEL ( - name sushi.orders, - owner jen, - cron '@daily', - batch_size 30, - time_column ds -); - -SELECT - id::INT AS id, /* Primary key */ - customer_id::INT AS customer_id, /* Id of customer who made the order */ - waiter_id::INT AS waiter_id, /* Id of waiter who took the order */ - start_ts::TEXT AS start_ts, /* Start timestamp */ - end_ts::TEXT AS end_ts, /* End timestamp */ - ds::TEXT AS ds /* Date of order */ -FROM raw.orders -WHERE - ds BETWEEN @start_ds AND @end_ds diff --git a/example/models/top_waiters.sql b/example/models/top_waiters.sql index 0d207c916a..2cd74fc99a 100644 --- a/example/models/top_waiters.sql +++ b/example/models/top_waiters.sql @@ -17,4 +17,4 @@ WHERE ) ORDER BY revenue DESC -LIMIT 10 \ No newline at end of file +LIMIT 10 diff --git a/example/models/waiter_revenue_by_day.sql b/example/models/waiter_revenue_by_day.sql index 7eae4e420f..b428ad264d 100644 --- a/example/models/waiter_revenue_by_day.sql +++ b/example/models/waiter_revenue_by_day.sql @@ -20,4 +20,4 @@ WHERE o.ds BETWEEN @start_ds AND @end_ds GROUP BY o.waiter_id, - o.ds \ No newline at end of file + o.ds diff --git a/example/models/waiters.sql b/example/models/waiters.sql index 10d0e8d2b7..e7beb4f369 100644 --- a/example/models/waiters.sql +++ b/example/models/waiters.sql @@ -11,4 +11,4 @@ SELECT DISTINCT ds::TEXT AS ds FROM sushi.orders AS o WHERE - ds BETWEEN @start_ds AND @end_ds \ No newline at end of file + ds BETWEEN @start_ds AND @end_ds diff --git a/example/tests/test_orders.yaml b/example/tests/test_orders.yaml deleted file mode 100644 index 1f6996b6cb..0000000000 --- a/example/tests/test_orders.yaml +++ /dev/null @@ -1,91 +0,0 @@ -test_single_order: - model: sushi.orders - inputs: - raw.orders: - rows: - - id: 1 - customer_id: 1 - waiter_id: 1 - start_ts: 2022-01-01 01:59:00 - end_ts: 2022-01-01 02:29:00 - ds: 2022-01-01 - outputs: - vars: - start: 2022-01-01 - end: 2022-01-01 - latest: 2022-01-01 - query: - rows: - - id: 1 - customer_id: 1 - waiter_id: 1 - start_ts: '2022-01-01 01:59:00' - end_ts: '2022-01-01 02:29:00' - ds: '2022-01-01' -test_multiple_orders: - model: sushi.orders - inputs: - raw.orders: - rows: - - id: 1 - customer_id: 1 - waiter_id: 1 - start_ts: 2022-01-01 01:59:00 - end_ts: 2022-01-01 02:29:00 - ds: 2022-01-01 - - id: 2 - customer_id: 2 - waiter_id: 2 - start_ts: 2022-01-01 01:59:00 - end_ts: 2022-01-01 02:29:00 - ds: 2022-01-01 - outputs: - vars: - start: 2022-01-01 - end: 2022-01-01 - latest: 2022-01-01 - query: - rows: - - id: 1 - customer_id: 1 - waiter_id: 1 - start_ts: '2022-01-01 01:59:00' - end_ts: '2022-01-01 02:29:00' - ds: '2022-01-01' - - id: 2 - customer_id: 2 - waiter_id: 2 - start_ts: '2022-01-01 01:59:00' - end_ts: '2022-01-01 02:29:00' - ds: '2022-01-01' - -test_multiple_ds: - model: sushi.orders - inputs: - raw.orders: - rows: - - id: 1 - customer_id: 1 - waiter_id: 1 - start_ts: 2022-01-01 01:59:00 - end_ts: 2022-01-01 02:29:00 - ds: 2022-01-01 - - id: 2 - customer_id: 2 - waiter_id: 2 - start_ts: 2022-01-02 01:59:00 - end_ts: 2022-01-02 02:29:00 - ds: 2022-01-02 - outputs: - vars: - start: 2022-01-01 - end: 2022-01-01 - latest: 2022-01-01 - query: - rows: - - id: 1 - customer_id: 1 - waiter_id: 1 - start_ts: '2022-01-01 01:59:00' - end_ts: '2022-01-01 02:29:00' - ds: '2022-01-01' diff --git a/sqlmesh/core/engine_adapter.py b/sqlmesh/core/engine_adapter.py index 2bc03d3736..3bc5d7e5b9 100644 --- a/sqlmesh/core/engine_adapter.py +++ b/sqlmesh/core/engine_adapter.py @@ -92,6 +92,8 @@ def replace_query(self, table_name: str, query_or_df: QueryOrDF): if self.supports_partitions: self.insert_overwrite(table_name, query_or_df) else: + if isinstance(query_or_df, pd.DataFrame): + query_or_df = next(pandas_to_sql(query_or_df)) self.execute( exp.Create( kind="TABLE", diff --git a/sqlmesh/core/model.py b/sqlmesh/core/model.py index 55084de5a7..fe5e381174 100644 --- a/sqlmesh/core/model.py +++ b/sqlmesh/core/model.py @@ -291,6 +291,12 @@ exp.to_identifier(value[0]) if len(value) == 1 else exp.Tuple(expressions=value) ), "time_column": lambda value: value.expression, + "depends_on_": lambda value: exp.Tuple(expressions=value), + "columns_": lambda value: exp.Schema( + expressions=[ + exp.ColumnDef(this=exp.to_column(c), kind=t) for c, t in value.items() + ] + ), } EXEC_PREFIX = "__EXEC__ " @@ -431,7 +437,7 @@ def _columns_validator(cls, v: t.Any) -> t.Optional[t.Dict[str, exp.DataType]]: @validator("depends_on_", pre=True) def _depends_on_validator(cls, v: t.Any) -> t.Optional[t.Set[str]]: - if isinstance(v, exp.Array): + if isinstance(v, (exp.Array, exp.Tuple)): return { exp.table_name(table.name if table.is_string else table.sql()) for table in v.expressions @@ -836,7 +842,7 @@ def _render_query( expand = set(expand) | {name for name in snapshots if name not in mapping} if key not in self._query_cache: - if self.is_sql: + if query_ or self.is_sql: macro_evaluator = MacroEvaluator( dialect or self.dialect, self.python_env ) @@ -865,7 +871,6 @@ def _expand(node: exp.Expression) -> exp.Expression: model = snapshots[name].model if name in snapshots else None if name in expand and model: return model._render_query( - model.query, start=start, end=end, latest=latest, @@ -1205,6 +1210,9 @@ def __init__(self, definition: str = "", **kwargs): self.meta = ModelMeta( **{ "depends_on": set(), + "description": "\n".join(comment.strip() for comment in meta.comments) + if meta + else None, **( {prop.name: prop.args.get("value") for prop in meta.expressions} if meta diff --git a/sqlmesh/core/snapshot.py b/sqlmesh/core/snapshot.py index 0f6e2ad6b4..09347f851d 100644 --- a/sqlmesh/core/snapshot.py +++ b/sqlmesh/core/snapshot.py @@ -594,9 +594,8 @@ def fingerprint_from_model( if model.name not in cache: data = [ - model.render_query().sql(identify=True, comments=False) - if model.is_sql - else str(sorted(model.python_env.items())), + model.render_query().sql(identify=True, comments=False), + str(sorted(model.python_env.items())), model.kind.name, model.cron, model.storage_format, diff --git a/tests/core/test_context.py b/tests/core/test_context.py index 942611c47a..50032e43e2 100644 --- a/tests/core/test_context.py +++ b/tests/core/test_context.py @@ -75,11 +75,8 @@ def test_custom_macros(sushi_context): def test_dag(sushi_context): assert set(sushi_context.dag.upstream("sushi.customer_revenue_by_day")) == { - "raw.items", "sushi.items", - "raw.orders", "sushi.orders", - "raw.order_items", "sushi.order_items", } @@ -102,58 +99,28 @@ def test_render(sushi_context, assert_exp_eq, mock_file_cache): CAST(o.ds AS TEXT) AS ds /* Date */ FROM ( SELECT - CAST(id AS INT) AS id, /* Primary key */ - CAST(customer_id AS INT) AS customer_id, /* Id of customer who made the order */ - CAST(waiter_id AS INT) AS waiter_id, /* Id of waiter who took the order */ - CAST(start_ts AS TEXT) AS start_ts, /* Start timestamp */ - CAST(end_ts AS TEXT) AS end_ts, /* End timestamp */ - CAST(ds AS TEXT) AS ds /* Date of order */ - FROM ( - SELECT - CAST(NULL AS INT) AS id, - CAST(NULL AS INT) AS customer_id, - CAST(NULL AS INT) AS waiter_id, - CAST(NULL AS INT) AS start_ts, - CAST(NULL AS INT) AS end_ts, - CAST(NULL AS TEXT) AS ds - ) AS orders - WHERE - ds <= '2021-01-01' AND ds >= '2021-01-01' + CAST(NULL AS INT) AS id, + CAST(NULL AS INT) AS customer_id, + CAST(NULL AS INT) AS waiter_id, + CAST(NULL AS INT) AS start_ts, + CAST(NULL AS INT) AS end_ts, + CAST(NULL AS TEXT) AS ds ) AS o LEFT JOIN ( SELECT - CAST(id AS INT) AS id, /* Primary key */ - CAST(order_id AS INT) AS order_id, /* Order id */ - CAST(item_id AS INT) AS item_id, /* Item id */ - CAST(quantity AS INT) AS quantity, /* Quantity of items ordered */ - CAST(ds AS TEXT) AS ds /* Date of order */ - FROM ( - SELECT - CAST(NULL AS INT) AS id, - CAST(NULL AS INT) AS order_id, - CAST(NULL AS INT) AS item_id, - CAST(NULL AS INT) AS quantity, - CAST(NULL AS TEXT) AS ds - ) AS order_items - WHERE - ds <= '2021-01-01' AND ds >= '2021-01-01' + CAST(NULL AS INT) AS id, + CAST(NULL AS INT) AS order_id, + CAST(NULL AS INT) AS item_id, + CAST(NULL AS INT) AS quantity, + CAST(NULL AS TEXT) AS ds ) AS oi ON o.ds = oi.ds AND o.id = oi.order_id LEFT JOIN ( SELECT - id AS id, /* Primary key */ - CAST(name AS TEXT) AS name, /* Name of the sushi */ - CAST(price AS DOUBLE) AS price, /* Price of the sushi */ - CAST(ds AS TEXT) AS ds /* Date */ - FROM ( - SELECT - CAST(NULL AS INT) AS id, - CAST(NULL AS TEXT) AS name, - CAST(NULL AS DOUBLE) AS price, - CAST(NULL AS TEXT) AS ds - ) AS items - WHERE - ds <= '2021-01-01' AND ds >= '2021-01-01' + CAST(NULL AS INT) AS id, + CAST(NULL AS TEXT) AS name, + CAST(NULL AS DOUBLE) AS price, + CAST(NULL AS TEXT) AS ds ) AS i ON oi.ds = i.ds AND oi.item_id = i.id WHERE @@ -204,58 +171,28 @@ def test_render(sushi_context, assert_exp_eq, mock_file_cache): CAST(o.ds AS TEXT) AS ds /* Date */ FROM ( SELECT - CAST(id AS INT) AS id, /* Primary key */ - CAST(customer_id AS INT) AS customer_id, /* Id of customer who made the order */ - CAST(waiter_id AS INT) AS waiter_id, /* Id of waiter who took the order */ - CAST(start_ts AS TEXT) AS start_ts, /* Start timestamp */ - CAST(end_ts AS TEXT) AS end_ts, /* End timestamp */ - CAST(ds AS TEXT) AS ds /* Date of order */ - FROM ( - SELECT - CAST(NULL AS INT) AS id, - CAST(NULL AS INT) AS customer_id, - CAST(NULL AS INT) AS waiter_id, - CAST(NULL AS INT) AS start_ts, - CAST(NULL AS INT) AS end_ts, - CAST(NULL AS TEXT) AS ds - ) AS orders - WHERE - ds <= '1970-01-01' AND ds >= '1970-01-01' + CAST(NULL AS INT) AS id, + CAST(NULL AS INT) AS customer_id, + CAST(NULL AS INT) AS waiter_id, + CAST(NULL AS INT) AS start_ts, + CAST(NULL AS INT) AS end_ts, + CAST(NULL AS TEXT) AS ds ) AS o LEFT JOIN ( SELECT - CAST(id AS INT) AS id, /* Primary key */ - CAST(order_id AS INT) AS order_id, /* Order id */ - CAST(item_id AS INT) AS item_id, /* Item id */ - CAST(quantity AS INT) AS quantity, /* Quantity of items ordered */ - CAST(ds AS TEXT) AS ds /* Date of order */ - FROM ( - SELECT - CAST(NULL AS INT) AS id, - CAST(NULL AS INT) AS order_id, - CAST(NULL AS INT) AS item_id, - CAST(NULL AS INT) AS quantity, - CAST(NULL AS TEXT) AS ds - ) AS order_items - WHERE - ds <= '1970-01-01' AND ds >= '1970-01-01' + CAST(NULL AS INT) AS id, + CAST(NULL AS INT) AS order_id, + CAST(NULL AS INT) AS item_id, + CAST(NULL AS INT) AS quantity, + CAST(NULL AS TEXT) AS ds ) AS oi ON o.ds = oi.ds AND o.id = oi.order_id LEFT JOIN ( SELECT - id AS id, /* Primary key */ - CAST(name AS TEXT) AS name, /* Name of the sushi */ - CAST(price AS DOUBLE) AS price, /* Price of the sushi */ - CAST(ds AS TEXT) AS ds /* Date */ - FROM ( - SELECT - CAST(NULL AS INT) AS id, - CAST(NULL AS TEXT) AS name, - CAST(NULL AS DOUBLE) AS price, - CAST(NULL AS TEXT) AS ds - ) AS items - WHERE - ds <= '1970-01-01' AND ds >= '1970-01-01' + CAST(NULL AS INT) AS id, + CAST(NULL AS TEXT) AS name, + CAST(NULL AS DOUBLE) AS price, + CAST(NULL AS TEXT) AS ds ) AS i ON oi.ds = i.ds AND oi.item_id = i.id WHERE diff --git a/tests/core/test_dag.py b/tests/core/test_dag.py index 7b7be85afa..e3a71db6cf 100644 --- a/tests/core/test_dag.py +++ b/tests/core/test_dag.py @@ -12,7 +12,7 @@ def test_no_downstream(sushi_context): def test_lineage(sushi_context): lineage = sushi_context.dag.lineage("sushi.order_items").sort() - assert lineage.index("sushi.order_items") > lineage.index("raw.order_items") + assert lineage.index("sushi.order_items") > lineage.index("sushi.items") assert lineage.index("sushi.customer_revenue_by_day") > lineage.index( "sushi.order_items" ) diff --git a/tests/core/test_integration.py b/tests/core/test_integration.py index 90bbc0888d..2a25cbfaf6 100644 --- a/tests/core/test_integration.py +++ b/tests/core/test_integration.py @@ -3,7 +3,6 @@ import pytest from pytest_mock.plugin import MockerFixture -from sqlglot import parse_one from sqlglot.expressions import DataType from sqlmesh.core.context import Context @@ -150,6 +149,7 @@ def validate_query_change( directly_modified = ["sushi.items"] indirectly_modified = [ + "sushi.order_items", "sushi.waiter_revenue_by_day", "sushi.customer_revenue_by_day", "sushi.top_waiters", @@ -228,13 +228,8 @@ def test_model_kind_change(from_: ModelKind, to: ModelKind, sushi_context: Conte def change_model_kind(context: Context, kind: ModelKind): if kind in (ModelKind.VIEW, ModelKind.EMBEDDED, ModelKind.FULL): - sushi_items_query_no_dates = parse_one( - "SELECT id::INT AS id, name::TEXT AS name, price::DOUBLE AS price, ds::TEXT AS ds FROM raw.items", - read=context.dialect, - ) context.upsert_model( "sushi.items", - query=sushi_items_query_no_dates, partitioned_by=None, audits={}, ) @@ -250,6 +245,7 @@ def validate_model_kind_change( ): directly_modified = ["sushi.items"] indirectly_modified = [ + "sushi.order_items", "sushi.waiter_revenue_by_day", "sushi.customer_revenue_by_day", "sushi.top_waiters", @@ -288,6 +284,7 @@ def test_environment_isolation(sushi_context: Context): ) directly_modified = ["sushi.items"] indirectly_modified = [ + "sushi.order_items", "sushi.waiter_revenue_by_day", "sushi.customer_revenue_by_day", "sushi.top_waiters", @@ -395,7 +392,6 @@ def test_no_override(sushi_context: Context) -> None: plan.set_choice(items, SnapshotChangeCategory.BREAKING) plan.set_choice(order_items, SnapshotChangeCategory.NON_BREAKING) assert items.is_new_version - print(waiter_revenue.fingerprint, waiter_revenue.version) assert waiter_revenue.is_new_version plan.set_choice(items, SnapshotChangeCategory.NON_BREAKING) assert not waiter_revenue.is_new_version @@ -490,6 +486,7 @@ def setup_rebase( assert plan.categorized == [context.snapshots["sushi.items"]] assert plan.indirectly_modified == { "sushi.items": { + "sushi.order_items", "sushi.waiter_revenue_by_day", "sushi.top_waiters", "sushi.customer_revenue_by_day", @@ -685,12 +682,17 @@ def change_data_type( ) -> None: model = context.models[model_name] - data_types = model.query.find_all(DataType) - for data_type in data_types: - if data_type.this == old_type: - data_type.set("this", new_type) + if model.is_sql: + data_types = model.query.find_all(DataType) + for data_type in data_types: + if data_type.this == old_type: + data_type.set("this", new_type) + else: + for k, v in model.columns_.items(): + if v.this == old_type: + model.columns_[k] = DataType.build(new_type) - context.upsert_model(model_name, query=model.query) + context.upsert_model(model_name, query=model.query, columns=model.columns_) def validate_plan_changes( diff --git a/tests/core/test_model.py b/tests/core/test_model.py index 38f70b17ba..8dbcb41af7 100644 --- a/tests/core/test_model.py +++ b/tests/core/test_model.py @@ -217,15 +217,6 @@ def test_json_serde(): def test_column_descriptions(sushi_context, assert_exp_eq): - assert sushi_context.models["sushi.orders"].column_descriptions == { - "id": "Primary key", - "customer_id": "Id of customer who made the order", - "waiter_id": "Id of waiter who took the order", - "start_ts": "Start timestamp", - "end_ts": "End timestamp", - "ds": "Date of order", - } - assert sushi_context.models[ "sushi.customer_revenue_by_day" ].column_descriptions == { diff --git a/tests/core/test_snapshot.py b/tests/core/test_snapshot.py index 57e1175c56..0c4f6195d5 100644 --- a/tests/core/test_snapshot.py +++ b/tests/core/test_snapshot.py @@ -229,7 +229,7 @@ def test_remove_intervals(snapshot: Snapshot): def test_fingerprint(model: Model, parent_model: Model): macro.get_registry() fingerprint = fingerprint_from_model(model, models={}) - original_fingerprint = "4168321870_0" + original_fingerprint = "2774500705_0" assert fingerprint == original_fingerprint assert fingerprint_from_model(model, physical_schema="x", models={}) != fingerprint