From ab7c4394bdc8a940c045143340bcbc82cbc5de2f Mon Sep 17 00:00:00 2001 From: Kshitij Aranke Date: Tue, 26 Sep 2023 16:02:21 +0100 Subject: [PATCH] Backport "Fix #8509: Support doc blocks in nested semantic model YAML (#8709)" --- .../unreleased/Fixes-20230926-001527.yaml | 6 + core/dbt/parser/manifest.py | 30 ++ core/dbt/parser/schema_renderer.py | 2 +- tests/functional/semantic_models/fixtures.py | 284 ++++++++++++++++++ .../semantic_models/test_semantic_models.py | 73 +++++ 5 files changed, 394 insertions(+), 1 deletion(-) create mode 100644 .changes/unreleased/Fixes-20230926-001527.yaml create mode 100644 tests/functional/semantic_models/fixtures.py create mode 100644 tests/functional/semantic_models/test_semantic_models.py diff --git a/.changes/unreleased/Fixes-20230926-001527.yaml b/.changes/unreleased/Fixes-20230926-001527.yaml new file mode 100644 index 00000000000..53d6b9151fd --- /dev/null +++ b/.changes/unreleased/Fixes-20230926-001527.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: Support doc blocks in nested semantic model YAML +time: 2023-09-26T00:15:27.328363+01:00 +custom: + Author: aranke + Issue: "8509" diff --git a/core/dbt/parser/manifest.py b/core/dbt/parser/manifest.py index 4ac3732bac4..1d99852f35c 100644 --- a/core/dbt/parser/manifest.py +++ b/core/dbt/parser/manifest.py @@ -98,6 +98,7 @@ ResultNode, ModelNode, NodeRelation, + SemanticModel, ) from dbt.contracts.graph.unparsed import NodeVersion from dbt.contracts.util import Writable @@ -1169,6 +1170,16 @@ def process_docs(self, config: RuntimeConfig): config.project_name, ) _process_docs_for_metrics(ctx, metric) + for semantic_model in self.manifest.semantic_models.values(): + if semantic_model.created_at < self.started_at: + continue + ctx = generate_runtime_docs_context( + config, + semantic_model, + self.manifest, + config.project_name, + ) + _process_docs_for_semantic_model(ctx, semantic_model) # Loops through all nodes and exposures, for each element in # 'sources' array finds the source node and updates the @@ -1398,6 +1409,25 @@ def _process_docs_for_metrics(context: Dict[str, Any], metric: Metric) -> None: metric.description = get_rendered(metric.description, context) +def _process_docs_for_semantic_model( + context: Dict[str, Any], semantic_model: SemanticModel +) -> None: + if semantic_model.description: + semantic_model.description = get_rendered(semantic_model.description, context) + + for dimension in semantic_model.dimensions: + if dimension.description: + dimension.description = get_rendered(dimension.description, context) + + for measure in semantic_model.measures: + if measure.description: + measure.description = get_rendered(measure.description, context) + + for entity in semantic_model.entities: + if entity.description: + entity.description = get_rendered(entity.description, context) + + def _process_refs( manifest: Manifest, current_project: str, node, dependencies: Optional[Mapping[str, Project]] ) -> None: diff --git a/core/dbt/parser/schema_renderer.py b/core/dbt/parser/schema_renderer.py index e0c54f247da..66b91fee1b4 100644 --- a/core/dbt/parser/schema_renderer.py +++ b/core/dbt/parser/schema_renderer.py @@ -42,7 +42,7 @@ def _is_norender_key(self, keypath: Keypath) -> bool: if ( len(keypath) >= 3 - and keypath[0] == "columns" + and keypath[0] in ("columns", "dimensions", "measures", "entities") and keypath[2] in ("tests", "description") ): return True diff --git a/tests/functional/semantic_models/fixtures.py b/tests/functional/semantic_models/fixtures.py new file mode 100644 index 00000000000..3fb65a3a4fb --- /dev/null +++ b/tests/functional/semantic_models/fixtures.py @@ -0,0 +1,284 @@ +simple_metricflow_time_spine_sql = """ +SELECT to_date('02/20/2023', 'mm/dd/yyyy') as date_day +""" + +models_people_sql = """ +select 1 as id, 'Drew' as first_name, 'Banin' as last_name, 'yellow' as favorite_color, true as loves_dbt, 5 as tenure, current_timestamp as created_at +union all +select 2 as id, 'Jeremy' as first_name, 'Cohen' as last_name, 'indigo' as favorite_color, true as loves_dbt, 4 as tenure, current_timestamp as created_at +union all +select 3 as id, 'Callum' as first_name, 'McCann' as last_name, 'emerald' as favorite_color, true as loves_dbt, 0 as tenure, current_timestamp as created_at +""" + +groups_yml = """ +version: 2 + +groups: + - name: some_group + owner: + email: me@gmail.com + - name: some_other_group + owner: + email: me@gmail.com +""" + +models_people_metrics_yml = """ +version: 2 + +metrics: + - name: number_of_people + label: "Number of people" + description: Total count of people + type: simple + type_params: + measure: people + meta: + my_meta: 'testing' +""" + +disabled_models_people_metrics_yml = """ +version: 2 + +metrics: + - name: number_of_people + config: + enabled: false + group: some_group + label: "Number of people" + description: Total count of people + type: simple + type_params: + measure: people + meta: + my_meta: 'testing' +""" + +semantic_model_people_yml = """ +version: 2 + +semantic_models: + - name: semantic_people + label: "Semantic People" + model: ref('people') + dimensions: + - name: favorite_color + label: "Favorite Color" + type: categorical + - name: created_at + label: "Created At" + type: TIME + type_params: + time_granularity: day + measures: + - name: years_tenure + label: "Years Tenure" + agg: SUM + expr: tenure + - name: people + label: "People" + agg: count + expr: id + entities: + - name: id + label: "Primary ID" + type: primary + defaults: + agg_time_dimension: created_at +""" + +semantic_model_descriptions = """ +{% docs semantic_model_description %} foo {% enddocs %} +{% docs dimension_description %} bar {% enddocs %} +{% docs measure_description %} baz {% enddocs %} +{% docs entity_description %} qux {% enddocs %} +""" + +semantic_model_people_yml_with_docs = """ +version: 2 + +semantic_models: + - name: semantic_people + model: ref('people') + description: "{{ doc('semantic_model_description') }}" + dimensions: + - name: favorite_color + type: categorical + description: "{{ doc('dimension_description') }}" + - name: created_at + type: TIME + type_params: + time_granularity: day + measures: + - name: years_tenure + agg: SUM + expr: tenure + description: "{{ doc('measure_description') }}" + - name: people + agg: count + expr: id + entities: + - name: id + description: "{{ doc('entity_description') }}" + type: primary + defaults: + agg_time_dimension: created_at +""" + +enabled_semantic_model_people_yml = """ +version: 2 + +semantic_models: + - name: semantic_people + label: "Semantic People" + model: ref('people') + config: + enabled: true + group: some_group + dimensions: + - name: favorite_color + type: categorical + - name: created_at + type: TIME + type_params: + time_granularity: day + measures: + - name: years_tenure + agg: SUM + expr: tenure + - name: people + agg: count + expr: id + entities: + - name: id + type: primary + defaults: + agg_time_dimension: created_at +""" + +disabled_semantic_model_people_yml = """ +version: 2 + +semantic_models: + - name: semantic_people + label: "Semantic People" + model: ref('people') + config: + enabled: false + dimensions: + - name: favorite_color + type: categorical + - name: created_at + type: TIME + type_params: + time_granularity: day + measures: + - name: years_tenure + agg: SUM + expr: tenure + - name: people + agg: count + expr: id + entities: + - name: id + type: primary + defaults: + agg_time_dimension: created_at +""" + +schema_yml = """models: + - name: fct_revenue + description: This is the model fct_revenue. It should be able to use doc blocks + +semantic_models: + - name: revenue + description: This is the revenue semantic model. It should be able to use doc blocks + model: ref('fct_revenue') + + defaults: + agg_time_dimension: ds + + measures: + - name: txn_revenue + expr: revenue + agg: sum + agg_time_dimension: ds + create_metric: true + - name: sum_of_things + expr: 2 + agg: sum + agg_time_dimension: ds + - name: has_revenue + expr: true + agg: sum_boolean + agg_time_dimension: ds + - name: discrete_order_value_p99 + expr: order_total + agg: percentile + agg_time_dimension: ds + agg_params: + percentile: 0.99 + use_discrete_percentile: True + use_approximate_percentile: False + - name: test_agg_params_optional_are_empty + expr: order_total + agg: percentile + agg_time_dimension: ds + agg_params: + percentile: 0.99 + - name: test_non_additive + expr: txn_revenue + agg: sum + non_additive_dimension: + name: ds + window_choice: max + + dimensions: + - name: ds + type: time + expr: created_at + type_params: + time_granularity: day + + entities: + - name: user + type: foreign + expr: user_id + - name: id + type: primary + +metrics: + - name: simple_metric + label: Simple Metric + type: simple + type_params: + measure: sum_of_things +""" + +schema_without_semantic_model_yml = """models: + - name: fct_revenue + description: This is the model fct_revenue. It should be able to use doc blocks +""" + +fct_revenue_sql = """select + 1 as id, + 10 as user_id, + 1000 as revenue, + current_timestamp as created_at""" + +metricflow_time_spine_sql = """ +with days as ( + {{dbt_utils.date_spine('day' + , "to_date('01/01/2000','mm/dd/yyyy')" + , "to_date('01/01/2027','mm/dd/yyyy')" + ) + }} +), + +final as ( + select cast(date_day as date) as date_day + from days +) + +select * +from final +""" diff --git a/tests/functional/semantic_models/test_semantic_models.py b/tests/functional/semantic_models/test_semantic_models.py new file mode 100644 index 00000000000..11fdfc32456 --- /dev/null +++ b/tests/functional/semantic_models/test_semantic_models.py @@ -0,0 +1,73 @@ +import pytest + +from dbt.contracts.graph.manifest import Manifest +from dbt.exceptions import CompilationError +from dbt.tests.util import run_dbt +from tests.functional.semantic_models.fixtures import ( + models_people_sql, + simple_metricflow_time_spine_sql, + semantic_model_people_yml, + models_people_metrics_yml, + semantic_model_people_yml_with_docs, + semantic_model_descriptions, +) + + +class TestSemanticModelDependsOn: + @pytest.fixture(scope="class") + def models(self): + return { + "people.sql": models_people_sql, + "metricflow_time_spine.sql": simple_metricflow_time_spine_sql, + "semantic_models.yml": semantic_model_people_yml, + "people_metrics.yml": models_people_metrics_yml, + } + + def test_depends_on(self, project): + manifest = run_dbt(["parse"]) + assert isinstance(manifest, Manifest) + + expected_depends_on_for_people_semantic_model = ["model.test.people"] + + number_of_people_metric = manifest.semantic_models["semantic_model.test.semantic_people"] + assert ( + number_of_people_metric.depends_on.nodes + == expected_depends_on_for_people_semantic_model + ) + + +class TestSemanticModelNestedDocs: + @pytest.fixture(scope="class") + def models(self): + return { + "people.sql": models_people_sql, + "metricflow_time_spine.sql": simple_metricflow_time_spine_sql, + "semantic_models.yml": semantic_model_people_yml_with_docs, + "people_metrics.yml": models_people_metrics_yml, + "docs.md": semantic_model_descriptions, + } + + def test_depends_on(self, project): + manifest = run_dbt(["parse"]) + node = manifest.semantic_models["semantic_model.test.semantic_people"] + + assert node.description == "foo" + assert node.dimensions[0].description == "bar" + assert node.measures[0].description == "baz" + assert node.entities[0].description == "qux" + + +class TestSemanticModelUnknownModel: + @pytest.fixture(scope="class") + def models(self): + return { + "not_people.sql": models_people_sql, + "metricflow_time_spine.sql": simple_metricflow_time_spine_sql, + "semantic_models.yml": semantic_model_people_yml, + "people_metrics.yml": models_people_metrics_yml, + } + + def test_unknown_model_raises_issue(self, project): + with pytest.raises(CompilationError) as excinfo: + run_dbt(["parse"]) + assert "depends on a node named 'people' which was not found" in str(excinfo.value)