Merge pull request #143 from Snowflake-Labs/jsummer/add-looker-support

Jsummer/add looker support

README.md

@@ -7,6 +7,25 @@ Please complete the instructions in [setup](#setup), then proceed to the instruc
 
 If you want to see what a semantic model looks like, skip to [Examples](#examples).
 
+## Table of Contents
+
+  * [Table of Contents](#table-of-contents)
+  * [Setup](#setup)
+  * [Streamlit App](#streamlit-app)
+  * [CLI Tool](#cli-tool)
+    + [Generation](#generation)
+    + [Validation](#validation)
+  * [Python](#python)
+    + [Generation](#generation-1)
+    + [Validation](#validation-1)
+  * [Usage](#usage)
+    + [Semantic Model Context Length Constraints](#semantic-model-context-length-constraints)
+    + [Auto-Generated Descriptions](#auto-generated-descriptions)
+    + [Additional Fields to Fill Out](#additional-fields-to-fill-out)
+  * [Examples](#examples)
+  * [Release](#release)
+
+
 ## Setup
 
 We currently leverage credentials saved as environment variables.
@@ -279,6 +298,19 @@ In addition, consider adding the following elements to your semantic model:
 2. Synonyms. Any additional synonyms for column names.
 3. Filters. Additional filters with their relevant `expr`.
 
+### Partner Semantic Support
+
+We continue to add support for partner semantic and metric layers. Our aim is to expedite the creation of Cortex Analyst semantic files using logic and metadata from partner tools.
+Please see below for details about current partner support. 
+
+**IMPORTANT**: Use the [Streamlit App](#streamlit-app) to leverage existing partner semantic/metric layers. 
+
+| Tool     | Method  | Requirements  |
+| -------- | ------- | ------- |
+| DBT      | We extract and translate metadata from [semantic_models](https://docs.getdbt.com/docs/build/semantic-models#semantic-models-components) in uploaded DBT yaml file(s) and merge with a generated Cortex Analyst semantic file table-by-table.    |  DBT models and sources leading up to the semantic model layer(s) must be tables/views in Snowflake.   |
+| Looker   |We materialize your Explore dataset in Looker as Snowflake table(s) and generate a Cortex Analyst semantic file. Metadata from your Explore fields can be merged with the generated Cortex Analyst semantic file. |  Looker Views referenced in the Looker Explores must be tables/views in Snowflake. Looker SDK credentials are required. Visit [Looker Authentication SDK Docs](https://cloud.google.com/looker/docs/api-auth#authentication_with_an_sdk) for more information. Install Looker's [API Explorer extension](https://cloud.google.com/looker/docs/api-explorer) from the Looker Marketplace to view API credentials directly.  |
+
+
 
 ## Examples
 

admin_apps/app.py

@@ -7,6 +7,7 @@
 from admin_apps.shared_utils import (  # noqa: E402
     GeneratorAppScreen,
     get_snowflake_connection,
+    set_sit_query_tag,
 )
 from semantic_model_generator.snowflake_utils.env_vars import (  # noqa: E402
     SNOWFLAKE_ACCOUNT_LOCATOR,
@@ -61,12 +62,19 @@ def verify_environment_setup() -> None:
 
 
 if __name__ == "__main__":
-    from admin_apps.journeys import builder, iteration
+    from admin_apps.journeys import builder, iteration, partner
 
     def onboarding_dialog() -> None:
         """
         Renders the initial screen where users can choose to create a new semantic model or edit an existing one.
         """
+
+        # Direct to specific page based instead of default onboarding if user comes from successful partner setup
+        if (
+            st.session_state.get("partner_setup", False)
+            and st.session_state.get("partner_tool", None) == "looker"
+        ):
+            builder.show()
         st.markdown(
             """
                 <div style="text-align: center;">
@@ -94,6 +102,18 @@ def onboarding_dialog() -> None:
                 type="primary",
             ):
                 iteration.show()
+            st.markdown("")
+            if st.button(
+                "**:package: Start with partner semantic model**",
+                use_container_width=True,
+                type="primary",
+            ):
+                set_sit_query_tag(
+                    get_snowflake_connection(),
+                    vendor="",
+                    action="start",
+                )
+                partner.show()
 
     verify_environment_setup()
 

admin_apps/journeys/builder.py

@@ -2,48 +2,18 @@
 from loguru import logger
 from snowflake.connector import ProgrammingError
 
-from admin_apps.shared_utils import GeneratorAppScreen, get_snowflake_connection
-from semantic_model_generator.generate_model import generate_model_str_from_snowflake
-from semantic_model_generator.snowflake_utils.snowflake_connector import (
-    fetch_databases,
-    fetch_schemas_in_database,
-    fetch_tables_views_in_schema,
+from admin_apps.shared_utils import (
+    GeneratorAppScreen,
+    format_snowflake_context,
+    get_available_databases,
+    get_available_schemas,
+    get_available_tables,
+    input_sample_value_num,
+    input_semantic_file_name,
+    run_generate_model_str_from_snowflake,
 )
 
 
-@st.cache_resource(show_spinner=False)
-def get_available_tables(schema: str) -> list[str]:
-    """
-    Simple wrapper around fetch_table_names to cache the results.
-
-    Returns: list of fully qualified table names
-    """
-
-    return fetch_tables_views_in_schema(get_snowflake_connection(), schema)
-
-
-@st.cache_resource(show_spinner=False)
-def get_available_schemas(db: str) -> list[str]:
-    """
-    Simple wrapper around fetch_schemas to cache the results.
-
-    Returns: list of schema names
-    """
-
-    return fetch_schemas_in_database(get_snowflake_connection(), db)
-
-
-@st.cache_resource(show_spinner=False)
-def get_available_databases() -> list[str]:
-    """
-    Simple wrapper around fetch_databases to cache the results.
-
-    Returns: list of database names
-    """
-
-    return fetch_databases(get_snowflake_connection())
-
-
 def update_schemas_and_tables() -> None:
     """
     Callback to run when the selected databases change. Ensures that if a database is deselected, the corresponding
@@ -103,16 +73,8 @@ def table_selector_dialog() -> None:
     st.write(
         "Please fill out the following fields to start building your semantic model."
     )
-    model_name = st.text_input(
-        "Semantic Model Name (no .yaml suffix)",
-        help="The name of the semantic model you are creating. This is separate from the filename, which we will set later.",
-    )
-    sample_values = st.selectbox(
-        "Maximum number of sample values per column",
-        list(range(1, 40)),
-        index=0,
-        help="NOTE: For dimensions, time measures, and measures, we enforce a minimum of 25, 3, and 3 sample values respectively.",
-    )
+    model_name = input_semantic_file_name()
+    sample_values = input_sample_value_num()
     st.markdown("")
 
     if "selected_databases" not in st.session_state:
@@ -133,6 +95,7 @@ def table_selector_dialog() -> None:
         placeholder="Select the databases that contain the tables you'd like to include in your semantic model.",
         on_change=update_schemas_and_tables,
         key="selected_databases",
+        # default=st.session_state.get("selected_databases", []),
     )
 
     st.multiselect(
@@ -141,35 +104,27 @@ def table_selector_dialog() -> None:
         placeholder="Select the schemas that contain the tables you'd like to include in your semantic model.",
         on_change=update_tables,
         key="selected_schemas",
+        format_func=lambda x: format_snowflake_context(x, -1),
     )
 
     st.multiselect(
         label="Tables",
         options=st.session_state.get("available_tables", []),
         placeholder="Select the tables you'd like to include in your semantic model.",
         key="selected_tables",
+        format_func=lambda x: format_snowflake_context(x, -1),
     )
 
     st.markdown("<div style='margin: 240px;'></div>", unsafe_allow_html=True)
     submit = st.button("Submit", use_container_width=True, type="primary")
     if submit:
-        if not model_name:
-            st.error("Please provide a name for your semantic model.")
-        elif not st.session_state["selected_tables"]:
-            st.error("Please select at least one table to proceed.")
-        else:
-            with st.spinner("Generating model. This may take a minute or two..."):
-                yaml_str = generate_model_str_from_snowflake(
-                    base_tables=st.session_state["selected_tables"],
-                    snowflake_account=st.session_state["account_name"],
-                    semantic_model_name=model_name,
-                    n_sample_values=sample_values,  # type: ignore
-                    conn=get_snowflake_connection(),
-                )
-
-                st.session_state["yaml"] = yaml_str
-                st.session_state["page"] = GeneratorAppScreen.ITERATION
-                st.rerun()
+        run_generate_model_str_from_snowflake(
+            model_name,
+            sample_values,
+            st.session_state["selected_tables"],
+        )
+        st.session_state["page"] = GeneratorAppScreen.ITERATION
+        st.rerun()
 
 
 def show() -> None:

admin_apps/journeys/iteration.py

@@ -10,16 +10,18 @@
 from streamlit.delta_generator import DeltaGenerator
 from streamlit_monaco import st_monaco
 
-from admin_apps.journeys.builder import get_available_databases, get_available_schemas
+from admin_apps.partner.partner_utils import integrate_partner_semantics
 from admin_apps.shared_utils import (
     GeneratorAppScreen,
     SnowflakeStage,
     add_logo,
     changed_from_last_validated_model,
     download_yaml,
+    format_snowflake_context,
+    get_available_databases,
+    get_available_schemas,
     get_snowflake_connection,
     init_session_states,
-    integrate_partner_semantics,
     upload_yaml,
     validate_and_upload_tmp_yaml,
 )
@@ -497,13 +499,14 @@ def yaml_editor(yaml_str: str) -> None:
             help=UPLOAD_HELP,
         ):
             upload_dialog(content)
-        if four.button(
-            "Partner Semantic",
-            use_container_width=True,
-            help=PARTNER_SEMANTIC_HELP,
-            disabled=not st.session_state["validated"],
-        ):
-            integrate_partner_semantics()
+        if st.session_state.get("partner_setup", False):
+            if four.button(
+                "Integrate Partner",
+                use_container_width=True,
+                help=PARTNER_SEMANTIC_HELP,
+                disabled=not st.session_state["validated"],
+            ):
+                integrate_partner_semantics()
 
     # Render the validation state (success=True, failed=False, editing=None) in the editor.
     if st.session_state.validated:
@@ -569,6 +572,7 @@ def stage_selector_container() -> None:
         options=available_schemas,
         index=None,
         key="selected_iteration_schema",
+        format_func=lambda x: format_snowflake_context(x, -1),
     )
     if stage_schema:
         # When a valid schema is selected, fetch the available stages in that schema.
@@ -584,6 +588,7 @@ def stage_selector_container() -> None:
         options=available_stages,
         index=None,
         key="selected_iteration_stage",
+        format_func=lambda x: format_snowflake_context(x, -1),
     )
 
 
@@ -646,8 +651,8 @@ def set_up_requirements() -> None:
 you think your semantic model is doing great and should be pushed to prod! Note that
 the semantic model must be validated to be uploaded."""
 
-PARTNER_SEMANTIC_HELP = """Have an existing semantic layer in a partner tool that's integrated
-with Snowflake? Use this feature to integrate partner semantic specs into Cortex Analyst's spec.
+PARTNER_SEMANTIC_HELP = """Uploaded semantic files from a partner tool?
+Use this feature to integrate partner semantic specs into Cortex Analyst's spec.
 Note that the Cortex Analyst semantic model must be validated before integrating partner semantics."""
 
 

admin_apps/journeys/partner.py

@@ -0,0 +1,25 @@
+import streamlit as st
+
+from admin_apps.partner.partner_utils import configure_partner_semantic
+
+
+@st.dialog("Partner Semantic Support", width="large")
+def partner_semantic_setup() -> None:
+    """
+    Renders the partner semantic setup dialog with instructions.
+    """
+
+    st.write(
+        """
+        Have an existing semantic layer in a partner tool that's integrated with Snowflake?
+        See the below instructions for integrating your partner semantic specs into Cortex Analyst's semantic file.
+        """
+    )
+    configure_partner_semantic()
+
+
+def show() -> None:
+    """
+    Runs partner setup dialog.
+    """
+    partner_semantic_setup()