Update

framework-evalanche/home.py

@@ -110,9 +110,11 @@ def show_eval_details(
                 label_visibility="collapsed",
                 height=200,
             )
-    for metric_name, assignments in evaluation["PARAM_ASSIGNMENTS"].items():
-        with st.expander(f"Parameter Assignments for **{metric_name}**"):
-            st.write(assignments)
+    st.write("**Metrics**:")
+    for metric_name in evaluation["METRIC_NAMES"]:
+        with st.expander(f"{metric_name}"):
+            st.write(f"Model: {evaluation['MODELS'][metric_name]}")
+            st.write(evaluation["PARAM_ASSIGNMENTS"][metric_name])
     button_container = row(5, vertical_align="center")
     if button_container.button("Run", use_container_width=True):
         click_func(evaluation)
@@ -149,6 +151,7 @@ def run_saved_eval(evaluation: Dict[str, Any]) -> None:
         # wants to automate an already saved evaluation
         st.session_state["source_sql"] = evaluation["SOURCE_SQL"]
         st.session_state["param_selection"] = evaluation["PARAM_ASSIGNMENTS"]
+        st.session_state["model_selection"] = evaluation["MODELS"]
         st.switch_page("pages/results.py")
 
 
@@ -171,6 +174,7 @@ def run_auto_eval(evaluation: Dict[str, Any]) -> None:
             metric for metric in metrics if metric.name in evaluation["METRIC_NAMES"]
         ]
         st.session_state["param_selection"] = evaluation["PARAM_ASSIGNMENTS"]
+        st.session_state["model_selection"] = evaluation["MODELS"]
         st.session_state["eval_funnel"] = "automated"
         try:
             result = st.session_state["session"].table(

framework-evalanche/pages/data.py

@@ -15,6 +15,8 @@
     fetch_columns,
     render_sidebar,
     table_data_selector,
+    select_model,
+    test_complete,
 )
 from src.metric_utils import metric_runner
 from src.snowflake_utils import (
@@ -53,6 +55,17 @@
 """
 
 
+def check_models(models: List[str]) -> None:
+    """Check if models are available in the Snowflake account region."""
+
+    for model in models:
+        available = test_complete(st.session_state["session"],
+                      model)
+        if not available:
+            st.error(f"Model {model} not available in region. Please select another.")
+            st.stop()
+
+
 def run_sql(sql: str) -> Union[None, DataFrame]:
     """Run SQL query and return DataFrame or surfaces Streamlit error."""
 
@@ -286,7 +299,7 @@ def pipeline_runner_dialog() -> None:
 def configure_metrics() -> None:
     """Dialog to configure metric parameters/inputs to data source columns."""
 
-    st.write("Select a column for each required parameter.")
+    st.write("Select a model and a column for each required parameter.")
     limit = 5
     if st.session_state.get("single_source_data", None) is None:
         validate_data_inputs()
@@ -306,9 +319,12 @@ def configure_metrics() -> None:
         except Exception as e:
             st.error(f"Error in pulling data: {e}")
     param_selection = {}  # Track parameter-column assignments for each metric
+    model_selection = {} # Track model selection for each metric
     for metric in st.session_state["selected_metrics"]:
         st.divider()
         st.write(f"**{metric.name}**: {metric.description}")
+        model = select_model(default = metric.model,
+                             keyname = metric.name)
         metric_params = (
             OrderedDict()
         )  # Track each parameter assignment for a single metric
@@ -322,8 +338,11 @@ def configure_metrics() -> None:
                 help=desc,
             )
         param_selection[metric.name] = metric_params
+        model_selection[metric.name] = model
     st.session_state["param_selection"] = param_selection
+    st.session_state["model_selection"] = model_selection
     if st.button("Run"):
+        check_models(st.session_state["model_selection"].values())
         run_eval()
 
 
@@ -356,6 +375,7 @@ def run_eval() -> None:
             st.session_state["metric_result_data"] = metric_runner(
                 session=st.session_state["session"],
                 metrics=st.session_state["selected_metrics"],
+                models=st.session_state["model_selection"],
                 param_assignments=st.session_state["param_selection"],
                 source_df=st.session_state["metric_result_data"],
                 source_sql=None,

framework-evalanche/pages/results.py

@@ -158,6 +158,7 @@ def save_eval() -> None:
             "METRIC_NAMES": [metric.name for metric in metrics],
             "DESCRIPTION": eval_description,  # Not passed to object creation but just inserted into table
             "SOURCE_SQL": st.session_state["source_sql"],
+            "MODELS": st.session_state["model_selection"],
             "PARAM_ASSIGNMENTS": st.session_state["param_selection"],
         }
 
@@ -171,6 +172,7 @@ def save_eval() -> None:
                     eval_name=eval_metadata["EVAL_NAME"],
                     metrics=metrics,
                     source_sql=eval_metadata["SOURCE_SQL"],
+                    models=eval_metadata["MODELS"],
                     param_assignments=eval_metadata["PARAM_ASSIGNMENTS"],
                 )
                 st.success(
@@ -235,6 +237,7 @@ def automate_eval() -> None:
             "METRIC_NAMES": [metric.name for metric in metrics],
             "DESCRIPTION": eval_description,  # Not passed to object creation but just inserted into table
             "SOURCE_SQL": st.session_state["source_sql"],
+            "MODELS": st.session_state["model_selection"],
             "PARAM_ASSIGNMENTS": st.session_state["param_selection"],
         }
         try:
@@ -247,6 +250,7 @@ def automate_eval() -> None:
                     warehouse=warehouse,
                     eval_name=eval_metadata["EVAL_NAME"],
                     metrics=metrics,
+                    models=eval_metadata["MODELS"],
                     source_sql=eval_metadata["SOURCE_SQL"],
                     param_assignments=eval_metadata["PARAM_ASSIGNMENTS"],
                 )

framework-evalanche/setup.sql

@@ -15,6 +15,7 @@ CREATE TABLE IF NOT EXISTS CORTEX_ANALYST_UTILITIES.EVALUATION.SAVED_EVALUATIONS
 DESCRIPTION VARCHAR,
 METRIC_NAMES ARRAY,
 SOURCE_SQL VARCHAR,
+MODELS VARIANT,
 PARAM_ASSIGNMENTS VARIANT,
 ASSOCIATED_OBJECTS VARIANT)
 COMMENT = '{"origin": "sf_sit",
@@ -26,6 +27,7 @@ CREATE TABLE IF NOT EXISTS CORTEX_ANALYST_UTILITIES.EVALUATION.AUTO_EVALUATIONS
 DESCRIPTION VARCHAR,
 METRIC_NAMES ARRAY,
 SOURCE_SQL VARCHAR,
+MODELS VARIANT,
 PARAM_ASSIGNMENTS VARIANT,
 ASSOCIATED_OBJECTS VARIANT)
 COMMENT = '{"origin": "sf_sit",

framework-evalanche/src/app_utils.py

@@ -22,6 +22,55 @@
     }
     """
 
+models = [
+    'llama3.2-1b',
+    'llama3.2-3b',
+    'llama3.1-8b',
+    'llama3.1-70b',
+    'llama3.1-405b',
+    'snowflake-arctic',
+    'reka-core',
+    'reka-flash',
+    'mistral-large2',
+    'mixtral-8x7b',
+    'mistral-7b',
+    'jamba-instruct',
+    'jamba-1.5-mini',
+    'jamba-1.5-large',
+    'gemma-7b',
+]
+
+def select_model(
+        keyname: str,
+        default: Optional[str] = None,
+                 ) -> List[str]:
+    """Renders selectbox for model selection.
+
+    Args:
+        default (string): Default model to select.
+
+    Returns:
+        string: Selected model.
+    """
+
+    return st.selectbox("Select model",
+                        models,
+                        index=models.index(default) if default in models else None,
+                        key=f"{keyname}_model_selector",)
+
+
+def test_complete(session, model, prompt = "Repeat the word hello once and only once. Do not say anything else.") -> bool:
+    from snowflake.cortex import Complete
+    from snowflake.snowpark.exceptions import SnowparkSQLException
+
+    """Returns True if selected model is supported in region and returns False otherwise."""
+    try:
+        response = Complete(model, prompt, session = session)
+        return True
+    except SnowparkSQLException as e:
+        if 'unknown model' in str(e):
+            return False
+
 
 def fetch_metrics() -> List[Metric]:
     """Combines metrics and custom metrics, if any, and returns list of metrics."""
@@ -287,7 +336,7 @@ def try_parse_json(value: str) -> Any:
 
 def fetch_evals(
     table_name: str,
-    json_cols=["METRIC_NAMES", "PARAM_ASSIGNMENTS", "ASSOCIATED_OBJECTS"],
+    json_cols=["METRIC_NAMES", "PARAM_ASSIGNMENTS", "MODELS", "ASSOCIATED_OBJECTS"],
 ) -> List[Optional[Dict[str, Optional[str]]]]:
     """
     Returns evaluation metadata from tables.