diff --git a/api/api/model_schema_api_test.go b/api/api/model_schema_api_test.go index 473469c9f..ee0af3c99 100644 --- a/api/api/model_schema_api_test.go +++ b/api/api/model_schema_api_test.go @@ -34,6 +34,8 @@ func TestModelSchemaController_GetAllSchemas(t *testing.T) { ID: models.ID(1), Spec: &models.SchemaSpec{ PredictionIDColumn: "prediction_id", + SessionIDColumn: "session_id", + RowIDColumn: "row_id", FeatureTypes: map[string]models.ValueType{ "featureA": models.Float64, "featureB": models.Boolean, @@ -41,7 +43,7 @@ func TestModelSchemaController_GetAllSchemas(t *testing.T) { }, ModelPredictionOutput: &models.ModelPredictionOutput{ BinaryClassificationOutput: &models.BinaryClassificationOutput{ - ActualLabelColumn: "actual_label", + ActualScoreColumn: "actual_score", NegativeClassLabel: "negative", PositiveClassLabel: "positive", PredictionScoreColumn: "prediction_score", @@ -55,6 +57,8 @@ func TestModelSchemaController_GetAllSchemas(t *testing.T) { ID: models.ID(2), Spec: &models.SchemaSpec{ PredictionIDColumn: "prediction_id", + SessionIDColumn: "session_id", + RowIDColumn: "row_id", FeatureTypes: map[string]models.ValueType{ "featureA": models.Float64, "featureB": models.Boolean, @@ -62,9 +66,8 @@ func TestModelSchemaController_GetAllSchemas(t *testing.T) { }, ModelPredictionOutput: &models.ModelPredictionOutput{ RankingOutput: &models.RankingOutput{ - PredictionGroupIDColumn: "session_id", - RankScoreColumn: "score", - RelevanceScoreColumn: "relevance_score", + RankScoreColumn: "score", + RelevanceScoreColumn: "relevance_score", }, }, }, @@ -80,6 +83,8 @@ func TestModelSchemaController_GetAllSchemas(t *testing.T) { ID: models.ID(1), Spec: &models.SchemaSpec{ PredictionIDColumn: "prediction_id", + SessionIDColumn: "session_id", + RowIDColumn: "row_id", FeatureTypes: map[string]models.ValueType{ "featureA": models.Float64, "featureB": models.Boolean, @@ -87,7 +92,7 @@ func TestModelSchemaController_GetAllSchemas(t *testing.T) { }, ModelPredictionOutput: &models.ModelPredictionOutput{ BinaryClassificationOutput: &models.BinaryClassificationOutput{ - ActualLabelColumn: "actual_label", + ActualScoreColumn: "actual_score", NegativeClassLabel: "negative", PositiveClassLabel: "positive", PredictionScoreColumn: "prediction_score", @@ -101,6 +106,8 @@ func TestModelSchemaController_GetAllSchemas(t *testing.T) { ID: models.ID(2), Spec: &models.SchemaSpec{ PredictionIDColumn: "prediction_id", + SessionIDColumn: "session_id", + RowIDColumn: "row_id", FeatureTypes: map[string]models.ValueType{ "featureA": models.Float64, "featureB": models.Boolean, @@ -108,9 +115,8 @@ func TestModelSchemaController_GetAllSchemas(t *testing.T) { }, ModelPredictionOutput: &models.ModelPredictionOutput{ RankingOutput: &models.RankingOutput{ - PredictionGroupIDColumn: "session_id", - RankScoreColumn: "score", - RelevanceScoreColumn: "relevance_score", + RankScoreColumn: "score", + RelevanceScoreColumn: "relevance_score", }, }, }, @@ -189,7 +195,7 @@ func TestModelSchemaController_GetSchema(t *testing.T) { }, ModelPredictionOutput: &models.ModelPredictionOutput{ BinaryClassificationOutput: &models.BinaryClassificationOutput{ - ActualLabelColumn: "actual_label", + ActualScoreColumn: "actual_score", NegativeClassLabel: "negative", PositiveClassLabel: "positive", PredictionScoreColumn: "prediction_score", @@ -214,7 +220,7 @@ func TestModelSchemaController_GetSchema(t *testing.T) { }, ModelPredictionOutput: &models.ModelPredictionOutput{ BinaryClassificationOutput: &models.BinaryClassificationOutput{ - ActualLabelColumn: "actual_label", + ActualScoreColumn: "actual_score", NegativeClassLabel: "negative", PositiveClassLabel: "positive", PredictionScoreColumn: "prediction_score", @@ -287,6 +293,8 @@ func TestModelSchemaController_CreateOrUpdateSchema(t *testing.T) { body: []byte(`{ "spec": { "prediction_id_column":"prediction_id", + "session_id_column":"session_id", + "row_id_column":"row_id", "tag_columns": ["tags"], "feature_types": { "featureA": "float64", @@ -294,7 +302,6 @@ func TestModelSchemaController_CreateOrUpdateSchema(t *testing.T) { "featureC": "boolean" }, "model_prediction_output": { - "prediction_group_id_column": "session_id", "rank_score_column": "score", "relevance_score_column": "relevance_score", "output_class": "RankingOutput" @@ -307,6 +314,8 @@ func TestModelSchemaController_CreateOrUpdateSchema(t *testing.T) { ModelID: models.ID(1), Spec: &models.SchemaSpec{ PredictionIDColumn: "prediction_id", + SessionIDColumn: "session_id", + RowIDColumn: "row_id", TagColumns: []string{"tags"}, FeatureTypes: map[string]models.ValueType{ "featureA": models.Float64, @@ -315,10 +324,9 @@ func TestModelSchemaController_CreateOrUpdateSchema(t *testing.T) { }, ModelPredictionOutput: &models.ModelPredictionOutput{ RankingOutput: &models.RankingOutput{ - PredictionGroupIDColumn: "session_id", - RankScoreColumn: "score", - RelevanceScoreColumn: "relevance_score", - OutputClass: models.Ranking, + RankScoreColumn: "score", + RelevanceScoreColumn: "relevance_score", + OutputClass: models.Ranking, }, }, }, @@ -327,6 +335,8 @@ func TestModelSchemaController_CreateOrUpdateSchema(t *testing.T) { ModelID: models.ID(1), Spec: &models.SchemaSpec{ PredictionIDColumn: "prediction_id", + SessionIDColumn: "session_id", + RowIDColumn: "row_id", TagColumns: []string{"tags"}, FeatureTypes: map[string]models.ValueType{ "featureA": models.Float64, @@ -335,10 +345,9 @@ func TestModelSchemaController_CreateOrUpdateSchema(t *testing.T) { }, ModelPredictionOutput: &models.ModelPredictionOutput{ RankingOutput: &models.RankingOutput{ - PredictionGroupIDColumn: "session_id", - RankScoreColumn: "score", - RelevanceScoreColumn: "relevance_score", - OutputClass: models.Ranking, + RankScoreColumn: "score", + RelevanceScoreColumn: "relevance_score", + OutputClass: models.Ranking, }, }, }, @@ -352,6 +361,8 @@ func TestModelSchemaController_CreateOrUpdateSchema(t *testing.T) { ModelID: models.ID(1), Spec: &models.SchemaSpec{ PredictionIDColumn: "prediction_id", + SessionIDColumn: "session_id", + RowIDColumn: "row_id", TagColumns: []string{"tags"}, FeatureTypes: map[string]models.ValueType{ "featureA": models.Float64, @@ -360,10 +371,9 @@ func TestModelSchemaController_CreateOrUpdateSchema(t *testing.T) { }, ModelPredictionOutput: &models.ModelPredictionOutput{ RankingOutput: &models.RankingOutput{ - PredictionGroupIDColumn: "session_id", - RankScoreColumn: "score", - RelevanceScoreColumn: "relevance_score", - OutputClass: models.Ranking, + RankScoreColumn: "score", + RelevanceScoreColumn: "relevance_score", + OutputClass: models.Ranking, }, }, }, @@ -378,6 +388,8 @@ func TestModelSchemaController_CreateOrUpdateSchema(t *testing.T) { body: []byte(`{ "spec": { "prediction_id_column":"prediction_id", + "session_id_column":"session_id", + "row_id_column":"row_id", "tag_columns": ["tags"], "feature_types": { "featureA": "float64", @@ -385,7 +397,6 @@ func TestModelSchemaController_CreateOrUpdateSchema(t *testing.T) { "featureC": "boolean" }, "model_prediction_output": { - "prediction_group_id_column": "session_id", "rank_score_column": "score", "relevance_score_column": "relevance_score", "output_class": "RankingOutput" @@ -399,6 +410,8 @@ func TestModelSchemaController_CreateOrUpdateSchema(t *testing.T) { ModelID: models.ID(1), Spec: &models.SchemaSpec{ PredictionIDColumn: "prediction_id", + SessionIDColumn: "session_id", + RowIDColumn: "row_id", TagColumns: []string{"tags"}, FeatureTypes: map[string]models.ValueType{ "featureA": models.Float64, @@ -407,10 +420,9 @@ func TestModelSchemaController_CreateOrUpdateSchema(t *testing.T) { }, ModelPredictionOutput: &models.ModelPredictionOutput{ RankingOutput: &models.RankingOutput{ - PredictionGroupIDColumn: "session_id", - RankScoreColumn: "score", - RelevanceScoreColumn: "relevance_score", - OutputClass: models.Ranking, + RankScoreColumn: "score", + RelevanceScoreColumn: "relevance_score", + OutputClass: models.Ranking, }, }, }, @@ -419,6 +431,8 @@ func TestModelSchemaController_CreateOrUpdateSchema(t *testing.T) { ModelID: models.ID(1), Spec: &models.SchemaSpec{ PredictionIDColumn: "prediction_id", + SessionIDColumn: "session_id", + RowIDColumn: "row_id", TagColumns: []string{"tags"}, FeatureTypes: map[string]models.ValueType{ "featureA": models.Float64, @@ -427,10 +441,9 @@ func TestModelSchemaController_CreateOrUpdateSchema(t *testing.T) { }, ModelPredictionOutput: &models.ModelPredictionOutput{ RankingOutput: &models.RankingOutput{ - PredictionGroupIDColumn: "session_id", - RankScoreColumn: "score", - RelevanceScoreColumn: "relevance_score", - OutputClass: models.Ranking, + RankScoreColumn: "score", + RelevanceScoreColumn: "relevance_score", + OutputClass: models.Ranking, }, }, }, @@ -444,6 +457,8 @@ func TestModelSchemaController_CreateOrUpdateSchema(t *testing.T) { ModelID: models.ID(1), Spec: &models.SchemaSpec{ PredictionIDColumn: "prediction_id", + SessionIDColumn: "session_id", + RowIDColumn: "row_id", TagColumns: []string{"tags"}, FeatureTypes: map[string]models.ValueType{ "featureA": models.Float64, @@ -452,10 +467,9 @@ func TestModelSchemaController_CreateOrUpdateSchema(t *testing.T) { }, ModelPredictionOutput: &models.ModelPredictionOutput{ RankingOutput: &models.RankingOutput{ - PredictionGroupIDColumn: "session_id", - RankScoreColumn: "score", - RelevanceScoreColumn: "relevance_score", - OutputClass: models.Ranking, + RankScoreColumn: "score", + RelevanceScoreColumn: "relevance_score", + OutputClass: models.Ranking, }, }, }, @@ -470,6 +484,8 @@ func TestModelSchemaController_CreateOrUpdateSchema(t *testing.T) { body: []byte(`{ "spec": { "prediction_id_column":"prediction_id", + "session_id_column":"session_id", + "row_id_column":"row_id", "tag_columns": ["tags"], "feature_types": { "featureA": "float64", @@ -477,7 +493,7 @@ func TestModelSchemaController_CreateOrUpdateSchema(t *testing.T) { "featureC": "boolean" }, "model_prediction_output": { - "actual_label_column": "actual_label", + "actual_score_column": "actual_score", "negative_class_label": "negative", "prediction_score_column": "prediction_score", "prediction_label_column": "prediction_label", @@ -492,6 +508,8 @@ func TestModelSchemaController_CreateOrUpdateSchema(t *testing.T) { ModelID: models.ID(1), Spec: &models.SchemaSpec{ PredictionIDColumn: "prediction_id", + SessionIDColumn: "session_id", + RowIDColumn: "row_id", TagColumns: []string{"tags"}, FeatureTypes: map[string]models.ValueType{ "featureA": models.Float64, @@ -500,7 +518,7 @@ func TestModelSchemaController_CreateOrUpdateSchema(t *testing.T) { }, ModelPredictionOutput: &models.ModelPredictionOutput{ BinaryClassificationOutput: &models.BinaryClassificationOutput{ - ActualLabelColumn: "actual_label", + ActualScoreColumn: "actual_score", NegativeClassLabel: "negative", PredictionScoreColumn: "prediction_score", PredictionLabelColumn: "prediction_label", @@ -514,6 +532,8 @@ func TestModelSchemaController_CreateOrUpdateSchema(t *testing.T) { ModelID: models.ID(1), Spec: &models.SchemaSpec{ PredictionIDColumn: "prediction_id", + SessionIDColumn: "session_id", + RowIDColumn: "row_id", TagColumns: []string{"tags"}, FeatureTypes: map[string]models.ValueType{ "featureA": models.Float64, @@ -522,7 +542,7 @@ func TestModelSchemaController_CreateOrUpdateSchema(t *testing.T) { }, ModelPredictionOutput: &models.ModelPredictionOutput{ BinaryClassificationOutput: &models.BinaryClassificationOutput{ - ActualLabelColumn: "actual_label", + ActualScoreColumn: "actual_score", NegativeClassLabel: "negative", PredictionScoreColumn: "prediction_score", PredictionLabelColumn: "prediction_label", @@ -541,6 +561,8 @@ func TestModelSchemaController_CreateOrUpdateSchema(t *testing.T) { ModelID: models.ID(1), Spec: &models.SchemaSpec{ PredictionIDColumn: "prediction_id", + SessionIDColumn: "session_id", + RowIDColumn: "row_id", TagColumns: []string{"tags"}, FeatureTypes: map[string]models.ValueType{ "featureA": models.Float64, @@ -549,7 +571,7 @@ func TestModelSchemaController_CreateOrUpdateSchema(t *testing.T) { }, ModelPredictionOutput: &models.ModelPredictionOutput{ BinaryClassificationOutput: &models.BinaryClassificationOutput{ - ActualLabelColumn: "actual_label", + ActualScoreColumn: "actual_score", NegativeClassLabel: "negative", PredictionScoreColumn: "prediction_score", PredictionLabelColumn: "prediction_label", @@ -569,6 +591,8 @@ func TestModelSchemaController_CreateOrUpdateSchema(t *testing.T) { body: []byte(`{ "spec": { "prediction_id_column":"prediction_id", + "session_id_column":"session_id", + "row_id_column":"row_id", "tag_columns": ["tags"], "feature_types": { "featureA": "float64", @@ -588,6 +612,8 @@ func TestModelSchemaController_CreateOrUpdateSchema(t *testing.T) { ModelID: models.ID(1), Spec: &models.SchemaSpec{ PredictionIDColumn: "prediction_id", + SessionIDColumn: "session_id", + RowIDColumn: "row_id", TagColumns: []string{"tags"}, FeatureTypes: map[string]models.ValueType{ "featureA": models.Float64, @@ -607,6 +633,8 @@ func TestModelSchemaController_CreateOrUpdateSchema(t *testing.T) { ModelID: models.ID(1), Spec: &models.SchemaSpec{ PredictionIDColumn: "prediction_id", + SessionIDColumn: "session_id", + RowIDColumn: "row_id", TagColumns: []string{"tags"}, FeatureTypes: map[string]models.ValueType{ "featureA": models.Float64, @@ -631,6 +659,8 @@ func TestModelSchemaController_CreateOrUpdateSchema(t *testing.T) { ModelID: models.ID(1), Spec: &models.SchemaSpec{ PredictionIDColumn: "prediction_id", + SessionIDColumn: "session_id", + RowIDColumn: "row_id", TagColumns: []string{"tags"}, FeatureTypes: map[string]models.ValueType{ "featureA": models.Float64, @@ -656,6 +686,8 @@ func TestModelSchemaController_CreateOrUpdateSchema(t *testing.T) { body: []byte(`{ "spec": { "prediction_id_column":"prediction_id", + "session_id_column":"session_id", + "row_id_column":"row_id", "tag_columns": ["tags"], "feature_types": { "featureA": "float64", @@ -663,7 +695,6 @@ func TestModelSchemaController_CreateOrUpdateSchema(t *testing.T) { "featureC": "boolean" }, "model_prediction_output": { - "prediction_group_id_column": "session_id", "rank_score_column": "score", "relevance_score_column": "relevance_score", "output_class": "RankingOutput" @@ -676,6 +707,8 @@ func TestModelSchemaController_CreateOrUpdateSchema(t *testing.T) { ModelID: models.ID(1), Spec: &models.SchemaSpec{ PredictionIDColumn: "prediction_id", + SessionIDColumn: "session_id", + RowIDColumn: "row_id", TagColumns: []string{"tags"}, FeatureTypes: map[string]models.ValueType{ "featureA": models.Float64, @@ -684,10 +717,9 @@ func TestModelSchemaController_CreateOrUpdateSchema(t *testing.T) { }, ModelPredictionOutput: &models.ModelPredictionOutput{ RankingOutput: &models.RankingOutput{ - PredictionGroupIDColumn: "session_id", - RankScoreColumn: "score", - RelevanceScoreColumn: "relevance_score", - OutputClass: models.Ranking, + RankScoreColumn: "score", + RelevanceScoreColumn: "relevance_score", + OutputClass: models.Ranking, }, }, }, @@ -707,6 +739,8 @@ func TestModelSchemaController_CreateOrUpdateSchema(t *testing.T) { body: []byte(`{ "spec": { "prediction_id_column":"prediction_id", + "session_id_column":"session_id", + "row_id_column":"row_id", "tag_columns": ["tags"], "feature_types": { "featureA": "float64", @@ -714,7 +748,6 @@ func TestModelSchemaController_CreateOrUpdateSchema(t *testing.T) { "featureC": "boolean" }, "model_prediction_output": { - "prediction_group_id_column": "session_id", "rank_score_column": "score", "relevance_score_column": "relevance_score", "output_class": "RankingOutput" @@ -756,6 +789,8 @@ func TestModelSchemaController_CreateOrUpdateSchema(t *testing.T) { func Benchmark_Unmarshal(b *testing.B) { data := []byte(` { "prediction_id_column":"prediction_id", + "session_id_column":"session_id", + "row_id_column":"row_id", "tag_columns": ["tags"], "feature_types": { "featureA": "float64", @@ -763,7 +798,7 @@ func Benchmark_Unmarshal(b *testing.B) { "featureC": "boolean" }, "model_prediction_output": { - "actual_label_column": "actual_label", + "actual_score_column": "actual_score", "negative_class_label": "negative", "prediction_score_column": "prediction_score", "prediction_label_column": "prediction_label", diff --git a/api/api/versions_api_test.go b/api/api/versions_api_test.go index 7032810bd..aa8581f3a 100644 --- a/api/api/versions_api_test.go +++ b/api/api/versions_api_test.go @@ -722,12 +722,13 @@ func TestPatchVersion(t *testing.T) { ModelSchema: &models.ModelSchema{ Spec: &models.SchemaSpec{ PredictionIDColumn: "prediction_id", + SessionIDColumn: "session_id", + RowIDColumn: "row_id", ModelPredictionOutput: &models.ModelPredictionOutput{ RankingOutput: &models.RankingOutput{ - PredictionGroupIDColumn: "session_id", - RankScoreColumn: "score", - RelevanceScoreColumn: "relevance_score", - OutputClass: models.Ranking, + RankScoreColumn: "score", + RelevanceScoreColumn: "relevance_score", + OutputClass: models.Ranking, }, }, FeatureTypes: map[string]models.ValueType{ @@ -776,12 +777,13 @@ func TestPatchVersion(t *testing.T) { ModelSchema: &models.ModelSchema{ Spec: &models.SchemaSpec{ PredictionIDColumn: "prediction_id", + SessionIDColumn: "session_id", + RowIDColumn: "row_id", ModelPredictionOutput: &models.ModelPredictionOutput{ RankingOutput: &models.RankingOutput{ - PredictionGroupIDColumn: "session_id", - RankScoreColumn: "score", - RelevanceScoreColumn: "relevance_score", - OutputClass: models.Ranking, + RankScoreColumn: "score", + RelevanceScoreColumn: "relevance_score", + OutputClass: models.Ranking, }, }, FeatureTypes: map[string]models.ValueType{ @@ -812,12 +814,13 @@ func TestPatchVersion(t *testing.T) { ModelSchema: &models.ModelSchema{ Spec: &models.SchemaSpec{ PredictionIDColumn: "prediction_id", + SessionIDColumn: "session_id", + RowIDColumn: "row_id", ModelPredictionOutput: &models.ModelPredictionOutput{ RankingOutput: &models.RankingOutput{ - PredictionGroupIDColumn: "session_id", - RankScoreColumn: "score", - RelevanceScoreColumn: "relevance_score", - OutputClass: models.Ranking, + RankScoreColumn: "score", + RelevanceScoreColumn: "relevance_score", + OutputClass: models.Ranking, }, }, FeatureTypes: map[string]models.ValueType{ @@ -853,12 +856,13 @@ func TestPatchVersion(t *testing.T) { ModelSchema: &models.ModelSchema{ Spec: &models.SchemaSpec{ PredictionIDColumn: "prediction_id", + SessionIDColumn: "session_id", + RowIDColumn: "row_id", ModelPredictionOutput: &models.ModelPredictionOutput{ RankingOutput: &models.RankingOutput{ - PredictionGroupIDColumn: "session_id", - RankScoreColumn: "score", - RelevanceScoreColumn: "relevance_score", - OutputClass: models.Ranking, + RankScoreColumn: "score", + RelevanceScoreColumn: "relevance_score", + OutputClass: models.Ranking, }, }, FeatureTypes: map[string]models.ValueType{ @@ -886,12 +890,13 @@ func TestPatchVersion(t *testing.T) { ModelSchema: &models.ModelSchema{ Spec: &models.SchemaSpec{ PredictionIDColumn: "prediction_id", + SessionIDColumn: "session_id", + RowIDColumn: "row_id", ModelPredictionOutput: &models.ModelPredictionOutput{ RankingOutput: &models.RankingOutput{ - PredictionGroupIDColumn: "session_id", - RankScoreColumn: "score", - RelevanceScoreColumn: "relevance_score", - OutputClass: models.Ranking, + RankScoreColumn: "score", + RelevanceScoreColumn: "relevance_score", + OutputClass: models.Ranking, }, }, FeatureTypes: map[string]models.ValueType{ @@ -1446,12 +1451,13 @@ func TestCreateVersion(t *testing.T) { ModelSchema: &models.ModelSchema{ Spec: &models.SchemaSpec{ PredictionIDColumn: "prediction_id", + SessionIDColumn: "session_id", + RowIDColumn: "row_id", ModelPredictionOutput: &models.ModelPredictionOutput{ RankingOutput: &models.RankingOutput{ - PredictionGroupIDColumn: "session_id", - RankScoreColumn: "score", - RelevanceScoreColumn: "relevance_score", - OutputClass: models.Ranking, + RankScoreColumn: "score", + RelevanceScoreColumn: "relevance_score", + OutputClass: models.Ranking, }, }, FeatureTypes: map[string]models.ValueType{ @@ -1499,12 +1505,13 @@ func TestCreateVersion(t *testing.T) { ModelSchema: &models.ModelSchema{ Spec: &models.SchemaSpec{ PredictionIDColumn: "prediction_id", + SessionIDColumn: "session_id", + RowIDColumn: "row_id", ModelPredictionOutput: &models.ModelPredictionOutput{ RankingOutput: &models.RankingOutput{ - PredictionGroupIDColumn: "session_id", - RankScoreColumn: "score", - RelevanceScoreColumn: "relevance_score", - OutputClass: models.Ranking, + RankScoreColumn: "score", + RelevanceScoreColumn: "relevance_score", + OutputClass: models.Ranking, }, }, FeatureTypes: map[string]models.ValueType{ @@ -1532,12 +1539,13 @@ func TestCreateVersion(t *testing.T) { ModelSchema: &models.ModelSchema{ Spec: &models.SchemaSpec{ PredictionIDColumn: "prediction_id", + SessionIDColumn: "session_id", + RowIDColumn: "row_id", ModelPredictionOutput: &models.ModelPredictionOutput{ RankingOutput: &models.RankingOutput{ - PredictionGroupIDColumn: "session_id", - RankScoreColumn: "score", - RelevanceScoreColumn: "relevance_score", - OutputClass: models.Ranking, + RankScoreColumn: "score", + RelevanceScoreColumn: "relevance_score", + OutputClass: models.Ranking, }, }, FeatureTypes: map[string]models.ValueType{ @@ -1570,12 +1578,13 @@ func TestCreateVersion(t *testing.T) { ModelSchema: &models.ModelSchema{ Spec: &models.SchemaSpec{ PredictionIDColumn: "prediction_id", + SessionIDColumn: "session_id", + RowIDColumn: "row_id", ModelPredictionOutput: &models.ModelPredictionOutput{ RankingOutput: &models.RankingOutput{ - PredictionGroupIDColumn: "session_id", - RankScoreColumn: "score", - RelevanceScoreColumn: "relevance_score", - OutputClass: models.Ranking, + RankScoreColumn: "score", + RelevanceScoreColumn: "relevance_score", + OutputClass: models.Ranking, }, }, FeatureTypes: map[string]models.ValueType{ @@ -1598,12 +1607,13 @@ func TestCreateVersion(t *testing.T) { ModelSchema: &models.ModelSchema{ Spec: &models.SchemaSpec{ PredictionIDColumn: "prediction_id", + SessionIDColumn: "session_id", + RowIDColumn: "row_id", ModelPredictionOutput: &models.ModelPredictionOutput{ RankingOutput: &models.RankingOutput{ - PredictionGroupIDColumn: "session_id", - RankScoreColumn: "score", - RelevanceScoreColumn: "relevance_score", - OutputClass: models.Ranking, + RankScoreColumn: "score", + RelevanceScoreColumn: "relevance_score", + OutputClass: models.Ranking, }, }, FeatureTypes: map[string]models.ValueType{ diff --git a/api/client/model_binary_classification_output.go b/api/client/model_binary_classification_output.go index a93108ed5..65e8299ab 100644 --- a/api/client/model_binary_classification_output.go +++ b/api/client/model_binary_classification_output.go @@ -21,7 +21,7 @@ var _ MappedNullable = &BinaryClassificationOutput{} // BinaryClassificationOutput struct for BinaryClassificationOutput type BinaryClassificationOutput struct { PredictionScoreColumn string `json:"prediction_score_column"` - ActualScoreColumn *string `json:"actual_score_column,omitempty"` + ActualScoreColumn string `json:"actual_score_column"` PositiveClassLabel string `json:"positive_class_label"` NegativeClassLabel string `json:"negative_class_label"` ScoreThreshold *float32 `json:"score_threshold,omitempty"` @@ -34,9 +34,10 @@ type _BinaryClassificationOutput BinaryClassificationOutput // This constructor will assign default values to properties that have it defined, // and makes sure properties required by API are set, but the set of arguments // will change when the set of required properties is changed -func NewBinaryClassificationOutput(predictionScoreColumn string, positiveClassLabel string, negativeClassLabel string, outputClass ModelPredictionOutputClass) *BinaryClassificationOutput { +func NewBinaryClassificationOutput(predictionScoreColumn string, actualScoreColumn string, positiveClassLabel string, negativeClassLabel string, outputClass ModelPredictionOutputClass) *BinaryClassificationOutput { this := BinaryClassificationOutput{} this.PredictionScoreColumn = predictionScoreColumn + this.ActualScoreColumn = actualScoreColumn this.PositiveClassLabel = positiveClassLabel this.NegativeClassLabel = negativeClassLabel this.OutputClass = outputClass @@ -75,36 +76,28 @@ func (o *BinaryClassificationOutput) SetPredictionScoreColumn(v string) { o.PredictionScoreColumn = v } -// GetActualScoreColumn returns the ActualScoreColumn field value if set, zero value otherwise. +// GetActualScoreColumn returns the ActualScoreColumn field value func (o *BinaryClassificationOutput) GetActualScoreColumn() string { - if o == nil || IsNil(o.ActualScoreColumn) { + if o == nil { var ret string return ret } - return *o.ActualScoreColumn + + return o.ActualScoreColumn } -// GetActualScoreColumnOk returns a tuple with the ActualScoreColumn field value if set, nil otherwise +// GetActualScoreColumnOk returns a tuple with the ActualScoreColumn field value // and a boolean to check if the value has been set. func (o *BinaryClassificationOutput) GetActualScoreColumnOk() (*string, bool) { - if o == nil || IsNil(o.ActualScoreColumn) { + if o == nil { return nil, false } - return o.ActualScoreColumn, true -} - -// HasActualScoreColumn returns a boolean if a field has been set. -func (o *BinaryClassificationOutput) HasActualScoreColumn() bool { - if o != nil && !IsNil(o.ActualScoreColumn) { - return true - } - - return false + return &o.ActualScoreColumn, true } -// SetActualScoreColumn gets a reference to the given string and assigns it to the ActualScoreColumn field. +// SetActualScoreColumn sets field value func (o *BinaryClassificationOutput) SetActualScoreColumn(v string) { - o.ActualScoreColumn = &v + o.ActualScoreColumn = v } // GetPositiveClassLabel returns the PositiveClassLabel field value @@ -222,9 +215,7 @@ func (o BinaryClassificationOutput) MarshalJSON() ([]byte, error) { func (o BinaryClassificationOutput) ToMap() (map[string]interface{}, error) { toSerialize := map[string]interface{}{} toSerialize["prediction_score_column"] = o.PredictionScoreColumn - if !IsNil(o.ActualScoreColumn) { - toSerialize["actual_score_column"] = o.ActualScoreColumn - } + toSerialize["actual_score_column"] = o.ActualScoreColumn toSerialize["positive_class_label"] = o.PositiveClassLabel toSerialize["negative_class_label"] = o.NegativeClassLabel if !IsNil(o.ScoreThreshold) { @@ -240,6 +231,7 @@ func (o *BinaryClassificationOutput) UnmarshalJSON(bytes []byte) (err error) { // that every required field exists as a key in the generic map. requiredProperties := []string{ "prediction_score_column", + "actual_score_column", "positive_class_label", "negative_class_label", "output_class", diff --git a/api/cluster/resource/templater_test.go b/api/cluster/resource/templater_test.go index dd77ec065..49bfa0340 100644 --- a/api/cluster/resource/templater_test.go +++ b/api/cluster/resource/templater_test.go @@ -232,6 +232,8 @@ func TestCreateInferenceServiceSpec(t *testing.T) { ModelID: models.ID(1), Spec: &models.SchemaSpec{ PredictionIDColumn: "prediction_id", + SessionIDColumn: "session_id", + RowIDColumn: "row_id", TagColumns: []string{"tags"}, FeatureTypes: map[string]models.ValueType{ "featureA": models.Float64, @@ -241,9 +243,8 @@ func TestCreateInferenceServiceSpec(t *testing.T) { }, ModelPredictionOutput: &models.ModelPredictionOutput{ RankingOutput: &models.RankingOutput{ - PredictionGroupIDColumn: "session_id", - RankScoreColumn: "score", - RelevanceScoreColumn: "relevance_score", + RankScoreColumn: "score", + RelevanceScoreColumn: "relevance_score", }, }, }, @@ -859,6 +860,8 @@ func TestCreateInferenceServiceSpec(t *testing.T) { ModelID: models.ID(1), Spec: &models.SchemaSpec{ PredictionIDColumn: "prediction_id", + SessionIDColumn: "session_id", + RowIDColumn: "row_id", TagColumns: []string{"tags"}, FeatureTypes: map[string]models.ValueType{ "featureA": models.Float64, @@ -868,9 +871,8 @@ func TestCreateInferenceServiceSpec(t *testing.T) { }, ModelPredictionOutput: &models.ModelPredictionOutput{ RankingOutput: &models.RankingOutput{ - PredictionGroupIDColumn: "session_id", - RankScoreColumn: "score", - RelevanceScoreColumn: "relevance_score", + RankScoreColumn: "score", + RelevanceScoreColumn: "relevance_score", }, }, }, diff --git a/api/models/model_schema.go b/api/models/model_schema.go index 9ef91aa13..67b1a01cc 100644 --- a/api/models/model_schema.go +++ b/api/models/model_schema.go @@ -37,6 +37,8 @@ type ModelSchema struct { // SchemaSpec type SchemaSpec struct { PredictionIDColumn string `json:"prediction_id_column"` + SessionIDColumn string `json:"session_id_column"` + RowIDColumn string `json:"row_id_column"` ModelPredictionOutput *ModelPredictionOutput `json:"model_prediction_output"` TagColumns []string `json:"tag_columns"` FeatureTypes map[string]ValueType `json:"feature_types"` @@ -125,7 +127,7 @@ func (m ModelPredictionOutput) MarshalJSON() ([]byte, error) { // BinaryClassificationOutput is specification for prediction of binary classification model type BinaryClassificationOutput struct { - ActualLabelColumn string `json:"actual_label_column"` + ActualScoreColumn string `json:"actual_score_column"` NegativeClassLabel string `json:"negative_class_label"` PredictionScoreColumn string `json:"prediction_score_column"` PredictionLabelColumn string `json:"prediction_label_column"` @@ -136,10 +138,9 @@ type BinaryClassificationOutput struct { // RankingOutput is specification for prediction of ranking model type RankingOutput struct { - PredictionGroupIDColumn string `json:"prediction_group_id_column"` - RankScoreColumn string `json:"rank_score_column"` - RelevanceScoreColumn string `json:"relevance_score_column"` - OutputClass ModelPredictionOutputClass `json:"output_class" validate:"required"` + RankScoreColumn string `json:"rank_score_column"` + RelevanceScoreColumn string `json:"relevance_score_column"` + OutputClass ModelPredictionOutputClass `json:"output_class" validate:"required"` } // Regression is specification for prediction of regression model diff --git a/api/service/model_schema_service_test.go b/api/service/model_schema_service_test.go index 3ceaee3c1..dd545e9e3 100644 --- a/api/service/model_schema_service_test.go +++ b/api/service/model_schema_service_test.go @@ -38,6 +38,8 @@ func Test_modelSchemaService_List(t *testing.T) { ID: models.ID(1), Spec: &models.SchemaSpec{ PredictionIDColumn: "prediction_id", + SessionIDColumn: "session_id", + RowIDColumn: "row_id", FeatureTypes: map[string]models.ValueType{ "featureA": models.Float64, "featureB": models.Boolean, @@ -45,7 +47,7 @@ func Test_modelSchemaService_List(t *testing.T) { }, ModelPredictionOutput: &models.ModelPredictionOutput{ BinaryClassificationOutput: &models.BinaryClassificationOutput{ - ActualLabelColumn: "actual_label", + ActualScoreColumn: "actual_score", NegativeClassLabel: "negative", PositiveClassLabel: "positive", PredictionScoreColumn: "prediction_score", @@ -59,6 +61,8 @@ func Test_modelSchemaService_List(t *testing.T) { ID: models.ID(2), Spec: &models.SchemaSpec{ PredictionIDColumn: "prediction_id", + SessionIDColumn: "session_id", + RowIDColumn: "row_id", FeatureTypes: map[string]models.ValueType{ "featureA": models.Float64, "featureB": models.Boolean, @@ -66,9 +70,8 @@ func Test_modelSchemaService_List(t *testing.T) { }, ModelPredictionOutput: &models.ModelPredictionOutput{ RankingOutput: &models.RankingOutput{ - PredictionGroupIDColumn: "session_id", - RankScoreColumn: "score", - RelevanceScoreColumn: "relevance_score", + RankScoreColumn: "score", + RelevanceScoreColumn: "relevance_score", }, }, }, @@ -86,6 +89,8 @@ func Test_modelSchemaService_List(t *testing.T) { ID: models.ID(1), Spec: &models.SchemaSpec{ PredictionIDColumn: "prediction_id", + SessionIDColumn: "session_id", + RowIDColumn: "row_id", FeatureTypes: map[string]models.ValueType{ "featureA": models.Float64, "featureB": models.Boolean, @@ -93,7 +98,7 @@ func Test_modelSchemaService_List(t *testing.T) { }, ModelPredictionOutput: &models.ModelPredictionOutput{ BinaryClassificationOutput: &models.BinaryClassificationOutput{ - ActualLabelColumn: "actual_label", + ActualScoreColumn: "actual_score", NegativeClassLabel: "negative", PositiveClassLabel: "positive", PredictionScoreColumn: "prediction_score", @@ -107,6 +112,8 @@ func Test_modelSchemaService_List(t *testing.T) { ID: models.ID(2), Spec: &models.SchemaSpec{ PredictionIDColumn: "prediction_id", + SessionIDColumn: "session_id", + RowIDColumn: "row_id", FeatureTypes: map[string]models.ValueType{ "featureA": models.Float64, "featureB": models.Boolean, @@ -114,9 +121,8 @@ func Test_modelSchemaService_List(t *testing.T) { }, ModelPredictionOutput: &models.ModelPredictionOutput{ RankingOutput: &models.RankingOutput{ - PredictionGroupIDColumn: "session_id", - RankScoreColumn: "score", - RelevanceScoreColumn: "relevance_score", + RankScoreColumn: "score", + RelevanceScoreColumn: "relevance_score", }, }, }, @@ -177,6 +183,8 @@ func Test_modelSchemaService_Save(t *testing.T) { ModelID: models.ID(1), Spec: &models.SchemaSpec{ PredictionIDColumn: "prediction_id", + SessionIDColumn: "session_id", + RowIDColumn: "row_id", FeatureTypes: map[string]models.ValueType{ "featureA": models.Float64, "featureB": models.Boolean, @@ -184,7 +192,7 @@ func Test_modelSchemaService_Save(t *testing.T) { }, ModelPredictionOutput: &models.ModelPredictionOutput{ BinaryClassificationOutput: &models.BinaryClassificationOutput{ - ActualLabelColumn: "actual_label", + ActualScoreColumn: "actual_score", NegativeClassLabel: "negative", PositiveClassLabel: "positive", PredictionScoreColumn: "prediction_score", @@ -197,6 +205,8 @@ func Test_modelSchemaService_Save(t *testing.T) { ModelID: models.ID(1), Spec: &models.SchemaSpec{ PredictionIDColumn: "prediction_id", + SessionIDColumn: "session_id", + RowIDColumn: "row_id", FeatureTypes: map[string]models.ValueType{ "featureA": models.Float64, "featureB": models.Boolean, @@ -204,7 +214,7 @@ func Test_modelSchemaService_Save(t *testing.T) { }, ModelPredictionOutput: &models.ModelPredictionOutput{ BinaryClassificationOutput: &models.BinaryClassificationOutput{ - ActualLabelColumn: "actual_label", + ActualScoreColumn: "actual_score", NegativeClassLabel: "negative", PositiveClassLabel: "positive", PredictionScoreColumn: "prediction_score", @@ -221,6 +231,8 @@ func Test_modelSchemaService_Save(t *testing.T) { ModelID: models.ID(1), Spec: &models.SchemaSpec{ PredictionIDColumn: "prediction_id", + SessionIDColumn: "session_id", + RowIDColumn: "row_id", FeatureTypes: map[string]models.ValueType{ "featureA": models.Float64, "featureB": models.Boolean, @@ -228,7 +240,7 @@ func Test_modelSchemaService_Save(t *testing.T) { }, ModelPredictionOutput: &models.ModelPredictionOutput{ BinaryClassificationOutput: &models.BinaryClassificationOutput{ - ActualLabelColumn: "actual_label", + ActualScoreColumn: "actual_score", NegativeClassLabel: "negative", PositiveClassLabel: "positive", PredictionScoreColumn: "prediction_score", @@ -243,6 +255,8 @@ func Test_modelSchemaService_Save(t *testing.T) { ModelID: models.ID(1), Spec: &models.SchemaSpec{ PredictionIDColumn: "prediction_id", + SessionIDColumn: "session_id", + RowIDColumn: "row_id", FeatureTypes: map[string]models.ValueType{ "featureA": models.Float64, "featureB": models.Boolean, @@ -250,7 +264,7 @@ func Test_modelSchemaService_Save(t *testing.T) { }, ModelPredictionOutput: &models.ModelPredictionOutput{ BinaryClassificationOutput: &models.BinaryClassificationOutput{ - ActualLabelColumn: "actual_label", + ActualScoreColumn: "actual_score", NegativeClassLabel: "negative", PositiveClassLabel: "positive", PredictionScoreColumn: "prediction_score", @@ -268,6 +282,8 @@ func Test_modelSchemaService_Save(t *testing.T) { ModelID: models.ID(1), Spec: &models.SchemaSpec{ PredictionIDColumn: "prediction_id", + SessionIDColumn: "session_id", + RowIDColumn: "row_id", FeatureTypes: map[string]models.ValueType{ "featureA": models.Float64, "featureB": models.Boolean, @@ -275,7 +291,7 @@ func Test_modelSchemaService_Save(t *testing.T) { }, ModelPredictionOutput: &models.ModelPredictionOutput{ BinaryClassificationOutput: &models.BinaryClassificationOutput{ - ActualLabelColumn: "actual_label", + ActualScoreColumn: "actual_score", NegativeClassLabel: "negative", PositiveClassLabel: "positive", PredictionScoreColumn: "prediction_score", @@ -292,6 +308,8 @@ func Test_modelSchemaService_Save(t *testing.T) { ModelID: models.ID(1), Spec: &models.SchemaSpec{ PredictionIDColumn: "prediction_id", + SessionIDColumn: "session_id", + RowIDColumn: "row_id", FeatureTypes: map[string]models.ValueType{ "featureA": models.Float64, "featureB": models.Boolean, @@ -299,7 +317,7 @@ func Test_modelSchemaService_Save(t *testing.T) { }, ModelPredictionOutput: &models.ModelPredictionOutput{ BinaryClassificationOutput: &models.BinaryClassificationOutput{ - ActualLabelColumn: "actual_label", + ActualScoreColumn: "actual_score", NegativeClassLabel: "negative", PositiveClassLabel: "positive", PredictionScoreColumn: "prediction_score", @@ -347,6 +365,8 @@ func Test_modelSchemaService_Delete(t *testing.T) { ModelID: models.ID(1), Spec: &models.SchemaSpec{ PredictionIDColumn: "prediction_id", + SessionIDColumn: "session_id", + RowIDColumn: "row_id", FeatureTypes: map[string]models.ValueType{ "featureA": models.Float64, "featureB": models.Boolean, @@ -354,7 +374,7 @@ func Test_modelSchemaService_Delete(t *testing.T) { }, ModelPredictionOutput: &models.ModelPredictionOutput{ BinaryClassificationOutput: &models.BinaryClassificationOutput{ - ActualLabelColumn: "actual_label", + ActualScoreColumn: "actual_score", NegativeClassLabel: "negative", PositiveClassLabel: "positive", PredictionScoreColumn: "prediction_score", @@ -371,6 +391,8 @@ func Test_modelSchemaService_Delete(t *testing.T) { ModelID: models.ID(1), Spec: &models.SchemaSpec{ PredictionIDColumn: "prediction_id", + SessionIDColumn: "session_id", + RowIDColumn: "row_id", FeatureTypes: map[string]models.ValueType{ "featureA": models.Float64, "featureB": models.Boolean, @@ -378,7 +400,7 @@ func Test_modelSchemaService_Delete(t *testing.T) { }, ModelPredictionOutput: &models.ModelPredictionOutput{ BinaryClassificationOutput: &models.BinaryClassificationOutput{ - ActualLabelColumn: "actual_label", + ActualScoreColumn: "actual_score", NegativeClassLabel: "negative", PositiveClassLabel: "positive", PredictionScoreColumn: "prediction_score", @@ -397,6 +419,8 @@ func Test_modelSchemaService_Delete(t *testing.T) { ModelID: models.ID(1), Spec: &models.SchemaSpec{ PredictionIDColumn: "prediction_id", + SessionIDColumn: "session_id", + RowIDColumn: "row_id", FeatureTypes: map[string]models.ValueType{ "featureA": models.Float64, "featureB": models.Boolean, @@ -404,7 +428,7 @@ func Test_modelSchemaService_Delete(t *testing.T) { }, ModelPredictionOutput: &models.ModelPredictionOutput{ BinaryClassificationOutput: &models.BinaryClassificationOutput{ - ActualLabelColumn: "actual_label", + ActualScoreColumn: "actual_score", NegativeClassLabel: "negative", PositiveClassLabel: "positive", PredictionScoreColumn: "prediction_score", @@ -421,6 +445,8 @@ func Test_modelSchemaService_Delete(t *testing.T) { ModelID: models.ID(1), Spec: &models.SchemaSpec{ PredictionIDColumn: "prediction_id", + SessionIDColumn: "session_id", + RowIDColumn: "row_id", FeatureTypes: map[string]models.ValueType{ "featureA": models.Float64, "featureB": models.Boolean, @@ -428,7 +454,7 @@ func Test_modelSchemaService_Delete(t *testing.T) { }, ModelPredictionOutput: &models.ModelPredictionOutput{ BinaryClassificationOutput: &models.BinaryClassificationOutput{ - ActualLabelColumn: "actual_label", + ActualScoreColumn: "actual_score", NegativeClassLabel: "negative", PositiveClassLabel: "positive", PredictionScoreColumn: "prediction_score", @@ -474,6 +500,8 @@ func Test_modelSchemaService_FindByID(t *testing.T) { ID: models.ID(1), Spec: &models.SchemaSpec{ PredictionIDColumn: "prediction_id", + SessionIDColumn: "session_id", + RowIDColumn: "row_id", FeatureTypes: map[string]models.ValueType{ "featureA": models.Float64, "featureB": models.Boolean, @@ -481,7 +509,7 @@ func Test_modelSchemaService_FindByID(t *testing.T) { }, ModelPredictionOutput: &models.ModelPredictionOutput{ BinaryClassificationOutput: &models.BinaryClassificationOutput{ - ActualLabelColumn: "actual_label", + ActualScoreColumn: "actual_score", NegativeClassLabel: "negative", PositiveClassLabel: "positive", PredictionScoreColumn: "prediction_score", @@ -502,6 +530,8 @@ func Test_modelSchemaService_FindByID(t *testing.T) { ID: models.ID(1), Spec: &models.SchemaSpec{ PredictionIDColumn: "prediction_id", + SessionIDColumn: "session_id", + RowIDColumn: "row_id", FeatureTypes: map[string]models.ValueType{ "featureA": models.Float64, "featureB": models.Boolean, @@ -509,7 +539,7 @@ func Test_modelSchemaService_FindByID(t *testing.T) { }, ModelPredictionOutput: &models.ModelPredictionOutput{ BinaryClassificationOutput: &models.BinaryClassificationOutput{ - ActualLabelColumn: "actual_label", + ActualScoreColumn: "actual_score", NegativeClassLabel: "negative", PositiveClassLabel: "positive", PredictionScoreColumn: "prediction_score", diff --git a/api/storage/model_schema_storage_test.go b/api/storage/model_schema_storage_test.go index d82b289f1..776fabd96 100644 --- a/api/storage/model_schema_storage_test.go +++ b/api/storage/model_schema_storage_test.go @@ -44,6 +44,8 @@ func Test_modelSchemaStorage_Save(t *testing.T) { ModelID: m.ID, Spec: &models.SchemaSpec{ PredictionIDColumn: "prediction_id", + SessionIDColumn: "session_id", + RowIDColumn: "row_id", TagColumns: []string{"tag"}, FeatureTypes: map[string]models.ValueType{ "featureA": models.Float64, @@ -53,7 +55,7 @@ func Test_modelSchemaStorage_Save(t *testing.T) { }, ModelPredictionOutput: &models.ModelPredictionOutput{ BinaryClassificationOutput: &models.BinaryClassificationOutput{ - ActualLabelColumn: "actual_label", + ActualScoreColumn: "actual_score", NegativeClassLabel: "negative", PositiveClassLabel: "positive", PredictionLabelColumn: "prediction_label", @@ -69,10 +71,9 @@ func Test_modelSchemaStorage_Save(t *testing.T) { schema.Spec.ModelPredictionOutput = &models.ModelPredictionOutput{ RankingOutput: &models.RankingOutput{ - PredictionGroupIDColumn: "session_id", - RankScoreColumn: "score", - RelevanceScoreColumn: "relevance_score", - OutputClass: models.Ranking, + RankScoreColumn: "score", + RelevanceScoreColumn: "relevance_score", + OutputClass: models.Ranking, }, } _, err = modelSchemaStorage.Save(context.Background(), schema) @@ -81,10 +82,9 @@ func Test_modelSchemaStorage_Save(t *testing.T) { require.NoError(t, err) assert.Equal(t, newSchema.Spec.ModelPredictionOutput, &models.ModelPredictionOutput{ RankingOutput: &models.RankingOutput{ - PredictionGroupIDColumn: "session_id", - RankScoreColumn: "score", - RelevanceScoreColumn: "relevance_score", - OutputClass: models.Ranking, + RankScoreColumn: "score", + RelevanceScoreColumn: "relevance_score", + OutputClass: models.Ranking, }, }) }) @@ -123,6 +123,8 @@ func Test_modelSchemaStorage_SaveThroughVersion(t *testing.T) { ModelID: m.ID, Spec: &models.SchemaSpec{ PredictionIDColumn: "prediction_id", + SessionIDColumn: "session_id", + RowIDColumn: "row_id", TagColumns: []string{"tag"}, FeatureTypes: map[string]models.ValueType{ "featureA": models.Float64, @@ -132,7 +134,7 @@ func Test_modelSchemaStorage_SaveThroughVersion(t *testing.T) { }, ModelPredictionOutput: &models.ModelPredictionOutput{ BinaryClassificationOutput: &models.BinaryClassificationOutput{ - ActualLabelColumn: "actual_label", + ActualScoreColumn: "actual_score", NegativeClassLabel: "negative", PositiveClassLabel: "positive", PredictionLabelColumn: "prediction_label", @@ -152,7 +154,7 @@ func Test_modelSchemaStorage_SaveThroughVersion(t *testing.T) { assert.Equal(t, &models.ModelPredictionOutput{ BinaryClassificationOutput: &models.BinaryClassificationOutput{ - ActualLabelColumn: "actual_label", + ActualScoreColumn: "actual_score", NegativeClassLabel: "negative", PositiveClassLabel: "positive", PredictionLabelColumn: "prediction_label", @@ -166,7 +168,7 @@ func Test_modelSchemaStorage_SaveThroughVersion(t *testing.T) { assert.Equal(t, models.ID(1), newSchema.ID) assert.Equal(t, &models.ModelPredictionOutput{ BinaryClassificationOutput: &models.BinaryClassificationOutput{ - ActualLabelColumn: "actual_label", + ActualScoreColumn: "actual_score", NegativeClassLabel: "negative", PositiveClassLabel: "positive", PredictionLabelColumn: "prediction_label", @@ -207,6 +209,8 @@ func Test_modelSchemaStorage_FindAll_Delete(t *testing.T) { ModelID: m.ID, Spec: &models.SchemaSpec{ PredictionIDColumn: "prediction_id", + SessionIDColumn: "session_id", + RowIDColumn: "row_id", TagColumns: []string{"tag"}, FeatureTypes: map[string]models.ValueType{ "featureA": models.Float64, @@ -216,7 +220,7 @@ func Test_modelSchemaStorage_FindAll_Delete(t *testing.T) { }, ModelPredictionOutput: &models.ModelPredictionOutput{ BinaryClassificationOutput: &models.BinaryClassificationOutput{ - ActualLabelColumn: "actual_label", + ActualScoreColumn: "actual_score", NegativeClassLabel: "negative", PositiveClassLabel: "positive", PredictionLabelColumn: "prediction_label", @@ -230,6 +234,8 @@ func Test_modelSchemaStorage_FindAll_Delete(t *testing.T) { ModelID: m.ID, Spec: &models.SchemaSpec{ PredictionIDColumn: "prediction_id", + SessionIDColumn: "session_id", + RowIDColumn: "row_id", TagColumns: []string{"tag"}, FeatureTypes: map[string]models.ValueType{ "featureA": models.Float64, @@ -239,10 +245,9 @@ func Test_modelSchemaStorage_FindAll_Delete(t *testing.T) { }, ModelPredictionOutput: &models.ModelPredictionOutput{ RankingOutput: &models.RankingOutput{ - PredictionGroupIDColumn: "session_id", - RankScoreColumn: "score", - RelevanceScoreColumn: "relevance_score", - OutputClass: models.Ranking, + RankScoreColumn: "score", + RelevanceScoreColumn: "relevance_score", + OutputClass: models.Ranking, }, }, }, @@ -251,6 +256,8 @@ func Test_modelSchemaStorage_FindAll_Delete(t *testing.T) { ModelID: m.ID, Spec: &models.SchemaSpec{ PredictionIDColumn: "prediction_id", + SessionIDColumn: "session_id", + RowIDColumn: "row_id", TagColumns: []string{"tag"}, FeatureTypes: map[string]models.ValueType{ "featureA": models.Float64, @@ -319,6 +326,8 @@ func Test_modelSchemaStorage_FindByID(t *testing.T) { ModelID: m.ID, Spec: &models.SchemaSpec{ PredictionIDColumn: "prediction_id", + SessionIDColumn: "session_id", + RowIDColumn: "row_id", TagColumns: []string{"tag"}, FeatureTypes: map[string]models.ValueType{ "featureA": models.Float64, @@ -328,7 +337,7 @@ func Test_modelSchemaStorage_FindByID(t *testing.T) { }, ModelPredictionOutput: &models.ModelPredictionOutput{ BinaryClassificationOutput: &models.BinaryClassificationOutput{ - ActualLabelColumn: "actual_label", + ActualScoreColumn: "actual_score", NegativeClassLabel: "negative", PositiveClassLabel: "positive", PredictionLabelColumn: "prediction_label", @@ -344,10 +353,9 @@ func Test_modelSchemaStorage_FindByID(t *testing.T) { schema.Spec.ModelPredictionOutput = &models.ModelPredictionOutput{ RankingOutput: &models.RankingOutput{ - PredictionGroupIDColumn: "session_id", - RankScoreColumn: "score", - RelevanceScoreColumn: "relevance_score", - OutputClass: models.Ranking, + RankScoreColumn: "score", + RelevanceScoreColumn: "relevance_score", + OutputClass: models.Ranking, }, } _, err = modelSchemaStorage.Save(context.Background(), schema) @@ -356,10 +364,9 @@ func Test_modelSchemaStorage_FindByID(t *testing.T) { require.NoError(t, err) assert.Equal(t, newSchema.Spec.ModelPredictionOutput, &models.ModelPredictionOutput{ RankingOutput: &models.RankingOutput{ - PredictionGroupIDColumn: "session_id", - RankScoreColumn: "score", - RelevanceScoreColumn: "relevance_score", - OutputClass: models.Ranking, + RankScoreColumn: "score", + RelevanceScoreColumn: "relevance_score", + OutputClass: models.Ranking, }, }) diff --git a/python/observation-publisher/conf/environment/example-override.yaml b/python/observation-publisher/conf/environment/example-override.yaml index d645b450d..16b1022e5 100644 --- a/python/observation-publisher/conf/environment/example-override.yaml +++ b/python/observation-publisher/conf/environment/example-override.yaml @@ -7,7 +7,7 @@ inference_schema: model_prediction_output: output_class: "BinaryClassificationOutput" prediction_score_column: "score" - actual_label_column: "actual_label" + actual_score_column: "actual_score" positive_class_label: "positive" negative_class_label: "negative" score_threshold: 0.5 diff --git a/python/observation-publisher/tests/test_config.py b/python/observation-publisher/tests/test_config.py index 199042aa2..acc94e810 100644 --- a/python/observation-publisher/tests/test_config.py +++ b/python/observation-publisher/tests/test_config.py @@ -24,7 +24,7 @@ def test_config_initialization(): model_prediction_output=dict( output_class="BinaryClassificationOutput", prediction_score_column="score", - actual_label_column="actual_label", + actual_score_column="actual_score", positive_class_label="positive", negative_class_label="negative", score_threshold=0.5, diff --git a/python/observation-publisher/tests/test_observation_sink.py b/python/observation-publisher/tests/test_observation_sink.py index 58b5df3c5..d82192a98 100644 --- a/python/observation-publisher/tests/test_observation_sink.py +++ b/python/observation-publisher/tests/test_observation_sink.py @@ -1,5 +1,4 @@ import dataclasses -import time from datetime import datetime from typing import Optional @@ -12,7 +11,7 @@ from google.cloud.bigquery import SchemaField from merlin.observability.inference import (BinaryClassificationOutput, InferenceSchema, RankingOutput, - ValueType) + ValueType, ObservationType) from pandas._testing import assert_frame_equal from requests import Response @@ -29,7 +28,7 @@ def binary_classification_inference_schema() -> InferenceSchema: }, model_prediction_output=BinaryClassificationOutput( prediction_score_column="prediction_score", - actual_label_column="actual_label", + actual_score_column="actual_score", positive_class_label="fraud", negative_class_label="non fraud", score_threshold=0.5, @@ -42,8 +41,8 @@ def binary_classification_inference_logs() -> pd.DataFrame: request_timestamp = datetime(2024, 1, 1, 0, 0, 0).astimezone(tz.UTC) return pd.DataFrame.from_records( [ - [0.8, 0.4, "1234", "a", request_timestamp, "0.1.0", "non fraud"], - [0.5, 0.9, "1234", "b", request_timestamp, "0.1.0", "fraud"], + [0.8, 0.4, "1234", "a", request_timestamp, "0.1.0"], + [0.5, 0.9, "1234", "b", request_timestamp, "0.1.0"], ], columns=[ "rating", @@ -52,27 +51,40 @@ def binary_classification_inference_logs() -> pd.DataFrame: "row_id", "request_timestamp", "model_version", - "_prediction_label", ], ) +@pytest.fixture +def ranking_inference_schema() -> InferenceSchema: + return InferenceSchema( + feature_types={ + "rating": ValueType.FLOAT64, + }, + model_prediction_output=RankingOutput( + rank_score_column="rank_score", + relevance_score_column="relevance_score_column", + ), + session_id_column="order_id", + row_id_column="driver_id", + ) + + @pytest.fixture def ranking_inference_logs() -> pd.DataFrame: request_timestamp = datetime(2024, 1, 1, 0, 0, 0).astimezone(tz.UTC) return pd.DataFrame.from_records( [ - [5.0, 1.0, "1234", "1001", request_timestamp, 1], - [4.0, 0.9, "1234", "1002", request_timestamp, 1], - [3.0, 0.8, "1234", "1003", request_timestamp, 1], + [5.0, 1.0, "1234", "1001", request_timestamp], + [4.0, 0.9, "1234", "1002", request_timestamp], + [3.0, 0.8, "1234", "1003", request_timestamp], ], columns=[ "rating", "rank_score", "order_id", "driver_id", - "request_timestamp", - "_rank" + "request_timestamp" ], ) @@ -98,7 +110,7 @@ def _post_file( ) -def test_binary_classification_model_preprocessing_for_arize( +def test_binary_classification_model_arize_schema( binary_classification_inference_schema: InferenceSchema, binary_classification_inference_logs: pd.DataFrame, ): @@ -109,31 +121,26 @@ def test_binary_classification_model_preprocessing_for_arize( "0.1.0", arize_client, ) - arize_sink.write(binary_classification_inference_logs) + df = binary_classification_inference_schema.preprocess( + binary_classification_inference_logs, + [ObservationType.PREDICTION] + ) + arize_sink.write(df) -def test_ranking_model_preprocessing_for_arize( +def test_ranking_model_arize_schema( + ranking_inference_schema: InferenceSchema, ranking_inference_logs: pd.DataFrame, ): - inference_schema = InferenceSchema( - feature_types={ - "rating": ValueType.FLOAT64, - }, - model_prediction_output=RankingOutput( - rank_score_column="rank_score", - relevance_score_column="relevance_score_column", - ), - session_id_column="order_id", - row_id_column="driver_id", - ) arize_client = MockArizeClient(api_key="test", space_key="test") arize_sink = ArizeSink( - inference_schema, + ranking_inference_schema, "test-model", "0.1.0", arize_client, ) - arize_sink.write(ranking_inference_logs) + df = ranking_inference_schema.preprocess(ranking_inference_logs, [ObservationType.PREDICTION]) + arize_sink.write(df) @pytest.mark.integration diff --git a/python/sdk/client/models/binary_classification_output.py b/python/sdk/client/models/binary_classification_output.py index 253c9df07..9cf020c12 100644 --- a/python/sdk/client/models/binary_classification_output.py +++ b/python/sdk/client/models/binary_classification_output.py @@ -31,7 +31,7 @@ class BinaryClassificationOutput(BaseModel): BinaryClassificationOutput """ # noqa: E501 prediction_score_column: StrictStr - actual_score_column: Optional[StrictStr] = None + actual_score_column: StrictStr positive_class_label: StrictStr negative_class_label: StrictStr score_threshold: Optional[Union[StrictFloat, StrictInt]] = None diff --git a/python/sdk/client/models/regression_output.py b/python/sdk/client/models/regression_output.py index ca12db692..68fe5b604 100644 --- a/python/sdk/client/models/regression_output.py +++ b/python/sdk/client/models/regression_output.py @@ -31,7 +31,7 @@ class RegressionOutput(BaseModel): RegressionOutput """ # noqa: E501 prediction_score_column: StrictStr - actual_score_column: Optional[StrictStr] = None + actual_score_column: StrictStr output_class: ModelPredictionOutputClass __properties: ClassVar[List[str]] = ["prediction_score_column", "actual_score_column", "output_class"] diff --git a/python/sdk/merlin/model_schema.py b/python/sdk/merlin/model_schema.py index 821f59928..4f05e532e 100644 --- a/python/sdk/merlin/model_schema.py +++ b/python/sdk/merlin/model_schema.py @@ -11,11 +11,10 @@ PredictionOutput, RankingOutput, RegressionOutput, - ValueType, + ValueType, DEFAULT_PREDICTION_ID_COLUMN, DEFAULT_SESSION_ID_COLUMN, DEFAULT_ROW_ID_COLUMN, ) from merlin.util import autostr, extract_optional_value_with_default - @autostr @dataclass_json @dataclass @@ -57,7 +56,9 @@ def from_model_schema_response( model_id=response.model_id, spec=InferenceSchema( feature_types=feature_types, - prediction_id_column=response_spec.prediction_id_column, + prediction_id_column=extract_optional_value_with_default(response_spec.prediction_id_column, DEFAULT_PREDICTION_ID_COLUMN), + session_id_column=extract_optional_value_with_default(response_spec.session_id_column, DEFAULT_SESSION_ID_COLUMN), + row_id_column=extract_optional_value_with_default(response_spec.row_id_column, DEFAULT_ROW_ID_COLUMN), tag_columns=response_spec.tag_columns, model_prediction_output=prediction_output, ), @@ -77,9 +78,7 @@ def model_prediction_output_from_response( if isinstance(actual_instance, client.BinaryClassificationOutput): return BinaryClassificationOutput( prediction_score_column=actual_instance.prediction_score_column, - actual_label_column=extract_optional_value_with_default( - actual_instance.actual_label_column, "" - ), + actual_score_column=actual_instance.actual_score_column, positive_class_label=actual_instance.positive_class_label, negative_class_label=actual_instance.negative_class_label, score_threshold=extract_optional_value_with_default( @@ -88,9 +87,7 @@ def model_prediction_output_from_response( ) elif isinstance(actual_instance, client.RegressionOutput): return RegressionOutput( - actual_score_column=extract_optional_value_with_default( - actual_instance.actual_score_column, "" - ), + actual_score_column=actual_instance.actual_score_column, prediction_score_column=actual_instance.prediction_score_column, ) elif isinstance(actual_instance, client.RankingOutput): @@ -98,7 +95,6 @@ def model_prediction_output_from_response( relevance_score_column=extract_optional_value_with_default( actual_instance.relevance_score_column, "" ), - prediction_group_id_column=actual_instance.prediction_group_id_column, rank_score_column=actual_instance.rank_score_column, ) raise ValueError( @@ -111,7 +107,7 @@ def _to_client_prediction_output_spec(self) -> client.ModelPredictionOutput: return client.ModelPredictionOutput( client.BinaryClassificationOutput( prediction_score_column=prediction_output.prediction_score_column, - actual_label_column=prediction_output.actual_label_column, + actual_score_column=prediction_output.actual_score_column, positive_class_label=prediction_output.positive_class_label, negative_class_label=prediction_output.negative_class_label, score_threshold=prediction_output.score_threshold, @@ -134,7 +130,6 @@ def _to_client_prediction_output_spec(self) -> client.ModelPredictionOutput: return client.ModelPredictionOutput( client.RankingOutput( relevance_score_column=prediction_output.relevance_score_column, - prediction_group_id_column=prediction_output.prediction_group_id_column, rank_score_column=prediction_output.rank_score_column, output_class=client.ModelPredictionOutputClass( RankingOutput.__name__ diff --git a/python/sdk/merlin/observability/inference.py b/python/sdk/merlin/observability/inference.py index dc6287069..16caf2bcf 100644 --- a/python/sdk/merlin/observability/inference.py +++ b/python/sdk/merlin/observability/inference.py @@ -8,6 +8,11 @@ from dataclasses_json import DataClassJsonMixin, config, dataclass_json +DEFAULT_PREDICTION_ID_COLUMN = "prediction_id" +DEFAULT_SESSION_ID_COLUMN = "session_id" +DEFAULT_ROW_ID_COLUMN = "row_id" + + class ObservationType(Enum): """ Supported observation types. @@ -265,9 +270,9 @@ class InferenceSchema: decoder=PredictionOutput.decode, ) ) - session_id_column: str = "session_id" - row_id_column: str = "row_id" - prediction_id_column: str = "prediction_id" + session_id_column: str = DEFAULT_SESSION_ID_COLUMN + row_id_column: str = DEFAULT_ROW_ID_COLUMN + prediction_id_column: str = DEFAULT_PREDICTION_ID_COLUMN tag_columns: Optional[List[str]] = None @property diff --git a/python/sdk/test/integration_test.py b/python/sdk/test/integration_test.py index d477eb2dc..3645986b3 100644 --- a/python/sdk/test/integration_test.py +++ b/python/sdk/test/integration_test.py @@ -190,7 +190,7 @@ def test_model_schema( prediction_id_column="prediction_id", model_prediction_output=BinaryClassificationOutput( prediction_score_column="score", - actual_label_column="actual", + actual_score_column="actual", positive_class_label="completed", negative_class_label="non_complete", score_threshold=0.7 diff --git a/python/sdk/test/model_schema_test.py b/python/sdk/test/model_schema_test.py index 500745e62..9d98b05de 100644 --- a/python/sdk/test/model_schema_test.py +++ b/python/sdk/test/model_schema_test.py @@ -23,7 +23,7 @@ model_prediction_output=client.ModelPredictionOutput( client.BinaryClassificationOutput( prediction_score_column="prediction_score", - actual_label_column="actual_label", + actual_score_column="actual_score", positive_class_label="positive", negative_class_label="negative", score_threshold=0.5, @@ -46,7 +46,7 @@ }, model_prediction_output=BinaryClassificationOutput( prediction_score_column="prediction_score", - actual_label_column="actual_label", + actual_score_column="actual_score", positive_class_label="positive", negative_class_label="negative", score_threshold=0.5 @@ -113,7 +113,6 @@ model_prediction_output=client.ModelPredictionOutput( client.RankingOutput( rank_score_column="score", - prediction_group_id_column="session_id", relevance_score_column="relevance_score", output_class=client.ModelPredictionOutputClass.RANKINGOUTPUT ) @@ -134,7 +133,6 @@ }, model_prediction_output=RankingOutput( rank_score_column="score", - prediction_group_id_column="session_id", relevance_score_column="relevance_score" ) ), diff --git a/python/sdk/test/model_test.py b/python/sdk/test/model_test.py index 62ff07ded..179daaab5 100644 --- a/python/sdk/test/model_test.py +++ b/python/sdk/test/model_test.py @@ -1332,7 +1332,6 @@ class TestModel: model_prediction_output=client.ModelPredictionOutput( client.RankingOutput( rank_score_column="score", - prediction_group_id_column="session_id", relevance_score_column="relevance_score", output_class=client.ModelPredictionOutputClass.RANKINGOUTPUT ) @@ -1353,7 +1352,6 @@ class TestModel: }, model_prediction_output=RankingOutput( rank_score_column="score", - prediction_group_id_column="session_id", relevance_score_column="relevance_score" ) ), diff --git a/swagger.yaml b/swagger.yaml index e83dd57f0..b572803b7 100644 --- a/swagger.yaml +++ b/swagger.yaml @@ -1424,6 +1424,7 @@ components: type: object required: - prediction_score_column + - actual_score_column - positive_class_label - negative_class_label - output_class