diff --git a/api/queue/work/model_service_deployment.go b/api/queue/work/model_service_deployment.go index ce8f4ace4..52517ef50 100644 --- a/api/queue/work/model_service_deployment.go +++ b/api/queue/work/model_service_deployment.go @@ -107,11 +107,9 @@ func (depl *ModelServiceDeployment) Deploy(job *queue.Job) error { } defer func() { - deploymentCounter.WithLabelValues(model.Project.Name, model.Name, fmt.Sprint(endpoint.Status), fmt.Sprint(isRedeployment)).Inc() + deploymentCounter.WithLabelValues(model.Project.Name, model.Name, fmt.Sprint(deployment.Status), fmt.Sprint(isRedeployment)).Inc() // record the deployment result - deployment.Status = endpoint.Status - deployment.Error = endpoint.Message deployment.UpdatedAt = time.Now() if _, err := depl.DeploymentStorage.Save(deployment); err != nil { log.Warnf("unable to update deployment history", err) @@ -125,6 +123,8 @@ func (depl *ModelServiceDeployment) Deploy(job *queue.Job) error { modelOpt, err := depl.generateModelOptions(ctx, model, version) if err != nil { + deployment.Status = models.EndpointFailed + deployment.Error = err.Error() endpoint.Message = err.Error() return err } @@ -132,12 +132,17 @@ func (depl *ModelServiceDeployment) Deploy(job *queue.Job) error { modelService := models.NewService(model, version, modelOpt, endpoint) ctl, ok := depl.ClusterControllers[endpoint.EnvironmentName] if !ok { + deployment.Status = models.EndpointFailed + deployment.Error = err.Error() + endpoint.Message = err.Error() return fmt.Errorf("unable to find cluster controller for environment %s", endpoint.EnvironmentName) } svc, err := ctl.Deploy(ctx, modelService) if err != nil { log.Errorf("unable to deploy version endpoint for model: %s, version: %s, reason: %v", model.Name, version.ID, err) + deployment.Status = models.EndpointFailed + deployment.Error = err.Error() endpoint.Message = err.Error() return err } @@ -154,6 +159,8 @@ func (depl *ModelServiceDeployment) Deploy(job *queue.Job) error { endpoint.InferenceServiceName = svc.CurrentIsvcName endpoint.Message = "" // reset message + deployment.Status = endpoint.Status + return nil } diff --git a/ui/src/pages/version/HistoryDetails.js b/ui/src/pages/version/HistoryDetails.js index f732022f3..4de931aac 100644 --- a/ui/src/pages/version/HistoryDetails.js +++ b/ui/src/pages/version/HistoryDetails.js @@ -46,7 +46,8 @@ const RevisionPanel = ({ deployments, deploymentsLoaded, endpoint }) => { const deployedRevision = orderedDeployments.find( (deployment) => - deployment.status === "running" || deployment.status === "serving" + (deployment.status === "running" || deployment.status === "serving") && + deployment.error === "" ) || { id: null }; const canBeExpanded = (deployment) => {