-
Notifications
You must be signed in to change notification settings - Fork 43
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat: Add observability publisher deployment (#550)
<!-- Thanks for sending a pull request! Here are some tips for you: 1. Run unit tests and ensure that they are passing 2. If your change introduces any API changes, make sure to update the e2e tests 3. Make sure documentation is updated for your PR! --> # Description <!-- Briefly describe the motivation for the change. Please include illustrations where appropriate. --> Add capability to deploy observability publisher (please suggest the better name if you have any) that consume prediction log that is produced by the model. This observability publisher will be deployed once the model is serving or model redeployment if the model already serve the traffic. # Modifications <!-- Summarize the key code changes. --> * Adding event producer (`api/pkg/observability/event/event.go`) for observability publisher. This event producer will produce deployment event for observability publisher, which later on is consumed by the deployment worker in merlin control plane to do actual deployment. * Actual action for deployment (`api/queue/work/observability_publisher_deployment.go`) that consume event that is produced by the producer. There are several condition: * Deployment is skip if the more latest revision is already queued (revision in DB is greater than the one is observed by the `observability_publisher_deployment` * Deployment will be delayed by requeue if there is still ongoing deployment for previous revision * Deployment will be skip if there is ongoing deployment for newer revision compare to current observed revision by the `observability_publisher_deployment` Once the conditions are met, it will deploy or undeploy k8s deployment and secret manifest * Add deployer (`api/pkg/observability/deployment/deployer.go`) to interact with k8s control plane. There are 3 methods in this deployer * `Deploy` -> Create or update k8s secret and deployment manifest * `Undeploy` -> Delete k8s secret and deployment manifest * `GetDeployedManifest` -> Get deployed manifest that contains k8s secret and manifest, and status of deployment * Adding new field in model `observability_supported` as a gate whether the model is allowed to publish observability data. Observability publisher only be deployed if the model `observability_supported` is true and the model endpoint also enable model observability # Tests <!-- Besides the existing / updated automated tests, what specific scenarios should be tested? Consider the backward compatibility of the changes, whether corner cases are covered, etc. Please describe the tests and check the ones that have been completed. Eg: - [x] Deploying new and existing standard models - [ ] Deploying PyFunc models --> # Checklist - [x] Added PR label - [x] Added unit test, integration, and/or e2e tests - [x] Tested locally - [ ] Updated documentation - [ ] Update Swagger spec if the PR introduce API changes - [ ] Regenerated Golang and Python client if the PR introduces API changes # Release Notes <!-- Does this PR introduce a user-facing change? If no, just write "NONE" in the release-note block below. If yes, a release note is required. Enter your extended release note in the block below. If the PR requires additional action from users switching to the new release, include the string "action required". For more information about release notes, see kubernetes' guide here: http://git.k8s.io/community/contributors/guide/release-notes.md --> ```release-note ```
- Loading branch information
1 parent
82fe2bf
commit 444f9eb
Showing
36 changed files
with
4,982 additions
and
94 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,38 @@ | ||
| package config | ||
|
|
||
| import "time" | ||
|
|
||
| // ObservabilityPublisher | ||
| type ObservabilityPublisher struct { | ||
| ArizeSink ArizeSink | ||
| BigQuerySink BigQuerySink | ||
| KafkaConsumer KafkaConsumer | ||
| ImageName string | ||
| DefaultResources ResourceRequestsLimits | ||
| EnvironmentName string | ||
| Replicas int32 | ||
| TargetNamespace string | ||
| ServiceAccountName string | ||
| DeploymentTimeout time.Duration `default:"30m"` | ||
| } | ||
|
|
||
| // KafkaConsumer | ||
| type KafkaConsumer struct { | ||
| Brokers string `validate:"required"` | ||
| BatchSize int | ||
| GroupID string | ||
| AdditionalConsumerConfig map[string]string | ||
| } | ||
|
|
||
| // ArizeSink | ||
| type ArizeSink struct { | ||
| APIKey string | ||
| SpaceKey string | ||
| } | ||
|
|
||
| // BigQuerySink | ||
| type BigQuerySink struct { | ||
| Project string | ||
| Dataset string | ||
| TTLDays int | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,67 @@ | ||
| package models | ||
|
|
||
| import ( | ||
| "fmt" | ||
| ) | ||
|
|
||
| // PublisherStatus | ||
| type PublisherStatus string | ||
|
|
||
| const ( | ||
| Pending PublisherStatus = "pending" | ||
| Running PublisherStatus = "running" | ||
| Failed PublisherStatus = "failed" | ||
| Terminated PublisherStatus = "terminated" | ||
| ) | ||
|
|
||
| // ObservabilityPublisher | ||
| type ObservabilityPublisher struct { | ||
| ID ID `gorm:"id"` | ||
| VersionModelID ID `gorm:"version_model_id"` | ||
| VersionID ID `gorm:"version_id"` | ||
| Revision int `gorm:"revision"` | ||
| Status PublisherStatus `gorm:"status"` | ||
| ModelSchemaSpec *SchemaSpec `gorm:"model_schema_spec"` | ||
| CreatedUpdated | ||
| } | ||
|
|
||
| type ActionType string | ||
|
|
||
| const ( | ||
| DeployPublisher ActionType = "deploy" | ||
| UndeployPublisher ActionType = "delete" | ||
| ) | ||
|
|
||
| type WorkerData struct { | ||
| Project string | ||
| ModelSchemaSpec *SchemaSpec | ||
| Metadata Metadata | ||
| ModelName string | ||
| ModelVersion string | ||
| Revision int | ||
| TopicSource string | ||
| } | ||
|
|
||
| func NewWorkerData(modelVersion *Version, model *Model, observabilityPublisher *ObservabilityPublisher) *WorkerData { | ||
| return &WorkerData{ | ||
| ModelName: model.Name, | ||
| Project: model.Project.Name, | ||
| ModelSchemaSpec: observabilityPublisher.ModelSchemaSpec, | ||
| Metadata: Metadata{ | ||
| App: fmt.Sprintf("%s-observability-publisher", model.Name), | ||
| Component: "worker", | ||
| Stream: model.Project.Stream, | ||
| Team: model.Project.Team, | ||
| Labels: model.Project.Labels, | ||
| }, | ||
| ModelVersion: modelVersion.ID.String(), | ||
| Revision: observabilityPublisher.Revision, | ||
| TopicSource: getPredictionLogTopicForVersion(model.Project.Name, model.Name, modelVersion.ID.String()), | ||
| } | ||
| } | ||
|
|
||
| type ObservabilityPublisherJob struct { | ||
| ActionType ActionType | ||
| Publisher *ObservabilityPublisher | ||
| WorkerData *WorkerData | ||
| } |
Oops, something went wrong.