feat: Per-model metric customization (#126)

triton-inference-server · Nov 6, 2024 · 3948525 · 3948525
1 parent 578491f
commit 3948525
Show file tree

Hide file tree

Showing 2 changed files with 87 additions and 1 deletion.
diff --git a/include/triton/common/error.h b/include/triton/common/error.h
@@ -25,8 +25,8 @@
 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #pragma once
 
-#include <string>
 #include <cstdint>
+#include <string>
 
 namespace triton { namespace common {
 

diff --git a/protobuf/model_config.proto b/protobuf/model_config.proto
@@ -1869,6 +1869,85 @@ message ModelResponseCache
   bool enable = 1;
 }
 
+//@@
+//@@  .. cpp:var:: message ModelMetrics
+//@@
+//@@     The metrics setting of this model.
+//@@     NOTE: Consider reusing this message body for backend metric custom
+//@@     configuration.
+//@@
+message ModelMetrics
+{
+  //@@
+  //@@  .. cpp:var:: message MetricControl
+  //@@
+  //@@     Override metrics settings of this model.
+  //@@
+  message MetricControl
+  {
+    //@@
+    //@@  .. cpp:var:: message MetricIdentifier
+    //@@
+    //@@     Specify metrics to be overridden with metric_option.
+    //@@
+    message MetricIdentifier
+    {
+      //@@  .. cpp:var:: string family
+      //@@
+      //@@     The name of the metric family to override with the custom value.
+      //@@     All core histogram metrics reported by Triton are customizable.
+      //@@
+      // https://github.com/triton-inference-server/server/blob/main/docs/user_guide/metrics.md#histograms
+      //@@
+      string family = 1;
+    }
+
+    //@@  .. cpp:var:: message HistogramOptions
+    //@@
+    //@@     Histogram metrics options.
+    //@@
+    message HistogramOptions
+    {
+      //@@  .. cpp:var:: double buckets (repeated)
+      //@@
+      //@@     Repeated double type in ascending order for histogram bucket
+      //@@     boundaries. Each bucket value represents a range less than or
+      //@@     equal to itself. The range greater than the largest bucket value
+      //@@     is allocated implicitly.
+      //@@     For example, [ -5.0, -2, 0, 3.5, 5 ].
+      //@@
+      repeated double buckets = 1;
+    }
+
+    //@@  .. cpp:var:: MetricIdentifier metric_identifier
+    //@@
+    //@@     The identifier defining metrics to be overridden with the
+    //@@     metric_options.
+    //@@
+    MetricIdentifier metric_identifier = 1;
+
+    //@@  .. cpp:var:: oneof metric_options
+    //@@
+    //@@     The value to override the metrics defined in metric_identifier.
+    //@@
+    oneof metric_options
+    {
+      //@@  .. cpp:var:: HistogramOptions histogram_options
+      //@@
+      //@@     Histogram options.
+      //@@
+      HistogramOptions histogram_options = 2;
+    }
+  }
+
+  //@@
+  //@@  .. cpp::var:: MetricControl metric_control (repeated)
+  //@@
+  //@@     Optional custom configuration for selected metrics.
+  //@@
+  repeated MetricControl metric_control = 1;
+}
+
 //@@
 //@@.. cpp:var:: message ModelConfig
 //@@
@@ -2076,4 +2155,11 @@ message ModelConfig
   //@@     model.
   //@@
   ModelResponseCache response_cache = 24;
+
+  //@@  .. cpp:var:: ModelMetrics model_metrics
+  //@@
+  //@@     Optional setting for custom metrics configuration for this model.
+  //@@     Application default is applied to metrics that are not specified.
+  //@@
+  ModelMetrics model_metrics = 26;
 }