Merge pull request #56 from stackhpc/fix/mistral-tokenizer

Default to mistral tokenizer for mistral models
stackhpc · Nov 2, 2024 · 7a8ce15 · 7a8ce15
2 parents 3d6c6ea + 4e28893
commit 7a8ce15
Showing 1 changed file with 4 additions and 0 deletions.
diff --git a/charts/azimuth-llm/templates/api/deployment.yml b/charts/azimuth-llm/templates/api/deployment.yml
@@ -34,6 +34,10 @@ spec:
           - --max-model-len
           - {{ .Values.api.modelMaxContextLength | quote }}
           {{- end -}}
+          {{- if and (not (has "--tokenizer-model" .Values.api.extraArgs)) (hasPrefix "mistralai/" .Values.huggingface.model) -}}
+          - --tokenizer-mode
+          - mistral
+          {{- end -}}
           {{- if .Values.api.extraArgs -}}
           {{- .Values.api.extraArgs | toYaml | nindent 10 }}
           {{- end -}}