diff --git a/requirements-common.txt b/requirements-common.txt index 3de0f98e7c0c3..3cc7bba8f84db 100644 --- a/requirements-common.txt +++ b/requirements-common.txt @@ -12,7 +12,7 @@ uvicorn[standard] pydantic >= 2.0 # Required for OpenAI server. prometheus_client >= 0.18.0 tiktoken == 0.6.0 # Required for DBRX tokenizer -lm-format-enforcer == 0.9.3 +lm-format-enforcer == 0.9.8 outlines == 0.0.34 # Requires torch >= 2.1.0 typing_extensions filelock >= 3.10.4 # filelock starts to support `mode` argument from 3.10.4 diff --git a/vllm/engine/arg_utils.py b/vllm/engine/arg_utils.py index 53f129598270c..8939a0d537281 100644 --- a/vllm/engine/arg_utils.py +++ b/vllm/engine/arg_utils.py @@ -197,7 +197,11 @@ def add_cli_args( default='outlines', choices=['outlines', 'lm-format-enforcer'], help='Which engine will be used for guided decoding' - ' (JSON schema / regex etc).') + ' (JSON schema / regex etc) by default. Currently support ' + 'https://github.com/outlines-dev/outlines and ' + 'https://github.com/noamgat/lm-format-enforcer.' + ' Can be overridden per request via guided_decoding_backend' + ' parameter.') # Parallel arguments parser.add_argument('--worker-use-ray', action='store_true', diff --git a/vllm/model_executor/layers/quantization/fp8.py b/vllm/model_executor/layers/quantization/fp8.py index 9dc0e86e1243d..8df82e0e18edd 100644 --- a/vllm/model_executor/layers/quantization/fp8.py +++ b/vllm/model_executor/layers/quantization/fp8.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, Optional, Tuple import torch from torch.nn import Module @@ -114,7 +114,7 @@ def apply_weights(self, return output -def per_tensor_quantize(tensor: torch.Tensor) -> tuple[torch.Tensor, float]: +def per_tensor_quantize(tensor: torch.Tensor) -> Tuple[torch.Tensor, float]: """Quantize a tensor using per-tensor static scaling factor. Args: