From 81705006f9a0dc490972c24fadbd1bc00f400e18 Mon Sep 17 00:00:00 2001 From: rasbt Date: Mon, 22 Apr 2024 19:24:06 +0000 Subject: [PATCH 1/5] Use new litserve accelerator="auto" setting --- litgpt/deploy/serve.py | 5 +++-- pyproject.toml | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/litgpt/deploy/serve.py b/litgpt/deploy/serve.py index 9df48ad98d..a6c5335ee9 100644 --- a/litgpt/deploy/serve.py +++ b/litgpt/deploy/serve.py @@ -99,7 +99,7 @@ def run_server( top_k: int = 200, max_new_tokens: int = 50, devices: int = 1, - accelerator: str = "cuda", + accelerator: str = "auto", port: int = 8000 ) -> None: """Serve a LitGPT model using LitServe @@ -114,7 +114,8 @@ def run_server( generated text but can also lead to more incoherent texts. max_new_tokens: The number of generation steps to take. devices: How many devices/GPUs to use. - accelerator: The type of accelerator to use. For example, "cuda" or "cpu". + accelerator: The type of accelerator to use. For example, "auto", "cuda", "cpu", or "mps".capitalize + The "auto" setting (default) chooses a GPU if available, and otherwise uses a CPU. port: The network port number on which the model is configured to be served. """ check_valid_checkpoint_dir(checkpoint_dir, model_filename="lit_model.pth") diff --git a/pyproject.toml b/pyproject.toml index b6fbec18b8..f0bd0d2ebc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,7 @@ dependencies = [ "torch>=2.2.0", "lightning==2.3.0.dev20240328", "jsonargparse[signatures]>=4.27.6", - "litserve==0.0.0.dev2", # imported by litgpt.deploy + "litserve @ git+https://github.com/Lightning-AI/litserve.git@aefcb13721b97b9da4ed9f7166b2f870bedfd92f", # imported by litgpt.deploy ] [project.urls] From d7a4ac45a3b9f89793aa6a1d5a078022bb5a0580 Mon Sep 17 00:00:00 2001 From: Sebastian Raschka Date: Mon, 22 Apr 2024 14:25:10 -0500 Subject: [PATCH 2/5] Update litgpt/deploy/serve.py --- litgpt/deploy/serve.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litgpt/deploy/serve.py b/litgpt/deploy/serve.py index a6c5335ee9..64ca25f771 100644 --- a/litgpt/deploy/serve.py +++ b/litgpt/deploy/serve.py @@ -114,7 +114,7 @@ def run_server( generated text but can also lead to more incoherent texts. max_new_tokens: The number of generation steps to take. devices: How many devices/GPUs to use. - accelerator: The type of accelerator to use. For example, "auto", "cuda", "cpu", or "mps".capitalize + accelerator: The type of accelerator to use. For example, "auto", "cuda", "cpu", or "mps". The "auto" setting (default) chooses a GPU if available, and otherwise uses a CPU. port: The network port number on which the model is configured to be served. """ From 6fff15afb5612ee178cdb01e69f6286d920c344c Mon Sep 17 00:00:00 2001 From: rasbt Date: Mon, 22 Apr 2024 20:38:24 +0000 Subject: [PATCH 3/5] update devices --- litgpt/deploy/serve.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litgpt/deploy/serve.py b/litgpt/deploy/serve.py index 64ca25f771..ebab4cbb32 100644 --- a/litgpt/deploy/serve.py +++ b/litgpt/deploy/serve.py @@ -40,7 +40,7 @@ def setup(self, device: str) -> None: fabric = L.Fabric( accelerator=device.type, - devices=1 if device.type=="cpu" else [device.index], # TODO: Update once LitServe supports "auto" + devices=[device.index], precision=precision, ) checkpoint_path = self.checkpoint_dir / "lit_model.pth" From fe57f8674ae5a5e11031511f9c47c70bca5706ad Mon Sep 17 00:00:00 2001 From: rasbt Date: Mon, 22 Apr 2024 22:20:21 +0000 Subject: [PATCH 4/5] restore --- litgpt/deploy/serve.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litgpt/deploy/serve.py b/litgpt/deploy/serve.py index ebab4cbb32..4a26e0b14f 100644 --- a/litgpt/deploy/serve.py +++ b/litgpt/deploy/serve.py @@ -40,7 +40,7 @@ def setup(self, device: str) -> None: fabric = L.Fabric( accelerator=device.type, - devices=[device.index], + devices=1 if device.type=="cpu" else [device.index], precision=precision, ) checkpoint_path = self.checkpoint_dir / "lit_model.pth" From 680b0ce3657c22fb07dc32290ce3ccafca954caf Mon Sep 17 00:00:00 2001 From: Sebastian Raschka Date: Wed, 24 Apr 2024 08:11:11 -0500 Subject: [PATCH 5/5] Update litserve --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index f0bd0d2ebc..1343429619 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,7 @@ dependencies = [ "torch>=2.2.0", "lightning==2.3.0.dev20240328", "jsonargparse[signatures]>=4.27.6", - "litserve @ git+https://github.com/Lightning-AI/litserve.git@aefcb13721b97b9da4ed9f7166b2f870bedfd92f", # imported by litgpt.deploy + "litserve>=0.1.0" # imported by litgpt.deploy ] [project.urls]