From f9903d850f9118b82c0396844d530c72685b6480 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Oct 2024 20:05:36 +0000 Subject: [PATCH 01/10] chore(deps): Bump charset-normalizer from 3.3.2 to 3.4.0 in /examples/langchain/langchainpy-localai-example (#3834) chore(deps): Bump charset-normalizer Bumps [charset-normalizer](https://github.com/Ousret/charset_normalizer) from 3.3.2 to 3.4.0. - [Release notes](https://github.com/Ousret/charset_normalizer/releases) - [Changelog](https://github.com/jawah/charset_normalizer/blob/master/CHANGELOG.md) - [Commits](https://github.com/Ousret/charset_normalizer/compare/3.3.2...3.4.0) --- updated-dependencies: - dependency-name: charset-normalizer dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- examples/langchain/langchainpy-localai-example/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/langchain/langchainpy-localai-example/requirements.txt b/examples/langchain/langchainpy-localai-example/requirements.txt index 4fa902497273..ff0bcce6f971 100644 --- a/examples/langchain/langchainpy-localai-example/requirements.txt +++ b/examples/langchain/langchainpy-localai-example/requirements.txt @@ -3,7 +3,7 @@ aiosignal==1.3.1 async-timeout==4.0.3 attrs==24.2.0 certifi==2024.8.30 -charset-normalizer==3.3.2 +charset-normalizer==3.4.0 colorama==0.4.6 dataclasses-json==0.6.7 debugpy==1.8.7 From ddd289d1af5a40852d3f22ea8e26341453b33a20 Mon Sep 17 00:00:00 2001 From: "LocalAI [bot]" <139863280+localai-bot@users.noreply.github.com> Date: Tue, 15 Oct 2024 00:03:40 +0200 Subject: [PATCH 02/10] chore: :arrow_up: Update ggerganov/llama.cpp to `a89f75e1b7b90cb2d4d4c52ca53ef9e9b466aa45` (#3837) :arrow_up: Update ggerganov/llama.cpp Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 739e0f9af160..47c5de4f9399 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ DETECT_LIBS?=true # llama.cpp versions GOLLAMA_REPO?=https://github.com/go-skynet/go-llama.cpp GOLLAMA_VERSION?=2b57a8ae43e4699d3dc5d1496a1ccd42922993be -CPPLLAMA_VERSION?=d4c19c0f5cdb1e512573e8c86c79e8d0238c73c4 +CPPLLAMA_VERSION?=a89f75e1b7b90cb2d4d4c52ca53ef9e9b466aa45 # go-rwkv version RWKV_REPO?=https://github.com/donomii/go-rwkv.cpp From 13e7432b890c630ea7b888ff2aed7d5df4501b13 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Oct 2024 09:39:55 +0200 Subject: [PATCH 03/10] chore(deps): Bump langchain-community from 0.3.1 to 0.3.2 in /examples/langchain/langchainpy-localai-example (#3831) chore(deps): Bump langchain-community Bumps [langchain-community](https://github.com/langchain-ai/langchain) from 0.3.1 to 0.3.2. - [Release notes](https://github.com/langchain-ai/langchain/releases) - [Commits](https://github.com/langchain-ai/langchain/compare/langchain-community==0.3.1...langchain-community==0.3.2) --- updated-dependencies: - dependency-name: langchain-community dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- examples/langchain/langchainpy-localai-example/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/langchain/langchainpy-localai-example/requirements.txt b/examples/langchain/langchainpy-localai-example/requirements.txt index ff0bcce6f971..647a95ca1a76 100644 --- a/examples/langchain/langchainpy-localai-example/requirements.txt +++ b/examples/langchain/langchainpy-localai-example/requirements.txt @@ -11,7 +11,7 @@ frozenlist==1.4.1 greenlet==3.1.1 idna==3.10 langchain==0.3.3 -langchain-community==0.3.1 +langchain-community==0.3.2 marshmallow==3.22.0 marshmallow-enum==1.5.1 multidict==6.1.0 From 53d1db1da0aec8eb070146198d24282dba4f20cd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Oct 2024 09:40:06 +0200 Subject: [PATCH 04/10] chore(deps): Bump yarl from 1.15.1 to 1.15.2 in /examples/langchain/langchainpy-localai-example (#3832) chore(deps): Bump yarl Bumps [yarl](https://github.com/aio-libs/yarl) from 1.15.1 to 1.15.2. - [Release notes](https://github.com/aio-libs/yarl/releases) - [Changelog](https://github.com/aio-libs/yarl/blob/master/CHANGES.rst) - [Commits](https://github.com/aio-libs/yarl/compare/v1.15.1...v1.15.2) --- updated-dependencies: - dependency-name: yarl dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- examples/langchain/langchainpy-localai-example/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/langchain/langchainpy-localai-example/requirements.txt b/examples/langchain/langchainpy-localai-example/requirements.txt index 647a95ca1a76..6916f4cefb51 100644 --- a/examples/langchain/langchainpy-localai-example/requirements.txt +++ b/examples/langchain/langchainpy-localai-example/requirements.txt @@ -30,4 +30,4 @@ tqdm==4.66.5 typing-inspect==0.9.0 typing_extensions==4.12.2 urllib3==2.2.3 -yarl==1.15.1 +yarl==1.15.2 From 18c35ee86f2990e0fec914cfa2de2cc62153015c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Oct 2024 09:40:30 +0200 Subject: [PATCH 05/10] chore(deps): Bump numpy from 2.1.1 to 2.1.2 in /examples/langchain/langchainpy-localai-example (#3833) chore(deps): Bump numpy Bumps [numpy](https://github.com/numpy/numpy) from 2.1.1 to 2.1.2. - [Release notes](https://github.com/numpy/numpy/releases) - [Changelog](https://github.com/numpy/numpy/blob/main/doc/RELEASE_WALKTHROUGH.rst) - [Commits](https://github.com/numpy/numpy/compare/v2.1.1...v2.1.2) --- updated-dependencies: - dependency-name: numpy dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- examples/langchain/langchainpy-localai-example/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/langchain/langchainpy-localai-example/requirements.txt b/examples/langchain/langchainpy-localai-example/requirements.txt index 6916f4cefb51..d4dfe947be5e 100644 --- a/examples/langchain/langchainpy-localai-example/requirements.txt +++ b/examples/langchain/langchainpy-localai-example/requirements.txt @@ -17,7 +17,7 @@ marshmallow-enum==1.5.1 multidict==6.1.0 mypy-extensions==1.0.0 numexpr==2.10.1 -numpy==2.1.1 +numpy==2.1.2 openai==1.51.2 openapi-schema-pydantic==1.2.4 packaging>=23.2 From 18f9e11f1a35d29c688ce82e4a574f2d47f6d21d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Oct 2024 09:40:53 +0200 Subject: [PATCH 06/10] chore(deps): Bump docs/themes/hugo-theme-relearn from `e1a1f01` to `007cc20` (#3835) chore(deps): Bump docs/themes/hugo-theme-relearn Bumps [docs/themes/hugo-theme-relearn](https://github.com/McShelby/hugo-theme-relearn) from `e1a1f01` to `007cc20`. - [Release notes](https://github.com/McShelby/hugo-theme-relearn/releases) - [Commits](https://github.com/McShelby/hugo-theme-relearn/compare/e1a1f01f4c34b1980e07f81dd0c690e0969c00ca...007cc20686f04ca1f911975f20f097175dd72a7f) --- updated-dependencies: - dependency-name: docs/themes/hugo-theme-relearn dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- docs/themes/hugo-theme-relearn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/themes/hugo-theme-relearn b/docs/themes/hugo-theme-relearn index e1a1f01f4c34..007cc20686f0 160000 --- a/docs/themes/hugo-theme-relearn +++ b/docs/themes/hugo-theme-relearn @@ -1 +1 @@ -Subproject commit e1a1f01f4c34b1980e07f81dd0c690e0969c00ca +Subproject commit 007cc20686f04ca1f911975f20f097175dd72a7f From 094f808549ff812b2273636d3404edb4715eed49 Mon Sep 17 00:00:00 2001 From: "LocalAI [bot]" <139863280+localai-bot@users.noreply.github.com> Date: Tue, 15 Oct 2024 09:41:11 +0200 Subject: [PATCH 07/10] chore: :arrow_up: Update ggerganov/whisper.cpp to `06a1da9daff94c1bf1b1d38950628264fe443f76` (#3836) :arrow_up: Update ggerganov/whisper.cpp Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 47c5de4f9399..d917e689d285 100644 --- a/Makefile +++ b/Makefile @@ -16,7 +16,7 @@ RWKV_VERSION?=661e7ae26d442f5cfebd2a0881b44e8c55949ec6 # whisper.cpp version WHISPER_REPO?=https://github.com/ggerganov/whisper.cpp -WHISPER_CPP_VERSION?=fdbfb460ed546452a5d53611bba66d10d842e719 +WHISPER_CPP_VERSION?=06a1da9daff94c1bf1b1d38950628264fe443f76 # bert.cpp version BERT_REPO?=https://github.com/go-skynet/go-bert.cpp From 97cf028175037eb35f9519e2a26851572ec0eba7 Mon Sep 17 00:00:00 2001 From: Franco Lombardo Date: Tue, 15 Oct 2024 09:41:39 +0200 Subject: [PATCH 08/10] chore: update integrations.md with LLPhant (#3838) Signed-off-by: Franco Lombardo --- docs/content/docs/integrations.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/content/docs/integrations.md b/docs/content/docs/integrations.md index 50f683c3fa63..0939ecdbc632 100644 --- a/docs/content/docs/integrations.md +++ b/docs/content/docs/integrations.md @@ -28,5 +28,6 @@ The list below is a list of software that integrates with LocalAI. - https://github.com/cedriking/spark - [Big AGI](https://github.com/enricoros/big-agi) is a powerful web interface entirely running in the browser, supporting LocalAI - [Midori AI Subsystem Manager](https://io.midori-ai.xyz/subsystem/manager/) is a powerful docker subsystem for running all types of AI programs +- [LLPhant](https://github.com/theodo-group/LLPhant) is a PHP library for interacting with LLMs and Vector Databases Feel free to open up a Pull request (by clicking at the "Edit page" below) to get a page for your project made or if you see a error on one of the pages! From b82577d6423b2c18a8d14b93d05b4f484a80a197 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Tue, 15 Oct 2024 09:41:53 +0200 Subject: [PATCH 09/10] fix(llama.cpp): consider also native builds (#3839) This is in order to identify also builds which are not using alternatives based on capabilities. For instance, there are cases when we build the backend only natively in the host. Signed-off-by: Ettore Di Giacinto --- pkg/model/initializers.go | 49 ++++++++++++++++++++++++++------------- 1 file changed, 33 insertions(+), 16 deletions(-) diff --git a/pkg/model/initializers.go b/pkg/model/initializers.go index c3b3717979e7..bd668ec25a2a 100644 --- a/pkg/model/initializers.go +++ b/pkg/model/initializers.go @@ -251,8 +251,22 @@ func selectGRPCProcessByHostCapabilities(backend, assetDir string, f16 bool) str // No GPU found or no specific binaries found, try to load the CPU variant(s) - // Select the Fallback by default - selectedProcess := backendPath(assetDir, LLamaCPPFallback) + // Select a binary based on availability/capability + selectedProcess := "" + + // Check if we have a native build (llama-cpp) and use that + if _, err := os.Stat(backendPath(assetDir, LLamaCPPFallback)); err == nil { + log.Debug().Msgf("[%s] %s variant available", LLamaCPPFallback, backend) + selectedProcess = backendPath(assetDir, LLamaCPPFallback) + } + + // Check if we have a native build (llama-cpp) and use that instead + // As a reminder, we do ultimately attempt again with the fallback variant + // If things fail with what we select here + if _, err := os.Stat(backendPath(assetDir, LLamaCPP)); err == nil { + log.Debug().Msgf("[%s] attempting to load with native variant", backend) + selectedProcess = backendPath(assetDir, LLamaCPP) + } // IF we find any optimized binary, we use that if xsysinfo.HasCPUCaps(cpuid.AVX2) { @@ -269,7 +283,7 @@ func selectGRPCProcessByHostCapabilities(backend, assetDir string, f16 bool) str } } - // Check if the binary exists! + // Safety measure: check if the binary exists otherwise return empty string if _, err := os.Stat(selectedProcess); err == nil { return selectedProcess } @@ -277,6 +291,21 @@ func selectGRPCProcessByHostCapabilities(backend, assetDir string, f16 bool) str return "" } +func attemptLoadingOnFailure(backend string, ml *ModelLoader, o *Options, err error) (*Model, error) { + // XXX: This is too backend specific(llama-cpp), remove this bit or generalize further + // We failed somehow starting the binary. For instance, could be that we are missing + // some libraries if running in binary-only mode. + // In this case, we attempt to load the model with the fallback variant. + + // If not llama-cpp backend, return the error immediately + if backend != LLamaCPP { + return nil, err + } + + log.Error().Msgf("[%s] Failed loading model, trying with fallback '%s', error: %s", backend, LLamaCPPFallback, err.Error()) + return ml.LoadModel(o.modelID, o.model, ml.grpcModel(LLamaCPPFallback, false, o)) +} + // starts the grpcModelProcess for the backend, and returns a grpc client // It also loads the model func (ml *ModelLoader) grpcModel(backend string, autodetect bool, o *Options) func(string, string, string) (*Model, error) { @@ -450,19 +479,7 @@ func (ml *ModelLoader) BackendLoader(opts ...Option) (client grpc.Backend, err e model, err := ml.LoadModel(o.modelID, o.model, ml.grpcModel(backendToConsume, AutoDetect, o)) if err != nil { - // XXX: This is too backend specific(llama-cpp), remove this bit or generalize further - // We failed somehow starting the binary. For instance, could be that we are missing - // some libraries if running in binary-only mode. - // In this case, we attempt to load the model with the fallback variant. - - // If not llama-cpp backend, return error immediately - if backend != LLamaCPP { - return nil, err - } - - // Otherwise attempt with fallback - log.Error().Msgf("[%s] Failed loading model, trying with fallback '%s'", backend, LLamaCPPFallback) - model, err = ml.LoadModel(o.modelID, o.model, ml.grpcModel(LLamaCPPFallback, false, o)) + model, err = attemptLoadingOnFailure(backend, ml, o, err) if err != nil { return nil, err } From 5f130febb84ed812b145da68eca0bee2de32b875 Mon Sep 17 00:00:00 2001 From: "LocalAI [bot]" <139863280+localai-bot@users.noreply.github.com> Date: Tue, 15 Oct 2024 23:41:29 +0200 Subject: [PATCH 10/10] chore: :arrow_up: Update ggerganov/whisper.cpp to `b6049060dd2341b7816d2bce7dc7451c1665828e` (#3842) :arrow_up: Update ggerganov/whisper.cpp Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index d917e689d285..154ae5587e6f 100644 --- a/Makefile +++ b/Makefile @@ -16,7 +16,7 @@ RWKV_VERSION?=661e7ae26d442f5cfebd2a0881b44e8c55949ec6 # whisper.cpp version WHISPER_REPO?=https://github.com/ggerganov/whisper.cpp -WHISPER_CPP_VERSION?=06a1da9daff94c1bf1b1d38950628264fe443f76 +WHISPER_CPP_VERSION?=b6049060dd2341b7816d2bce7dc7451c1665828e # bert.cpp version BERT_REPO?=https://github.com/go-skynet/go-bert.cpp