diff --git a/app/pkg/agent/agent.go b/app/pkg/agent/agent.go index 1f923577..f611a608 100644 --- a/app/pkg/agent/agent.go +++ b/app/pkg/agent/agent.go @@ -4,6 +4,9 @@ import ( "context" "strings" + "github.com/go-logr/zapr" + "go.uber.org/zap" + "github.com/jlewi/foyle/app/pkg/config" "github.com/jlewi/foyle/app/pkg/docs" "github.com/jlewi/foyle/app/pkg/logs" @@ -34,11 +37,18 @@ type Agent struct { } func NewAgent(cfg config.Config, client *openai.Client) (*Agent, error) { + if cfg.Agent == nil { + return nil, errors.New("Configuration is missing AgentConfig; configuration must define the agent field.") + } + + log := zapr.NewLogger(zap.L()) + log.Info("Creating agent", "config", cfg.Agent) return &Agent{ client: client, config: cfg, }, nil } + func (a *Agent) Generate(ctx context.Context, req *v1alpha1.GenerateRequest) (*v1alpha1.GenerateResponse, error) { blocks, err := a.completeWithRetries(ctx, req) @@ -76,7 +86,7 @@ func (a *Agent) completeWithRetries(ctx context.Context, req *v1alpha1.GenerateR }, } request := openai.ChatCompletionRequest{ - Model: oai.DefaultModel, + Model: a.config.GetModel(), Messages: messages, MaxTokens: 2000, Temperature: temperature, diff --git a/app/pkg/config/config.go b/app/pkg/config/config.go index 6a3115ed..22952be9 100644 --- a/app/pkg/config/config.go +++ b/app/pkg/config/config.go @@ -38,14 +38,20 @@ type Config struct { APIVersion string `json:"apiVersion" yaml:"apiVersion" yamltags:"required"` Kind string `json:"kind" yaml:"kind" yamltags:"required"` - Logging Logging `json:"logging" yaml:"logging"` - Server ServerConfig `json:"server" yaml:"server"` - Assets AssetConfig `json:"assets" yaml:"assets"` - OpenAI OpenAIConfig `json:"openai" yaml:"openai"` + Logging Logging `json:"logging" yaml:"logging"` + Server ServerConfig `json:"server" yaml:"server"` + Assets AssetConfig `json:"assets" yaml:"assets"` + Agent *AgentConfig `json:"agent,omitempty" yaml:"agent,omitempty"` + OpenAI *OpenAIConfig `json:"openai,omitempty" yaml:"openai,omitempty"` // AzureOpenAI contains configuration for Azure OpenAI. A non nil value means use Azure OpenAI. AzureOpenAI *AzureOpenAIConfig `json:"azureOpenAI,omitempty" yaml:"azureOpenAI,omitempty"` } +type AgentConfig struct { + // Model is the name of the model to use to generate completions + Model string `json:"model" yaml:"model"` +} + // ServerConfig configures the server type ServerConfig struct { // BindAddress is the address to bind to. Default is 0.0.0.0 @@ -71,6 +77,9 @@ type ServerConfig struct { type OpenAIConfig struct { // APIKeyFile is the path to the file containing the API key APIKeyFile string `json:"apiKeyFile" yaml:"apiKeyFile"` + + // BaseURL is the baseURL for the API. + BaseURL string `json:"baseURL" yaml:"baseURL"` } type AzureOpenAIConfig struct { @@ -128,6 +137,13 @@ type Logging struct { Level string `json:"level" yaml:"level"` } +func (c *Config) GetModel() string { + if c.Agent == nil || c.Agent.Model == "" { + return DefaultModel + } + + return c.Agent.Model +} func (c *Config) GetLogLevel() string { if c.Logging.Level == "" { return "info" @@ -163,10 +179,11 @@ func InitViper(cmd *cobra.Command) error { // make home directory the first search path viper.AddConfigPath("$HOME/." + appName) - // Without the replacer overriding with environment variables work + // Without the replacer overriding with environment variables doesn't work viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_")) viper.AutomaticEnv() // read in environment variables that match + setAgentDefaults() setServerDefaults() setAssetDefaults() @@ -274,6 +291,10 @@ func setAssetDefaults() { viper.SetDefault("assets.foyleExtension.uri", defaultFoyleImage) } +func setAgentDefaults() { + viper.SetDefault("agent.model", DefaultModel) +} + func DefaultConfigFile() string { return binHome() + "/config.yaml" } diff --git a/app/pkg/config/const.go b/app/pkg/config/const.go new file mode 100644 index 00000000..736ccd44 --- /dev/null +++ b/app/pkg/config/const.go @@ -0,0 +1,5 @@ +package config + +import "github.com/sashabaranov/go-openai" + +const DefaultModel = openai.GPT3Dot5Turbo0125 diff --git a/app/pkg/oai/client.go b/app/pkg/oai/client.go index 81adbd12..73d75fc3 100644 --- a/app/pkg/oai/client.go +++ b/app/pkg/oai/client.go @@ -15,7 +15,6 @@ import ( ) const ( - DefaultModel = openai.GPT3Dot5Turbo0125 // AzureOpenAIVersion is the version of the Azure OpenAI API to use. // For a list of versions see: @@ -46,11 +45,27 @@ func NewClient(cfg config.Config) (*openai.Client, error) { } } else { log.Info("Configuring OpenAI client") - apiKey, err := readAPIKey(cfg.OpenAI.APIKeyFile) - if err != nil { - return nil, err + if cfg.OpenAI == nil { + return nil, errors.New("OpenAI configuration is required") + } + + apiKey := "" + if cfg.OpenAI.APIKeyFile != "" { + var err error + apiKey, err = readAPIKey(cfg.OpenAI.APIKeyFile) + if err != nil { + return nil, err + } + } + // If baseURL is customized then we could be using a custom endpoint that may not require an API key + if apiKey == "" && cfg.OpenAI.BaseURL == "" { + return nil, errors.New("OpenAI APIKeyFile is required when using OpenAI") } clientConfig = openai.DefaultConfig(apiKey) + if cfg.OpenAI.BaseURL != "" { + log.Info("Using custom OpenAI BaseURL", "baseURL", cfg.OpenAI.BaseURL) + clientConfig.BaseURL = cfg.OpenAI.BaseURL + } } clientConfig.HTTPClient = httpClient client := openai.NewClientWithConfig(clientConfig) @@ -75,7 +90,7 @@ func buildAzureConfig(cfg config.Config) (openai.ClientConfig, error) { // Check that all required models are deployed required := map[string]bool{ - DefaultModel: true, + config.DefaultModel: true, } for _, d := range cfg.AzureOpenAI.Deployments { diff --git a/app/pkg/oai/client_test.go b/app/pkg/oai/client_test.go index 0834f7d5..61282e11 100644 --- a/app/pkg/oai/client_test.go +++ b/app/pkg/oai/client_test.go @@ -1,10 +1,12 @@ package oai import ( + "context" "os" "testing" "github.com/jlewi/foyle/app/pkg/config" + "github.com/sashabaranov/go-openai" ) func Test_BuildAzureAIConfig(t *testing.T) { @@ -22,7 +24,7 @@ func Test_BuildAzureAIConfig(t *testing.T) { BaseURL: "https://someurl.com", Deployments: []config.AzureDeployment{ { - Model: DefaultModel, + Model: config.DefaultModel, Deployment: "somedeployment", }, }, @@ -43,3 +45,35 @@ func Test_BuildAzureAIConfig(t *testing.T) { t.Fatalf("Expected BaseURL to be https://someurl.com but got %v", clientConfig.BaseURL) } } + +func Test_Ollama(t *testing.T) { + if os.Getenv("GITHUB_ACTIONS") != "" { + t.Skipf("Test_Ollama is a manual test that is skipped in CICD") + } + clientCfg := openai.DefaultConfig("") + clientCfg.BaseURL = "http://localhost:11434/v1" + client := openai.NewClientWithConfig(clientCfg) + + messages := []openai.ChatCompletionMessage{ + {Role: openai.ChatMessageRoleSystem, + Content: "You are a helpful assistant.", + }, + {Role: openai.ChatMessageRoleUser, + Content: "hello", + }, + } + + request := openai.ChatCompletionRequest{ + Model: "llama2", + Messages: messages, + MaxTokens: 2000, + Temperature: 0.9, + } + + resp, err := client.CreateChatCompletion(context.Background(), request) + if err != nil { + t.Fatalf("Failed to create chat completion: %v", err) + } + + t.Logf("Response: %+v", resp) +} diff --git a/docs/content/en/docs/ollama/_index.md b/docs/content/en/docs/ollama/_index.md new file mode 100644 index 00000000..5870b654 --- /dev/null +++ b/docs/content/en/docs/ollama/_index.md @@ -0,0 +1,39 @@ +--- +title: "Ollama" +description: "How to use Ollama with Foyle" +weight: 3 +--- + +## What You'll Learn + +How to configure Foyle to use models served by Ollama + +## Prerequisites + +1. Follow [Ollama's docs] to download Ollama and serve a model like `llama2` + +## Setup Foyle to use Ollama + +Foyle relies on [Ollama's OpenAI Chat Compatability API]() to interact with models served by Ollama. + + +1. Configure Foyle to use the appropriate Ollama baseURL + + ``` + foyle config set openai.baseURL=http://localhost:11434/v1 + ``` + + * Change the server and port to match how you are serving Ollama + * You may also need to change the scheme to https; e.g. if you are using a VPN like [Tailscale](https://tailscale.com/) + +1. Configure Foyle to use the appropriate Ollama model + + ``` + foyle config agent.model=llama2 + ``` + + * Change the model to match the model you are serving with Ollama + +1. You can leave the `apiKeyFile` unset since you aren't using an API key with Ollama + +1. That's it! You should now be able to use Foyle with Ollama