Add support for configuring and using Ollama models with Foyle: (#43)

- Add 'AgentConfig' struct to 'config' package with a 'Model' field for
specifying the model to use for completions
- Set default configuration for the 'AgentConfig' model to
'openai.GPT3Dot5Turbo0125'
- Update 'agent' package to handle missing AgentConfig in configuration
and log an error
- Update 'oai' package to handle missing OpenAI configuration and custom
BaseURL, and log BaseURL usage
- Add 'DefaultModel' constant for Ollama model name to 'const.go' in
'config' package
- Add a manual test for Ollama usage with 'Test_Ollama' in
'client_test.go' with a check for 'GITHUB_ACTIONS' environment variable
- Add documentation for using Ollama with Foyle in
'docs/content/en/docs/ollama/_index.md'
- Include steps for configuring Foyle to use Ollama's baseURL and model
in the documentation

app/pkg/agent/agent.go

@@ -4,6 +4,9 @@ import (
 	"context"
 	"strings"
 
+	"github.com/go-logr/zapr"
+	"go.uber.org/zap"
+
 	"github.com/jlewi/foyle/app/pkg/config"
 	"github.com/jlewi/foyle/app/pkg/docs"
 	"github.com/jlewi/foyle/app/pkg/logs"
@@ -34,11 +37,18 @@ type Agent struct {
 }
 
 func NewAgent(cfg config.Config, client *openai.Client) (*Agent, error) {
+	if cfg.Agent == nil {
+		return nil, errors.New("Configuration is missing AgentConfig; configuration must define the agent field.")
+	}
+
+	log := zapr.NewLogger(zap.L())
+	log.Info("Creating agent", "config", cfg.Agent)
 	return &Agent{
 		client: client,
 		config: cfg,
 	}, nil
 }
+
 func (a *Agent) Generate(ctx context.Context, req *v1alpha1.GenerateRequest) (*v1alpha1.GenerateResponse, error) {
 	blocks, err := a.completeWithRetries(ctx, req)
 
@@ -76,7 +86,7 @@ func (a *Agent) completeWithRetries(ctx context.Context, req *v1alpha1.GenerateR
 			},
 		}
 		request := openai.ChatCompletionRequest{
-			Model:       oai.DefaultModel,
+			Model:       a.config.GetModel(),
 			Messages:    messages,
 			MaxTokens:   2000,
 			Temperature: temperature,

app/pkg/config/config.go

@@ -38,14 +38,20 @@ type Config struct {
 	APIVersion string `json:"apiVersion" yaml:"apiVersion" yamltags:"required"`
 	Kind       string `json:"kind" yaml:"kind" yamltags:"required"`
 
-	Logging Logging      `json:"logging" yaml:"logging"`
-	Server  ServerConfig `json:"server" yaml:"server"`
-	Assets  AssetConfig  `json:"assets" yaml:"assets"`
-	OpenAI  OpenAIConfig `json:"openai" yaml:"openai"`
+	Logging Logging       `json:"logging" yaml:"logging"`
+	Server  ServerConfig  `json:"server" yaml:"server"`
+	Assets  AssetConfig   `json:"assets" yaml:"assets"`
+	Agent   *AgentConfig  `json:"agent,omitempty" yaml:"agent,omitempty"`
+	OpenAI  *OpenAIConfig `json:"openai,omitempty" yaml:"openai,omitempty"`
 	// AzureOpenAI contains configuration for Azure OpenAI. A non nil value means use Azure OpenAI.
 	AzureOpenAI *AzureOpenAIConfig `json:"azureOpenAI,omitempty" yaml:"azureOpenAI,omitempty"`
 }
 
+type AgentConfig struct {
+	// Model is the name of the model to use to generate completions
+	Model string `json:"model" yaml:"model"`
+}
+
 // ServerConfig configures the server
 type ServerConfig struct {
 	// BindAddress is the address to bind to. Default is 0.0.0.0
@@ -71,6 +77,9 @@ type ServerConfig struct {
 type OpenAIConfig struct {
 	// APIKeyFile is the path to the file containing the API key
 	APIKeyFile string `json:"apiKeyFile" yaml:"apiKeyFile"`
+
+	// BaseURL is the baseURL for the API.
+	BaseURL string `json:"baseURL" yaml:"baseURL"`
 }
 
 type AzureOpenAIConfig struct {
@@ -128,6 +137,13 @@ type Logging struct {
 	Level string `json:"level" yaml:"level"`
 }
 
+func (c *Config) GetModel() string {
+	if c.Agent == nil || c.Agent.Model == "" {
+		return DefaultModel
+	}
+
+	return c.Agent.Model
+}
 func (c *Config) GetLogLevel() string {
 	if c.Logging.Level == "" {
 		return "info"
@@ -163,10 +179,11 @@ func InitViper(cmd *cobra.Command) error {
 	// make home directory the first search path
 	viper.AddConfigPath("$HOME/." + appName)
 
-	// Without the replacer overriding with environment variables work
+	// Without the replacer overriding with environment variables doesn't work
 	viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_"))
 	viper.AutomaticEnv() // read in environment variables that match
 
+	setAgentDefaults()
 	setServerDefaults()
 	setAssetDefaults()
 
@@ -274,6 +291,10 @@ func setAssetDefaults() {
 	viper.SetDefault("assets.foyleExtension.uri", defaultFoyleImage)
 }
 
+func setAgentDefaults() {
+	viper.SetDefault("agent.model", DefaultModel)
+}
+
 func DefaultConfigFile() string {
 	return binHome() + "/config.yaml"
 }

app/pkg/config/const.go

@@ -0,0 +1,5 @@
+package config
+
+import "github.com/sashabaranov/go-openai"
+
+const DefaultModel = openai.GPT3Dot5Turbo0125

app/pkg/oai/client.go

@@ -15,7 +15,6 @@ import (
 )
 
 const (
-	DefaultModel = openai.GPT3Dot5Turbo0125
 
 	// AzureOpenAIVersion is the version of the Azure OpenAI API to use.
 	// For a list of versions see:
@@ -46,11 +45,27 @@ func NewClient(cfg config.Config) (*openai.Client, error) {
 		}
 	} else {
 		log.Info("Configuring OpenAI client")
-		apiKey, err := readAPIKey(cfg.OpenAI.APIKeyFile)
-		if err != nil {
-			return nil, err
+		if cfg.OpenAI == nil {
+			return nil, errors.New("OpenAI configuration is required")
+		}
+
+		apiKey := ""
+		if cfg.OpenAI.APIKeyFile != "" {
+			var err error
+			apiKey, err = readAPIKey(cfg.OpenAI.APIKeyFile)
+			if err != nil {
+				return nil, err
+			}
+		}
+		// If baseURL is customized then we could be using a custom endpoint that may not require an API key
+		if apiKey == "" && cfg.OpenAI.BaseURL == "" {
+			return nil, errors.New("OpenAI APIKeyFile is required when using OpenAI")
 		}
 		clientConfig = openai.DefaultConfig(apiKey)
+		if cfg.OpenAI.BaseURL != "" {
+			log.Info("Using custom OpenAI BaseURL", "baseURL", cfg.OpenAI.BaseURL)
+			clientConfig.BaseURL = cfg.OpenAI.BaseURL
+		}
 	}
 	clientConfig.HTTPClient = httpClient
 	client := openai.NewClientWithConfig(clientConfig)
@@ -75,7 +90,7 @@ func buildAzureConfig(cfg config.Config) (openai.ClientConfig, error) {
 
 	// Check that all required models are deployed
 	required := map[string]bool{
-		DefaultModel: true,
+		config.DefaultModel: true,
 	}
 
 	for _, d := range cfg.AzureOpenAI.Deployments {

app/pkg/oai/client_test.go

@@ -1,10 +1,12 @@
 package oai
 
 import (
+	"context"
 	"os"
 	"testing"
 
 	"github.com/jlewi/foyle/app/pkg/config"
+	"github.com/sashabaranov/go-openai"
 )
 
 func Test_BuildAzureAIConfig(t *testing.T) {
@@ -22,7 +24,7 @@ func Test_BuildAzureAIConfig(t *testing.T) {
 			BaseURL:    "https://someurl.com",
 			Deployments: []config.AzureDeployment{
 				{
-					Model:      DefaultModel,
+					Model:      config.DefaultModel,
 					Deployment: "somedeployment",
 				},
 			},
@@ -43,3 +45,35 @@ func Test_BuildAzureAIConfig(t *testing.T) {
 		t.Fatalf("Expected BaseURL to be https://someurl.com but got %v", clientConfig.BaseURL)
 	}
 }
+
+func Test_Ollama(t *testing.T) {
+	if os.Getenv("GITHUB_ACTIONS") != "" {
+		t.Skipf("Test_Ollama is a manual test that is skipped in CICD")
+	}
+	clientCfg := openai.DefaultConfig("")
+	clientCfg.BaseURL = "http://localhost:11434/v1"
+	client := openai.NewClientWithConfig(clientCfg)
+
+	messages := []openai.ChatCompletionMessage{
+		{Role: openai.ChatMessageRoleSystem,
+			Content: "You are a helpful assistant.",
+		},
+		{Role: openai.ChatMessageRoleUser,
+			Content: "hello",
+		},
+	}
+
+	request := openai.ChatCompletionRequest{
+		Model:       "llama2",
+		Messages:    messages,
+		MaxTokens:   2000,
+		Temperature: 0.9,
+	}
+
+	resp, err := client.CreateChatCompletion(context.Background(), request)
+	if err != nil {
+		t.Fatalf("Failed to create chat completion: %v", err)
+	}
+
+	t.Logf("Response: %+v", resp)
+}

docs/content/en/docs/ollama/_index.md

@@ -0,0 +1,39 @@
+---
+title: "Ollama"
+description: "How to use Ollama with Foyle"
+weight: 3
+---
+
+## What You'll Learn
+
+How to configure Foyle to use models served by Ollama
+
+## Prerequisites
+
+1. Follow [Ollama's docs] to download Ollama and serve a model like `llama2`  
+
+## Setup Foyle to use Ollama
+
+Foyle relies on [Ollama's OpenAI Chat Compatability API]() to interact with models served by Ollama.
+
+
+1. Configure Foyle to use the appropriate Ollama baseURL
+
+   ```
+   foyle config set openai.baseURL=http://localhost:11434/v1
+   ```
+
+   * Change the server and port to match how you are serving Ollama
+   * You may also need to change the scheme to https; e.g. if you are using a VPN like [Tailscale](https://tailscale.com/)
+
+1. Configure Foyle to use the appropriate Ollama model
+
+   ```
+   foyle config agent.model=llama2 
+   ```
+
+    * Change the model to match the model you are serving with Ollama
+
+1. You can leave the `apiKeyFile` unset since you aren't using an API key with Ollama
+
+1. That's it! You should now be able to use Foyle with Ollama