diff --git a/docker-compose.yml b/docker-compose.yml index 244650d..294351c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -38,4 +38,5 @@ services: - db environment: - DB_URL=postgres://postgres:postgres@db:5432/tools?sslmode=disable + - API_URL=http://app:3000/api restart: on-failure diff --git a/indexer/main.go b/indexer/main.go index fb215ef..ade2215 100644 --- a/indexer/main.go +++ b/indexer/main.go @@ -1,12 +1,14 @@ package main import ( + "bytes" "database/sql" "encoding/json" - "fmt" - "log" + "net/http" "os" + log "github.com/sirupsen/logrus" + "github.com/gptscript-ai/gptscript/pkg/builtin" "github.com/gptscript-ai/gptscript/pkg/types" _ "github.com/lib/pq" @@ -18,17 +20,30 @@ func main() { log.Fatal("DB_URL environment variable is not set") } + apiURL := os.Getenv("API_URL") + if apiURL == "" { + log.Fatal("API_URL environment variable is not set") + } + db, err := sql.Open("postgres", dbConnectionString) if err != nil { log.Fatal(err) } defer db.Close() + insertSystemTools(db) + reindexRemoteTools(db, apiURL) +} + +// insertSystemTools inserts gets all of the current system tools and loads them +// into the database. +func insertSystemTools(db *sql.DB) { for _, tool := range builtin.ListTools() { toolInArray := []types.Tool{tool} toolAsJSON, err := json.Marshal(toolInArray) if err != nil { - log.Fatal(err) + log.Error(err) + continue } _, err = db.Exec(` @@ -42,9 +57,70 @@ func main() { ) if err != nil { - log.Fatal(err) + log.Error(err) } } + log.Infof("Successfully inserted system tools into the database.") +} + +// reindexRemoteTools reindexes all of the current remote tools in the database. If +// an error occurs, it logs the error and continues to the next tool. +func reindexRemoteTools(db *sql.DB, apiURL string) { + rows, err := db.Query(` + SELECT * + FROM public."ToolEntry" + WHERE "systemTool" = false + `) + if err != nil { + log.Fatal(err) + } + defer rows.Close() + + for rows.Next() { + var ( + index string + reference string + content string + lastIndexedAt string + createdAt string + systemTool bool + ) + + err = rows.Scan(&index, &createdAt, &lastIndexedAt, &content, &reference, &systemTool) + if err != nil { + log.Fatal(err) + } - fmt.Println("Successfully inserted system tools into the database.") + if systemTool { + continue + } + + url := apiURL + "/" + reference + req, err := http.NewRequest("POST", url, bytes.NewBuffer([]byte{})) + if err != nil { + log.Fatal(err) + } + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + log.Fatal(err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + log.Errorf("Error indexing tool %s: %s", reference, resp.Status) + continue + } + + _, err = db.Exec(` + UPDATE public."ToolEntry" + SET "lastIndexedAt" = NOW() + WHERE reference = $1`, + reference, + ) + if err != nil { + log.Fatal(err) + } + } + log.Infof("Successfully completed the remote tool reindexing.") }