feat: add reindexing process to indexer

Signed-off-by: tylerslaton <[email protected]>

docker-compose.yml

@@ -38,4 +38,5 @@ services:
       - db
     environment:
       - DB_URL=postgres://postgres:postgres@db:5432/tools?sslmode=disable
+      - API_URL=http://app:3000/api
     restart: on-failure

indexer/main.go

@@ -1,12 +1,14 @@
 package main
 
 import (
+	"bytes"
 	"database/sql"
 	"encoding/json"
-	"fmt"
-	"log"
+	"net/http"
 	"os"
 
+	log "github.com/sirupsen/logrus"
+
 	"github.com/gptscript-ai/gptscript/pkg/builtin"
 	"github.com/gptscript-ai/gptscript/pkg/types"
 	_ "github.com/lib/pq"
@@ -18,17 +20,30 @@ func main() {
 		log.Fatal("DB_URL environment variable is not set")
 	}
 
+	apiURL := os.Getenv("API_URL")
+	if apiURL == "" {
+		log.Fatal("API_URL environment variable is not set")
+	}
+
 	db, err := sql.Open("postgres", dbConnectionString)
 	if err != nil {
 		log.Fatal(err)
 	}
 	defer db.Close()
 
+	insertSystemTools(db)
+	reindexRemoteTools(db, apiURL)
+}
+
+// insertSystemTools inserts gets all of the current system tools and loads them
+// into the database.
+func insertSystemTools(db *sql.DB) {
 	for _, tool := range builtin.ListTools() {
 		toolInArray := []types.Tool{tool}
 		toolAsJSON, err := json.Marshal(toolInArray)
 		if err != nil {
-			log.Fatal(err)
+			log.Error(err)
+			continue
 		}
 
 		_, err = db.Exec(`
@@ -42,9 +57,70 @@ func main() {
 		)
 
 		if err != nil {
-			log.Fatal(err)
+			log.Error(err)
 		}
 	}
+	log.Infof("Successfully inserted system tools into the database.")
+}
+
+// reindexRemoteTools reindexes all of the current remote tools in the database. If
+// an error occurs, it logs the error and continues to the next tool.
+func reindexRemoteTools(db *sql.DB, apiURL string) {
+	rows, err := db.Query(`
+		SELECT *
+		FROM public."ToolEntry"
+		WHERE "systemTool" = false
+	`)
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer rows.Close()
+
+	for rows.Next() {
+		var (
+			index         string
+			reference     string
+			content       string
+			lastIndexedAt string
+			createdAt     string
+			systemTool    bool
+		)
+
+		err = rows.Scan(&index, &createdAt, &lastIndexedAt, &content, &reference, &systemTool)
+		if err != nil {
+			log.Fatal(err)
+		}
 
-	fmt.Println("Successfully inserted system tools into the database.")
+		if systemTool {
+			continue
+		}
+
+		url := apiURL + "/" + reference
+		req, err := http.NewRequest("POST", url, bytes.NewBuffer([]byte{}))
+		if err != nil {
+			log.Fatal(err)
+		}
+		client := &http.Client{}
+		resp, err := client.Do(req)
+		if err != nil {
+			log.Fatal(err)
+		}
+		defer resp.Body.Close()
+
+		if resp.StatusCode != http.StatusOK {
+			log.Errorf("Error indexing tool %s: %s", reference, resp.Status)
+			continue
+		}
+
+		_, err = db.Exec(`
+			UPDATE public."ToolEntry"
+			SET "lastIndexedAt" = NOW()
+			WHERE reference = $1`,
+			reference,
+		)
+		if err != nil {
+			log.Fatal(err)
+		}
+	}
+	log.Infof("Successfully completed the remote tool reindexing.")
 }