From 0047890d4353bb10ee82b2c37a3be30bc1093a61 Mon Sep 17 00:00:00 2001 From: Hao Wu Date: Thu, 4 Jul 2024 10:29:45 +0800 Subject: [PATCH 1/4] Add logging and update headers for Claude3 chat stream --- api/chat_main_handler.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/api/chat_main_handler.go b/api/chat_main_handler.go index 366c166b..28589850 100644 --- a/api/chat_main_handler.go +++ b/api/chat_main_handler.go @@ -778,10 +778,12 @@ func (h *ChatHandler) chatStreamClaude3(w http.ResponseWriter, chatSession sqlc_ // convert data to json format jsonValue, _ := json.Marshal(jsonData) + log.Printf("%+v", string(jsonValue)) // create the request req, err := http.NewRequest("POST", chatModel.Url, bytes.NewBuffer(jsonValue)) if err != nil { + log.Printf("%+v", err) RespondWithError(w, http.StatusInternalServerError, "error.fail_to_make_request", err) return "", "", true } @@ -799,6 +801,7 @@ func (h *ChatHandler) chatStreamClaude3(w http.ResponseWriter, chatSession sqlc_ req.Header.Set("Accept", "text/event-stream") req.Header.Set("Cache-Control", "no-cache") req.Header.Set("Connection", "keep-alive") + req.Header.Set("anthropic-version", "2023-06-01") // create the http client and send the request client := &http.Client{ @@ -806,6 +809,7 @@ func (h *ChatHandler) chatStreamClaude3(w http.ResponseWriter, chatSession sqlc_ } resp, err := client.Do(req) if err != nil { + log.Printf("%+v", err) RespondWithError(w, http.StatusInternalServerError, "error.fail_to_do_request", err) return "", "", true } @@ -840,9 +844,12 @@ func (h *ChatHandler) chatStreamClaude3(w http.ResponseWriter, chatSession sqlc_ break } line, err := ioreader.ReadBytes('\n') - + log.Printf("%+v", string(line)) if err != nil { if errors.Is(err, io.EOF) { + if bytes.HasPrefix(line, []byte("{\"type\":\"error\"")) { + log.Println(string(line)) + } fmt.Println("End of stream reached") break // Exit loop if end of stream } @@ -874,7 +881,7 @@ func (h *ChatHandler) chatStreamClaude3(w http.ResponseWriter, chatSession sqlc_ } } if bytes.HasPrefix(line, []byte("{\"type\":\"content_block_delta\"")) { - answer = claude.AnswerFromBlockDelta(line) + answer += claude.AnswerFromBlockDelta(line) if len(answer) < 200 || len(answer)%2 == 0 { data, _ := json.Marshal(constructChatCompletionStreamReponse(answer_id, answer)) fmt.Fprintf(w, "data: %v\n\n", string(data)) From a729308ead559c97f9fc33c4f88d1d975e41e48b Mon Sep 17 00:00:00 2001 From: Hao Wu Date: Thu, 4 Jul 2024 10:32:54 +0800 Subject: [PATCH 2/4] Adjust token limit check to use full context window --- api/chat_main_handler.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/api/chat_main_handler.go b/api/chat_main_handler.go index 28589850..f8cac15f 100644 --- a/api/chat_main_handler.go +++ b/api/chat_main_handler.go @@ -196,7 +196,8 @@ func genAnswer(h *ChatHandler, w http.ResponseWriter, chatSessionUuid string, ch }) // check if total tokens exceed limit - if totalTokens > chatSession.MaxTokens*2/3 { + // context window, max token + if totalTokens > chatSession.MaxTokens { RespondWithError(w, http.StatusRequestEntityTooLarge, "error.token_length_exceed_limit", map[string]interface{}{ "max_tokens": chatSession.MaxTokens, From fdea1b9f9a428b188d60a14a0c202307818969eb Mon Sep 17 00:00:00 2001 From: Hao Wu Date: Thu, 4 Jul 2024 10:38:32 +0800 Subject: [PATCH 3/4] Disable logging of chat stream lines in Claude3 handler. --- api/chat_main_handler.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/chat_main_handler.go b/api/chat_main_handler.go index f8cac15f..4bff571c 100644 --- a/api/chat_main_handler.go +++ b/api/chat_main_handler.go @@ -845,7 +845,7 @@ func (h *ChatHandler) chatStreamClaude3(w http.ResponseWriter, chatSession sqlc_ break } line, err := ioreader.ReadBytes('\n') - log.Printf("%+v", string(line)) + // log.Printf("%+v", string(line)) if err != nil { if errors.Is(err, io.EOF) { if bytes.HasPrefix(line, []byte("{\"type\":\"error\"")) { From c7364da3f390b69cb10881a8d6ed4aff11e5e3cc Mon Sep 17 00:00:00 2001 From: Hao Wu Date: Thu, 4 Jul 2024 10:42:11 +0800 Subject: [PATCH 4/4] Simplify content block handling in chatStreamClaude3 --- api/chat_main_handler.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/api/chat_main_handler.go b/api/chat_main_handler.go index 4bff571c..3a0af05e 100644 --- a/api/chat_main_handler.go +++ b/api/chat_main_handler.go @@ -875,11 +875,9 @@ func (h *ChatHandler) chatStreamClaude3(w http.ResponseWriter, chatSession sqlc_ } if bytes.HasPrefix(line, []byte("{\"type\":\"content_block_start\"")) { answer = claude.AnswerFromBlockStart(line) - if len(answer) < 200 || len(answer)%2 == 0 { - data, _ := json.Marshal(constructChatCompletionStreamReponse(answer_id, answer)) - fmt.Fprintf(w, "data: %v\n\n", string(data)) - flusher.Flush() - } + data, _ := json.Marshal(constructChatCompletionStreamReponse(answer_id, answer)) + fmt.Fprintf(w, "data: %v\n\n", string(data)) + flusher.Flush() } if bytes.HasPrefix(line, []byte("{\"type\":\"content_block_delta\"")) { answer += claude.AnswerFromBlockDelta(line)