From f523f7831610d9c76b515a41029ed7dcd64fe9ef Mon Sep 17 00:00:00 2001 From: Keyrxng <106303466+Keyrxng@users.noreply.github.com> Date: Thu, 13 Jun 2024 22:20:07 +0100 Subject: [PATCH 1/3] chore: gpt-4o usage --- package.json | 2 +- src/parser/content-evaluator-module.ts | 35 +++++------------------ yarn.lock | 39 +++++--------------------- 3 files changed, 15 insertions(+), 61 deletions(-) diff --git a/package.json b/package.json index c41baabc..242705a8 100644 --- a/package.json +++ b/package.json @@ -34,7 +34,7 @@ "@ubiquibot/permit-generation": "1.2.2", "decimal.js": "10.4.3", "dotenv": "16.4.5", - "js-tiktoken": "1.0.10", + "js-tiktoken": "^1.0.12", "jsdom": "24.0.0", "lodash": "4.17.21", "markdown-it": "14.1.0", diff --git a/src/parser/content-evaluator-module.ts b/src/parser/content-evaluator-module.ts index 289e78d7..4a9baa69 100644 --- a/src/parser/content-evaluator-module.ts +++ b/src/parser/content-evaluator-module.ts @@ -71,7 +71,7 @@ export class ContentEvaluatorModule implements Module { return commentsWithScore; } - async _evaluateComments(specification: string, comments: string[]) { + async _evaluateComments(specification: string, comments: string[]): Promise { const prompt = this._generatePrompt(specification, comments); try { @@ -100,41 +100,21 @@ export class ContentEvaluatorModule implements Module { } _getOptimalModel(prompt: string) { - const encoder = encodingForModel("gpt-3.5-turbo"); + const encoder = encodingForModel("gpt-4o"); const totalSumOfTokens = encoder.encode(prompt).length; if (totalSumOfTokens <= 4097) { return "gpt-3.5-turbo"; } else if (totalSumOfTokens <= 16385) { - return "gpt-3.5-turbo-16k"; + return "gpt-3.5-turbo-16k"; // gpt-4o? } else { console.warn("Backup plan for development purposes only, but using gpt-4 due to huge context size"); - return "gpt-4-turbo-preview"; + return "gpt-4o"; } } async _sampleRelevanceScoreResults(specification: string, comments: string[]) { - const BATCH_SIZE = 10; - const evaluationPromises: ReturnType[] = []; - - for (let i = 0; i < BATCH_SIZE; ++i) { - evaluationPromises.push(this._evaluateComments(specification, comments)); - } - - const results = await Promise.all(evaluationPromises); - - // Calculate the sum of each column - const columnSums: Decimal[] = []; - for (let j = 0; j < results[0].length; j++) { - let sum = new Decimal(0); - for (let i = 0; i < results.length; i++) { - sum = sum.plus(results[i][j] || 0); - } - columnSums.push(sum); - } - - // Return the average of each column - return columnSums.map((sum) => sum.dividedBy(results.length)); + return await this._evaluateComments(specification, comments); } _generatePrompt(issue: string, comments: string[]) { @@ -145,8 +125,7 @@ export class ContentEvaluatorModule implements Module { .map((comment) => comment) .join( "\n" - )}\n\`\`\`\n\n\nTo what degree are each of the comments in the conversation relevant and valuable to further defining the issue specification? Please reply with ONLY an array of float numbers between 0 and 1, corresponding to each comment in the order they appear. Each float should represent the degree of relevance and added value of the comment to the issue. The total length of the array in your response should equal exactly ${ - comments.length - } elements.`; + )}\n\`\`\`\n\n\nTo what degree are each of the comments in the conversation relevant and valuable to further defining the issue specification? Please reply with ONLY an array of float numbers between 0 and 1, corresponding to each comment in the order they appear. Each float should represent the degree of relevance and added value of the comment to the issue. The total length of the array in your response should equal exactly ${comments.length + } elements.`; } } diff --git a/yarn.lock b/yarn.lock index d5c5ad98..0f237c7b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6122,10 +6122,10 @@ js-sha3@0.8.0: resolved "https://registry.yarnpkg.com/js-sha3/-/js-sha3-0.8.0.tgz#b9b7a5da73afad7dedd0f8c463954cbde6818840" integrity sha512-gF1cRrHhIzNfToc802P800N8PpXS+evLLXfsVpowqmAFR9uwbi89WvXg2QspOmXL8QL86J4T1EpFu+yUkwJY3Q== -js-tiktoken@1.0.10: - version "1.0.10" - resolved "https://registry.yarnpkg.com/js-tiktoken/-/js-tiktoken-1.0.10.tgz#2b343ec169399dcee8f9ef9807dbd4fafd3b30dc" - integrity sha512-ZoSxbGjvGyMT13x6ACo9ebhDha/0FHdKA+OsQcMOWcm1Zs7r90Rhk5lhERLzji+3rA7EKpXCgwXcM5fF3DMpdA== +js-tiktoken@^1.0.12: + version "1.0.12" + resolved "https://registry.yarnpkg.com/js-tiktoken/-/js-tiktoken-1.0.12.tgz#af0f5cf58e5e7318240d050c8413234019424211" + integrity sha512-L7wURW1fH9Qaext0VzaUDpFGVQgjkdE3Dgsy9/+yXyGEpBKnylTd0mU0bfbNkKDlXRb6TEsZkwuflu1B8uQbJQ== dependencies: base64-js "^1.5.1" @@ -7787,16 +7787,7 @@ string-length@^4.0.1: char-regex "^1.0.2" strip-ansi "^6.0.0" -"string-width-cjs@npm:string-width@^4.2.0": - version "4.2.3" - resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" - integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== - dependencies: - emoji-regex "^8.0.0" - is-fullwidth-code-point "^3.0.0" - strip-ansi "^6.0.1" - -string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.3: +"string-width-cjs@npm:string-width@^4.2.0", string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.3: version "4.2.3" resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== @@ -7873,14 +7864,7 @@ string_decoder@^1.1.1: dependencies: safe-buffer "~5.2.0" -"strip-ansi-cjs@npm:strip-ansi@^6.0.1": - version "6.0.1" - resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" - integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== - dependencies: - ansi-regex "^5.0.1" - -strip-ansi@^6.0.0, strip-ansi@^6.0.1: +"strip-ansi-cjs@npm:strip-ansi@^6.0.1", strip-ansi@^6.0.0, strip-ansi@^6.0.1: version "6.0.1" resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== @@ -8507,7 +8491,7 @@ which@^4.0.0: dependencies: isexe "^3.1.1" -"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0": +"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0", wrap-ansi@^7.0.0: version "7.0.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== @@ -8525,15 +8509,6 @@ wrap-ansi@^6.2.0: string-width "^4.1.0" strip-ansi "^6.0.0" -wrap-ansi@^7.0.0: - version "7.0.0" - resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" - integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== - dependencies: - ansi-styles "^4.0.0" - string-width "^4.1.0" - strip-ansi "^6.0.0" - wrap-ansi@^8.1.0: version "8.1.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-8.1.0.tgz#56dc22368ee570face1b49819975d9b9a5ead214" From e72afd5ec4ec56e31ed117df885c098332a203b1 Mon Sep 17 00:00:00 2001 From: Keyrxng <106303466+Keyrxng@users.noreply.github.com> Date: Fri, 14 Jun 2024 12:08:24 +0100 Subject: [PATCH 2/3] chore: gpt-4o only --- src/parser/content-evaluator-module.ts | 25 +++---------------------- 1 file changed, 3 insertions(+), 22 deletions(-) diff --git a/src/parser/content-evaluator-module.ts b/src/parser/content-evaluator-module.ts index 4a9baa69..3af27ca3 100644 --- a/src/parser/content-evaluator-module.ts +++ b/src/parser/content-evaluator-module.ts @@ -1,5 +1,4 @@ import Decimal from "decimal.js"; -import { encodingForModel } from "js-tiktoken"; import OpenAI from "openai"; import configuration from "../configuration/config-reader"; import { OPENAI_API_KEY } from "../configuration/constants"; @@ -50,7 +49,7 @@ export class ContentEvaluatorModule implements Module { async _processComment(comments: Readonly[], specificationBody: string) { const commentsWithScore: GithubCommentScore[] = [...comments]; const commentsBody = commentsWithScore.map((comment) => comment.content); - const relevance = await this._sampleRelevanceScoreResults(specificationBody, commentsBody); + const relevance = await this._evaluateComments(specificationBody, commentsBody); if (relevance.length !== commentsWithScore.length) { console.error("Relevance / Comment length mismatch! Skipping."); @@ -71,12 +70,12 @@ export class ContentEvaluatorModule implements Module { return commentsWithScore; } - async _evaluateComments(specification: string, comments: string[]): Promise { + async _evaluateComments(specification: string, comments: string[]): Promise { const prompt = this._generatePrompt(specification, comments); try { const response: OpenAI.Chat.ChatCompletion = await this._openAi.chat.completions.create({ - model: this._getOptimalModel(prompt), + model: "gpt-4o", messages: [ { role: "system", @@ -99,24 +98,6 @@ export class ContentEvaluatorModule implements Module { } } - _getOptimalModel(prompt: string) { - const encoder = encodingForModel("gpt-4o"); - const totalSumOfTokens = encoder.encode(prompt).length; - - if (totalSumOfTokens <= 4097) { - return "gpt-3.5-turbo"; - } else if (totalSumOfTokens <= 16385) { - return "gpt-3.5-turbo-16k"; // gpt-4o? - } else { - console.warn("Backup plan for development purposes only, but using gpt-4 due to huge context size"); - return "gpt-4o"; - } - } - - async _sampleRelevanceScoreResults(specification: string, comments: string[]) { - return await this._evaluateComments(specification, comments); - } - _generatePrompt(issue: string, comments: string[]) { if (!issue?.length) { throw new Error("Issue specification comment is missing or empty"); From f2f7db41a6892f5a65f91b9727d5771ab95aa42a Mon Sep 17 00:00:00 2001 From: Keyrxng <106303466+Keyrxng@users.noreply.github.com> Date: Fri, 14 Jun 2024 12:10:23 +0100 Subject: [PATCH 3/3] chore: remove js-tiktoken --- package.json | 3 +-- yarn.lock | 12 ------------ 2 files changed, 1 insertion(+), 14 deletions(-) diff --git a/package.json b/package.json index 242705a8..92afa93b 100644 --- a/package.json +++ b/package.json @@ -34,7 +34,6 @@ "@ubiquibot/permit-generation": "1.2.2", "decimal.js": "10.4.3", "dotenv": "16.4.5", - "js-tiktoken": "^1.0.12", "jsdom": "24.0.0", "lodash": "4.17.21", "markdown-it": "14.1.0", @@ -88,4 +87,4 @@ "@commitlint/config-conventional" ] } -} +} \ No newline at end of file diff --git a/yarn.lock b/yarn.lock index 0f237c7b..a1a1d3d0 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3641,11 +3641,6 @@ base-64@^0.1.0: resolved "https://registry.yarnpkg.com/base-64/-/base-64-0.1.0.tgz#780a99c84e7d600260361511c4877613bf24f6bb" integrity sha512-Y5gU45svrR5tI2Vt/X9GPd3L0HNIKzGu202EjxrXMpuc2V2CiKgemAbUUsqYmZJvPtCXoUKjNZwBJzsNScUbXA== -base64-js@^1.5.1: - version "1.5.1" - resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.5.1.tgz#1b1b440160a5bf7ad40b650f095963481903930a" - integrity sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA== - bech32@1.1.4: version "1.1.4" resolved "https://registry.yarnpkg.com/bech32/-/bech32-1.1.4.tgz#e38c9f37bf179b8eb16ae3a772b40c356d4832e9" @@ -6122,13 +6117,6 @@ js-sha3@0.8.0: resolved "https://registry.yarnpkg.com/js-sha3/-/js-sha3-0.8.0.tgz#b9b7a5da73afad7dedd0f8c463954cbde6818840" integrity sha512-gF1cRrHhIzNfToc802P800N8PpXS+evLLXfsVpowqmAFR9uwbi89WvXg2QspOmXL8QL86J4T1EpFu+yUkwJY3Q== -js-tiktoken@^1.0.12: - version "1.0.12" - resolved "https://registry.yarnpkg.com/js-tiktoken/-/js-tiktoken-1.0.12.tgz#af0f5cf58e5e7318240d050c8413234019424211" - integrity sha512-L7wURW1fH9Qaext0VzaUDpFGVQgjkdE3Dgsy9/+yXyGEpBKnylTd0mU0bfbNkKDlXRb6TEsZkwuflu1B8uQbJQ== - dependencies: - base64-js "^1.5.1" - js-tokens@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/js-tokens/-/js-tokens-4.0.0.tgz#19203fb59991df98e3a287050d4647cdeaf32499"