Skip to content

Commit

Permalink
Merge pull request ubiquity-os-marketplace#39 from ubq-testing/develo…
Browse files Browse the repository at this point in the history 
…pment

GPT-4o usage
  • Loading branch information
@0x4007
0x4007 authored Jun 24, 2024
2 parents 6798ea5 + f2f7db4 commit 7e11f5d
Show file tree
Hide file tree
Showing 3 changed files with 9 additions and 87 deletions.
3 changes: 1 addition & 2 deletions package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,6 @@
"@ubiquibot/permit-generation": "1.2.2",
"decimal.js": "10.4.3",
"dotenv": "16.4.5",
"js-tiktoken": "1.0.10",
"jsdom": "24.0.0",
"lodash": "4.17.21",
"markdown-it": "14.1.0",
Expand Down Expand Up @@ -89,4 +88,4 @@
"@commitlint/config-conventional"
]
}
}
}
50 changes: 5 additions & 45 deletions src/parser/content-evaluator-module.ts
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
import Decimal from "decimal.js";
import { encodingForModel } from "js-tiktoken";
import OpenAI from "openai";
import configuration from "../configuration/config-reader";
import { OPENAI_API_KEY } from "../configuration/constants";
Expand Down Expand Up @@ -50,7 +49,7 @@ export class ContentEvaluatorModule implements Module {
async _processComment(comments: Readonly<GithubCommentScore>[], specificationBody: string) {
const commentsWithScore: GithubCommentScore[] = [...comments];
const commentsBody = commentsWithScore.map((comment) => comment.content);
const relevance = await this._sampleRelevanceScoreResults(specificationBody, commentsBody);
const relevance = await this._evaluateComments(specificationBody, commentsBody);

if (relevance.length !== commentsWithScore.length) {
console.error("Relevance / Comment length mismatch! Skipping.");
Expand All @@ -71,12 +70,12 @@ export class ContentEvaluatorModule implements Module {
return commentsWithScore;
}

async _evaluateComments(specification: string, comments: string[]) {
async _evaluateComments(specification: string, comments: string[]): Promise<Decimal[]> {
const prompt = this._generatePrompt(specification, comments);

try {
const response: OpenAI.Chat.ChatCompletion = await this._openAi.chat.completions.create({
model: this._getOptimalModel(prompt),
model: "gpt-4o",
messages: [
{
role: "system",
Expand All @@ -99,44 +98,6 @@ export class ContentEvaluatorModule implements Module {
}
}

_getOptimalModel(prompt: string) {
const encoder = encodingForModel("gpt-3.5-turbo");
const totalSumOfTokens = encoder.encode(prompt).length;

if (totalSumOfTokens <= 4097) {
return "gpt-3.5-turbo";
} else if (totalSumOfTokens <= 16385) {
return "gpt-3.5-turbo-16k";
} else {
console.warn("Backup plan for development purposes only, but using gpt-4 due to huge context size");
return "gpt-4-turbo-preview";
}
}

async _sampleRelevanceScoreResults(specification: string, comments: string[]) {
const BATCH_SIZE = 10;
const evaluationPromises: ReturnType<typeof this._evaluateComments>[] = [];

for (let i = 0; i < BATCH_SIZE; ++i) {
evaluationPromises.push(this._evaluateComments(specification, comments));
}

const results = await Promise.all(evaluationPromises);

// Calculate the sum of each column
const columnSums: Decimal[] = [];
for (let j = 0; j < results[0].length; j++) {
let sum = new Decimal(0);
for (let i = 0; i < results.length; i++) {
sum = sum.plus(results[i][j] || 0);
}
columnSums.push(sum);
}

// Return the average of each column
return columnSums.map((sum) => sum.dividedBy(results.length));
}

_generatePrompt(issue: string, comments: string[]) {
if (!issue?.length) {
throw new Error("Issue specification comment is missing or empty");
Expand All @@ -145,8 +106,7 @@ export class ContentEvaluatorModule implements Module {
.map((comment) => comment)
.join(
"\n"
)}\n\`\`\`\n\n\nTo what degree are each of the comments in the conversation relevant and valuable to further defining the issue specification? Please reply with ONLY an array of float numbers between 0 and 1, corresponding to each comment in the order they appear. Each float should represent the degree of relevance and added value of the comment to the issue. The total length of the array in your response should equal exactly ${
comments.length
} elements.`;
)}\n\`\`\`\n\n\nTo what degree are each of the comments in the conversation relevant and valuable to further defining the issue specification? Please reply with ONLY an array of float numbers between 0 and 1, corresponding to each comment in the order they appear. Each float should represent the degree of relevance and added value of the comment to the issue. The total length of the array in your response should equal exactly ${comments.length
} elements.`;
}
}
43 changes: 3 additions & 40 deletions yarn.lock
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3646,11 +3646,6 @@ base-64@^0.1.0:
resolved "https://registry.yarnpkg.com/base-64/-/base-64-0.1.0.tgz#780a99c84e7d600260361511c4877613bf24f6bb"
integrity sha512-Y5gU45svrR5tI2Vt/X9GPd3L0HNIKzGu202EjxrXMpuc2V2CiKgemAbUUsqYmZJvPtCXoUKjNZwBJzsNScUbXA==

base64-js@^1.5.1:
version "1.5.1"
resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.5.1.tgz#1b1b440160a5bf7ad40b650f095963481903930a"
integrity sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==

[email protected]:
version "1.1.4"
resolved "https://registry.yarnpkg.com/bech32/-/bech32-1.1.4.tgz#e38c9f37bf179b8eb16ae3a772b40c356d4832e9"
Expand Down Expand Up @@ -6173,13 +6168,6 @@ [email protected]:
resolved "https://registry.yarnpkg.com/js-sha3/-/js-sha3-0.8.0.tgz#b9b7a5da73afad7dedd0f8c463954cbde6818840"
integrity sha512-gF1cRrHhIzNfToc802P800N8PpXS+evLLXfsVpowqmAFR9uwbi89WvXg2QspOmXL8QL86J4T1EpFu+yUkwJY3Q==

[email protected]:
version "1.0.10"
resolved "https://registry.yarnpkg.com/js-tiktoken/-/js-tiktoken-1.0.10.tgz#2b343ec169399dcee8f9ef9807dbd4fafd3b30dc"
integrity sha512-ZoSxbGjvGyMT13x6ACo9ebhDha/0FHdKA+OsQcMOWcm1Zs7r90Rhk5lhERLzji+3rA7EKpXCgwXcM5fF3DMpdA==
dependencies:
base64-js "^1.5.1"

js-tokens@^4.0.0:
version "4.0.0"
resolved "https://registry.yarnpkg.com/js-tokens/-/js-tokens-4.0.0.tgz#19203fb59991df98e3a287050d4647cdeaf32499"
Expand Down Expand Up @@ -7882,16 +7870,7 @@ string-length@^4.0.1:
char-regex "^1.0.2"
strip-ansi "^6.0.0"

"string-width-cjs@npm:string-width@^4.2.0":
version "4.2.3"
resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010"
integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==
dependencies:
emoji-regex "^8.0.0"
is-fullwidth-code-point "^3.0.0"
strip-ansi "^6.0.1"

string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.3:
"string-width-cjs@npm:string-width@^4.2.0", string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.3:
version "4.2.3"
resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010"
integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==
Expand Down Expand Up @@ -7968,14 +7947,7 @@ string_decoder@^1.1.1:
dependencies:
safe-buffer "~5.2.0"

"strip-ansi-cjs@npm:strip-ansi@^6.0.1":
version "6.0.1"
resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9"
integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==
dependencies:
ansi-regex "^5.0.1"

strip-ansi@^6.0.0, strip-ansi@^6.0.1:
"strip-ansi-cjs@npm:strip-ansi@^6.0.1", strip-ansi@^6.0.0, strip-ansi@^6.0.1:
version "6.0.1"
resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9"
integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==
Expand Down Expand Up @@ -8602,7 +8574,7 @@ which@^4.0.0:
dependencies:
isexe "^3.1.1"

"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0":
"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0", wrap-ansi@^7.0.0:
version "7.0.0"
resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43"
integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==
Expand All @@ -8620,15 +8592,6 @@ wrap-ansi@^6.2.0:
string-width "^4.1.0"
strip-ansi "^6.0.0"

wrap-ansi@^7.0.0:
version "7.0.0"
resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43"
integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==
dependencies:
ansi-styles "^4.0.0"
string-width "^4.1.0"
strip-ansi "^6.0.0"

wrap-ansi@^8.1.0:
version "8.1.0"
resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-8.1.0.tgz#56dc22368ee570face1b49819975d9b9a5ead214"
Expand Down

0 comments on commit 7e11f5d

Please sign in to comment.