Merge pull request #14 from ubq-testing/feat/dynamic-ground-truths

feat: dynamic ground truths

.cspell.json

@@ -31,7 +31,10 @@
     "nemo",
     "Reranking",
     "mistralai",
-    "OPENROUTER_API_KEY"
+    "Typeguard",
+    "typeguards",
+    "OPENROUTER_API_KEY",
+    "Openrouter"
   ],
   "dictionaries": ["typescript", "node", "software-terms"],
   "import": ["@cspell/dict-typescript/cspell-ext.json", "@cspell/dict-node/cspell-ext.json", "@cspell/dict-software-terms"],

.github/workflows/compute.yml

@@ -46,9 +46,9 @@ jobs:
         run: yarn tsx ./src/main.ts
         id: command-ask
         env:
-            SUPABASE_URL: ${{ secrets.SUPABASE_URL }}
-            SUPABASE_KEY: ${{ secrets.SUPABASE_KEY }}
-            VOYAGEAI_API_KEY: ${{ secrets.VOYAGEAI_API_KEY }}
-            OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
-            OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
-            UBIQUITY_OS_APP_NAME: ${{ secrets.UBIQUITY_OS_APP_NAME }}
+          SUPABASE_URL: ${{ secrets.SUPABASE_URL }}
+          SUPABASE_KEY: ${{ secrets.SUPABASE_KEY }}
+          VOYAGEAI_API_KEY: ${{ secrets.VOYAGEAI_API_KEY }}
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+          OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
+          UBIQUITY_OS_APP_NAME: ${{ secrets.UBIQUITY_OS_APP_NAME }}

.github/workflows/update-configuration.yml

@@ -18,4 +18,4 @@ jobs:
           commitMessage: "chore: updated manifest.json and dist build"
           nodeVersion: "20.10.0"
         env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -15,4 +15,4 @@ junit.xml
 cypress/screenshots
 script.ts
 .wrangler
-test-dashboard.md
+test-dashboard.md

package.json

@@ -17,7 +17,7 @@
     "knip-ci": "knip --no-exit-code --reporter json --config .github/knip.ts",
     "prepare": "husky install",
     "test": "jest --setupFiles dotenv/config --coverage",
-    "worker": "wrangler dev --env dev --port 5000"
+    "worker": "wrangler dev --env dev --port 4000"
   },
   "keywords": [
     "typescript",
@@ -66,7 +66,7 @@
     "prettier": "3.3.2",
     "ts-jest": "29.1.5",
     "tsx": "4.15.6",
-    "typescript": "5.4.5",
+    "typescript": "^5.6.3",
     "typescript-eslint": "7.13.1",
     "wrangler": "^3.81.0"
   },

src/adapters/openai/helpers/completions.ts

@@ -1,10 +1,12 @@
 import OpenAI from "openai";
 import { Context } from "../../../types";
 import { SuperOpenAi } from "./openai";
+import { CompletionsModelHelper, ModelApplications } from "../../../types/llm";
 const MAX_TOKENS = 7000;
 
 export interface CompletionsType {
   answer: string;
+  groundTruths: string[];
   tokenUsage: {
     input: number;
     output: number;
@@ -72,8 +74,47 @@ export class Completions extends SuperOpenAi {
     });
     const answer = res.choices[0].message;
     if (answer && answer.content && res.usage) {
-      return { answer: answer.content, tokenUsage: { input: res.usage.prompt_tokens, output: res.usage.completion_tokens, total: res.usage.total_tokens } };
+      return {
+        answer: answer.content,
+        groundTruths,
+        tokenUsage: { input: res.usage.prompt_tokens, output: res.usage.completion_tokens, total: res.usage.total_tokens },
+      };
     }
-    return { answer: "", tokenUsage: { input: 0, output: 0, total: 0 } };
+    return { answer: "", tokenUsage: { input: 0, output: 0, total: 0 }, groundTruths };
+  }
+
+  async createGroundTruthCompletion<TApp extends ModelApplications>(
+    context: Context,
+    groundTruthSource: string,
+    systemMsg: string,
+    model: CompletionsModelHelper<TApp>
+  ): Promise<string | null> {
+    const {
+      env: { OPENAI_API_KEY },
+      config: { openAiBaseUrl },
+    } = context;
+
+    const openAi = new OpenAI({
+      apiKey: OPENAI_API_KEY,
+      ...(openAiBaseUrl && { baseURL: openAiBaseUrl }),
+    });
+
+    const msgs = [
+      {
+        role: "system",
+        content: systemMsg,
+      },
+      {
+        role: "user",
+        content: groundTruthSource,
+      },
+    ] as OpenAI.Chat.Completions.ChatCompletionMessageParam[];
+
+    const res = await openAi.chat.completions.create({
+      messages: msgs,
+      model: model,
+    });
+
+    return res.choices[0].message.content;
   }
 }

src/handlers/ask-llm.ts

@@ -5,6 +5,8 @@ import { IssueSimilaritySearchResult } from "../adapters/supabase/helpers/issues
 import { recursivelyFetchLinkedIssues } from "../helpers/issue-fetching";
 import { formatChatHistory } from "../helpers/format-chat-history";
 import { optimizeContext } from "../helpers/issue";
+import { fetchRepoDependencies, fetchRepoLanguageStats } from "./ground-truths/chat-bot";
+import { findGroundTruths } from "./ground-truths/find-ground-truths";
 
 /**
  * Asks a question to GPT and returns the response
@@ -62,12 +64,14 @@ export async function askGpt(context: Context, question: string, formattedChat:
   // const reRankedChat = formattedChat.length > 0 ? await context.adapters.voyage.reranker.reRankResults(formattedChat.filter(text => text !== ""), question, 300) : [];
   similarText = similarText.filter((text) => text !== "");
   const rerankedText = similarText.length > 0 ? await context.adapters.voyage.reranker.reRankResults(similarText, question) : [];
-  return context.adapters.openai.completions.createCompletion(
-    question,
-    model,
-    rerankedText,
-    formattedChat,
-    ["typescript", "github", "cloudflare worker", "actions", "jest", "supabase", "openai"],
-    UBIQUITY_OS_APP_NAME
-  );
+
+  const languages = await fetchRepoLanguageStats(context);
+  const { dependencies, devDependencies } = await fetchRepoDependencies(context);
+  const groundTruths = await findGroundTruths(context, "chat-bot", {
+    languages,
+    dependencies,
+    devDependencies,
+  });
+
+  return context.adapters.openai.completions.createCompletion(question, model, rerankedText, formattedChat, groundTruths, UBIQUITY_OS_APP_NAME);
 }

src/handlers/ground-truths/chat-bot.ts

@@ -0,0 +1,64 @@
+import { Context } from "../../types";
+import { logger } from "../../helpers/errors";
+
+export async function fetchRepoDependencies(context: Context) {
+  const {
+    octokit,
+    payload: {
+      repository: {
+        owner: { login: owner },
+        name: repo,
+      },
+    },
+  } = context;
+
+  const { data: packageJson } = await octokit.repos.getContent({
+    owner,
+    repo,
+    path: "package.json",
+  });
+
+  if ("content" in packageJson) {
+    return extractDependencies(JSON.parse(Buffer.from(packageJson.content, "base64").toString()));
+  } else {
+    throw logger.error(`No package.json found in ${owner}/${repo}`);
+  }
+}
+
+export function extractDependencies(packageJson: Record<string, Record<string, string>>) {
+  const { dependencies, devDependencies } = packageJson;
+
+  return {
+    dependencies,
+    devDependencies,
+  };
+}
+
+export async function fetchRepoLanguageStats(context: Context) {
+  const {
+    octokit,
+    payload: {
+      repository: {
+        owner: { login: owner },
+        name: repo,
+      },
+    },
+  } = context;
+
+  const { data: languages } = await octokit.repos.listLanguages({
+    owner,
+    repo,
+  });
+
+  const totalBytes = Object.values(languages).reduce((acc, bytes) => acc + bytes, 0);
+
+  const stats = Object.entries(languages).reduce(
+    (acc, [language, bytes]) => {
+      acc[language] = bytes / totalBytes;
+      return acc;
+    },
+    {} as Record<string, number>
+  );
+
+  return Array.from(Object.entries(stats)).sort((a, b) => b[1] - a[1]);
+}

src/handlers/ground-truths/create-system-message.ts

@@ -0,0 +1,17 @@
+export function createGroundTruthSysMsg({ truthRules, example, conditions }: { truthRules: string[]; example: string[]; conditions?: string[] }) {
+  return `
+Using the input provided, your goal is to produce an array of strings that represent "Ground Truths."
+These ground truths are high-level abstractions that encapsulate the tech stack and dependencies of the repository.
+  
+Each ground truth should:
+- ${truthRules.join("\n- ")}
+  
+Example:
+${example.join("\n")}
+  
+${conditions ? `Conditions:\n${conditions.join("\n")}` : ""}
+  
+Generate similar ground truths adhering to a maximum of 10.
+  
+Return a JSON parsable array of strings representing the ground truths, without comment or directive.`;
+}

src/handlers/ground-truths/find-ground-truths.ts

@@ -0,0 +1,57 @@
+import { Context } from "../../types";
+import { AppParamsHelper, GroundTruthsSystemMessage, ModelApplications } from "../../types/llm";
+import { GROUND_TRUTHS_SYSTEM_MESSAGES } from "./prompts";
+import { chatBotPayloadTypeguard, codeReviewPayloadTypeguard } from "../../types/typeguards";
+import { validateGroundTruths } from "./validate";
+import { logger } from "../../helpers/errors";
+import { createGroundTruthSysMsg } from "./create-system-message";
+
+export async function findGroundTruths<TApp extends ModelApplications = ModelApplications>(
+  context: Context,
+  application: TApp,
+  params: AppParamsHelper<TApp>
+): Promise<string[]> {
+  const systemMsgObj = GROUND_TRUTHS_SYSTEM_MESSAGES[application];
+
+  // params are deconstructed to show quickly what's being passed to the function
+
+  if (chatBotPayloadTypeguard(params)) {
+    const { dependencies, devDependencies, languages } = params;
+    return findChatBotTruths(context, { dependencies, devDependencies, languages }, systemMsgObj);
+  } else if (codeReviewPayloadTypeguard(params)) {
+    const { taskSpecification } = params;
+    return findCodeReviewTruths(context, { taskSpecification }, systemMsgObj);
+  } else {
+    throw logger.error("Invalid payload type for ground truths");
+  }
+}
+
+async function findChatBotTruths(
+  context: Context,
+  params: AppParamsHelper<"chat-bot">,
+  systemMsgObj: GroundTruthsSystemMessage<"chat-bot">
+): Promise<string[]> {
+  const {
+    adapters: {
+      openai: { completions },
+    },
+  } = context;
+  const systemMsg = createGroundTruthSysMsg(systemMsgObj);
+  const truths = await completions.createGroundTruthCompletion<"chat-bot">(context, JSON.stringify(params), systemMsg, "o1-mini");
+  return validateGroundTruths(truths);
+}
+
+async function findCodeReviewTruths(
+  context: Context,
+  params: AppParamsHelper<"code-review">,
+  systemMsgObj: GroundTruthsSystemMessage<"code-review">
+): Promise<string[]> {
+  const {
+    adapters: {
+      openai: { completions },
+    },
+  } = context;
+  const systemMsg = createGroundTruthSysMsg(systemMsgObj);
+  const truths = await completions.createGroundTruthCompletion<"code-review">(context, params.taskSpecification, systemMsg, "gpt-4o");
+  return validateGroundTruths(truths);
+}

src/handlers/ground-truths/prompts.ts

@@ -0,0 +1,57 @@
+import { GroundTruthsSystemMessageTemplate, ModelApplications } from "../../types/llm";
+
+const CODE_REVIEW_GROUND_TRUTHS_SYSTEM_MESSAGE = {
+  example: [
+    `Using the input provided, your goal is to produce an array of strings that represent "Ground Truths."
+        These ground truths are high-level abstractions that encapsulate the key aspects of the task.
+        They serve to guide and inform our code review model's interpretation of the task by providing clear, concise, and explicit insights.
+        
+        Each ground truth should:
+        - Be succinct and easy to understand.
+        - Directly pertain to the task at hand.
+        - Focus on essential requirements, behaviors, or assumptions involved in the task.
+    
+        Example:
+        Task: Implement a function that adds two numbers.
+        Ground Truths:
+        - The function should accept two numerical inputs.
+        - The function should return the sum of the two inputs.
+        - Inputs must be validated to ensure they are numbers.
+        
+        Based on the given task, generate similar ground truths adhering to a maximum of 10.
+        
+        Return a JSON parsable array of strings representing the ground truths, without comment or directive.`,
+  ],
+  truthRules: [],
+  conditions: [],
+};
+
+const CHAT_BOT_GROUND_TRUTHS_SYSTEM_MESSAGE = {
+  truthRules: [
+    "Be succinct and easy to understand.",
+    "Use only the information provided in the input.",
+    "Focus on essential requirements, behaviors, or assumptions involved in the repository.",
+  ],
+  example: [
+    "Languages: { TypeScript: 60%, JavaScript: 15%, HTML: 10%, CSS: 5%, ... }",
+    "Dependencies: Esbuild, Wrangler, React, Tailwind CSS, ms, React-carousel, React-icons, ...",
+    "Dev Dependencies: @types/node, @types/jest, @mswjs, @testing-library/react, @testing-library/jest-dom, @Cypress ...",
+    "Ground Truths:",
+    "- The repo predominantly uses TypeScript, with JavaScript, HTML, and CSS also present.",
+    "- The repo is a React project that uses Tailwind CSS.",
+    "- The project is built with Esbuild and deployed with Wrangler, indicating a Cloudflare Workers project.",
+    "- The repo tests use Jest, Cypress, mswjs, and React Testing Library.",
+  ],
+  conditions: [
+    "Assume your output builds the foundation for a chatbot to understand the repository when asked an arbitrary query.",
+    "Do not list every language or dependency, focus on the most prevalent ones.",
+    "Focus on what is essential to understand the repository at a high level.",
+    "Brevity is key. Use zero formatting. Do not wrap in quotes, backticks, or other characters.",
+    `response === ["some", "array", "of", "strings"]`,
+  ],
+};
+
+export const GROUND_TRUTHS_SYSTEM_MESSAGES: Record<ModelApplications, GroundTruthsSystemMessageTemplate> = {
+  "code-review": CODE_REVIEW_GROUND_TRUTHS_SYSTEM_MESSAGE,
+  "chat-bot": CHAT_BOT_GROUND_TRUTHS_SYSTEM_MESSAGE,
+} as const;

src/handlers/ground-truths/validate.ts

@@ -0,0 +1,29 @@
+import { logger } from "../../helpers/errors";
+
+export function validateGroundTruths(truthsString: string | null): string[] {
+  let truths;
+  if (!truthsString) {
+    throw logger.error("Failed to generate ground truths");
+  }
+
+  try {
+    truths = JSON.parse(truthsString);
+  } catch (err) {
+    throw logger.error("Failed to parse ground truths");
+  }
+  if (!Array.isArray(truths)) {
+    throw logger.error("Ground truths must be an array");
+  }
+
+  if (truths.length > 10) {
+    throw logger.error("Ground truths must not exceed 10");
+  }
+
+  truths.forEach((truth: string) => {
+    if (typeof truth !== "string") {
+      throw logger.error("Each ground truth must be a string");
+    }
+  });
+
+  return truths;
+}

src/helpers/errors.ts

@@ -0,0 +1,3 @@
+import { Logs } from "@ubiquity-dao/ubiquibot-logger"; // import is fixed in #13
+
+export const logger = new Logs("debug");