/review

.env.example

@@ -10,11 +10,13 @@ SUPABASE_KEY=
 AUTO_PAY_MODE=
 ANALYTICS_MODE=
 
+
 # Use `trace` to get verbose logging or `info` to show less
 LOG_LEVEL=debug
 LOGDNA_INGESTION_KEY=
 OPENAI_API_HOST=https://api.openai.com
 OPENAI_API_KEY=
+OPENAI_TOKEN_LIMIT=
 CHATGPT_USER_PROMPT_FOR_IMPORTANT_WORDS="I need your help to find important words (e.g. unique adjectives) from github issue below and I want to parse them easily so please separate them using #(No other contexts needed). Please separate the words by # so I can parse them easily. Please answer simply as I only need the important words. Here is the issue content.\n"
 CHATGPT_USER_PROMPT_FOR_MEASURE_SIMILARITY='I have two github issues and I need to measure the possibility of the 2 issues are the same content (No other contents needed and give me only the number in %).\n Give me in number format and add % after the number.\nDo not tell other things since I only need the number (e.g. 85%). Here are two issues:\n 1. "%first%"\n2. "%second%"'
 SIMILARITY_THRESHOLD=80

package.json

@@ -56,6 +56,7 @@
     "node-html-parser": "^6.1.5",
     "node-html-to-image": "^3.3.0",
     "nodemon": "^2.0.19",
+    "openai": "^4.2.0",
     "parse5": "^7.1.2",
     "prettier": "^2.7.1",
     "probot": "^12.2.4",

src/bindings/config.ts

@@ -8,6 +8,7 @@ import {
   DEFAULT_PERMIT_BASE_URL,
   DEFAULT_TIME_RANGE_FOR_MAX_ISSUE,
   DEFAULT_TIME_RANGE_FOR_MAX_ISSUE_ENABLED,
+  DEFAULT_TOKEN_LIMIT,
 } from "../configs";
 import { getPayoutConfigByNetworkId } from "../helpers";
 import { ajv } from "../utils";
@@ -34,6 +35,8 @@ export const loadConfig = async (context: Context): Promise<BotConfig> => {
     registerWalletWithVerification,
     staleBountyTime,
     enableAccessControl,
+    openAIKey,
+    openAITokenLimit,
   } = await getWideConfig(context);
 
   const publicKey = await getScalarKey(process.env.X25519_PRIVATE_KEY);
@@ -99,6 +102,10 @@ export const loadConfig = async (context: Context): Promise<BotConfig> => {
     wallet: {
       registerWalletWithVerification: registerWalletWithVerification,
     },
+    review: {
+      apiKey: openAIKey,
+      tokenLimit: openAITokenLimit || DEFAULT_TOKEN_LIMIT,
+    },
     accessControl: enableAccessControl,
   };
 

src/configs/shared.ts

@@ -27,3 +27,5 @@ export const DEFAULT_DISQUALIFY_TIME = "7 days"; // 7 days
 export const DEFAULT_NETWORK_ID = 1; // ethereum
 export const DEFAULT_RPC_ENDPOINT = "https://rpc-bot.ubq.fi/v1/mainnet";
 export const DEFAULT_PERMIT_BASE_URL = "https://pay.ubq.fi";
+
+export const DEFAULT_TOKEN_LIMIT = 6000;

src/handlers/comment/commands.ts

@@ -6,6 +6,7 @@ export enum IssueCommentCommands {
   PAYOUT = "/payout", // request permit payout
   MULTIPLIER = "/multiplier", // set bounty multiplier (for treasury)
   QUERY = "/query",
+  REVIEW = "/review", // gpt-3.5-turbo-16k will review the pull request
   // Access Controls
 
   ALLOW = "/allow",

src/handlers/comment/handlers/index.ts

@@ -7,6 +7,7 @@ import { listAvailableCommands } from "./help";
 import { unassign } from "./unassign";
 import { registerWallet } from "./wallet";
 import { setAccess } from "./allow";
+import { review } from "./review";
 import { multiplier } from "./multiplier";
 import { BigNumber, ethers } from "ethers";
 import { addPenalty } from "../../../adapters/supabase";
@@ -36,6 +37,7 @@ export * from "./payout";
 export * from "./help";
 export * from "./multiplier";
 export * from "./query";
+export * from "./review";
 
 /**
  * Parses the comment body and figure out the command name a user wants
@@ -251,6 +253,12 @@ export const userCommands = (): UserCommands[] => {
       handler: query,
       callback: commandCallback,
     },
+    {
+      id: IssueCommentCommands.REVIEW,
+      description: `Compares the pull request code diff with the linked issue's specification to perform a review of the current pull request. \n  example usage: /review`,
+      handler: review,
+      callback: commandCallback,
+    },
     {
       id: IssueCommentCommands.MULTIPLIER,
       description: `Set the bounty payout multiplier for a specific contributor, and provide the reason for why. \n  example usage: "/wallet @user 0.5 'Multiplier reason'"`,

src/handlers/comment/handlers/review.ts

@@ -0,0 +1,126 @@
+import { getBotContext, getLogger } from "../../../bindings";
+import { Payload, StreamlinedComment } from "../../../types";
+import { getAllIssueComments, getPullByNumber } from "../../../helpers";
+import { CreateChatCompletionRequestMessage } from "openai/resources/chat";
+import { askGPT, getPRSpec, specCheckTemplate } from "../../../helpers/gpt";
+import { ErrorDiff } from "../../../utils/helpers";
+
+/**
+ * @notice Three calls to OpenAI are made. First for context, second for review and third for finalization.
+ * @returns Pull Request Report
+ */
+export const review = async (body: string) => {
+  const context = getBotContext();
+  const logger = getLogger();
+
+  const payload = context.payload as Payload;
+  const issue = payload.issue;
+
+  if (!issue) {
+    return ErrorDiff(`Payload issue is undefined.`);
+  }
+
+  if (!body) {
+    return ErrorDiff(`Payload body is undefined.`);
+  }
+
+  const isPr = await getPullByNumber(context, issue.number);
+
+  if (!isPr) {
+    return ErrorDiff(`Can only be used on pull requests.`);
+  }
+
+  const reviewRegex = /^\/review/;
+  const reviewRegexMatch = body.match(reviewRegex);
+
+  if (!reviewRegexMatch) {
+    return ErrorDiff(`Error matching regex for review`);
+  }
+
+  const streamlined: StreamlinedComment[] = [];
+  let chatHistory: CreateChatCompletionRequestMessage[] = [];
+  const commentsRaw = await getAllIssueComments(issue.number, "raw");
+
+  if (!commentsRaw) {
+    logger.info(`Error getting issue comments`);
+    return ErrorDiff(`Error getting issue comments.`);
+  }
+
+  // return a diff of the changes made in the PR
+  const comparePR = async () => {
+    const comparePR = await context.octokit.pulls.get({
+      owner: payload.repository.owner.login,
+      repo: payload.repository.name,
+      pull_number: issue.number,
+    });
+
+    const pr = comparePR.data;
+
+    const prDiff = await context.octokit.pulls.get({
+      owner: payload.repository.owner.login,
+      repo: payload.repository.name,
+      pull_number: pr.number,
+      mediaType: {
+        format: "diff",
+      },
+    });
+
+    const diffContent = prDiff.data;
+
+    return {
+      pr,
+      diff: diffContent,
+    };
+  };
+
+  const isPull = async () => {
+    if (isPr) {
+      const diff = await comparePR()
+        .then(({ diff }) => {
+          return diff;
+        })
+        .catch((error) => {
+          logger.info(`Error getting diff: ${error}`);
+          return ErrorDiff(`Error getting diff: ${error}`);
+        });
+
+      const spec = await getPRSpec(context, chatHistory, streamlined);
+
+      chatHistory = [];
+      chatHistory.push(
+        {
+          role: "system",
+          content: specCheckTemplate,
+        } as CreateChatCompletionRequestMessage,
+        {
+          role: "assistant",
+          content: "Spec for Pr: \n" + JSON.stringify(spec),
+        } as CreateChatCompletionRequestMessage,
+        {
+          role: "user",
+          content: `${issue.assignees[0].login}'s PR Diff: \n` + JSON.stringify(diff),
+        } as CreateChatCompletionRequestMessage
+      );
+
+      const gptResponse = await askGPT(`Pr review call for #${issue.number}`, chatHistory);
+
+      if (typeof gptResponse === "string") {
+        return gptResponse;
+      } else {
+        if (gptResponse.answer) {
+          return gptResponse.answer;
+        } else {
+          return ErrorDiff(`No answer found for issue #${issue.number}`);
+        }
+      }
+    } else {
+      return ErrorDiff(`No PR found for issue #${issue.number}`);
+    }
+  };
+
+  const res = await isPull();
+  if (res.startsWith("```diff\n")) {
+    return res;
+  }
+  return res + `\n###### Ensure the pull request requirements are in the linked issue's first comment and update it if the scope evolves.`;
+};