feat(agent): add removeDuplicateSuffixLines postprocess filter (#3392)

* feat: add removeDuplicateSuffixLines postprocess filter

* chore: remove duplicate suffix lines in postprocess filter

---------

Co-authored-by: Zhiming Ma <[email protected]>

clients/tabby-agent/src/codeCompletion/postprocess/index.ts

@@ -13,6 +13,7 @@ import { trimMultiLineInSingleLineMode } from "./trimMultiLineInSingleLineMode";
 import { dropDuplicated } from "./dropDuplicated";
 import { dropMinimum } from "./dropMinimum";
 import { calculateReplaceRange } from "./calculateReplaceRange";
+import { removeDuplicateSuffixLines } from "./removeDuplicateSuffixLines";
 import { normalizeIndentation } from "./normalizeIndentation";
 
 type ItemListFilter = (items: CompletionItem[]) => Promise<CompletionItem[]>;
@@ -55,6 +56,7 @@ export async function postCacheProcess(
     .then(applyFilter(normalizeIndentation))
     .then(applyFilter(dropDuplicated))
     .then(applyFilter(trimSpace))
+    .then(applyFilter(removeDuplicateSuffixLines))
     .then(applyFilter(dropMinimum))
     .then(applyFilter(calculateReplaceRange));
 }

clients/tabby-agent/src/codeCompletion/postprocess/removeDuplicateSuffixLines.test.ts

@@ -0,0 +1,120 @@
+import { documentContext, inline, assertFilterResult } from "./testUtils";
+import { removeDuplicateSuffixLines } from "./removeDuplicateSuffixLines";
+
+describe("removeDuplicateSuffixLines", () => {
+  const filter = removeDuplicateSuffixLines();
+
+  it("should remove duplicated array item", async () => {
+    const context = documentContext`
+      const items = [
+        ║
+        { id: 2 },
+        { id: 3 }
+      ];
+    `;
+    const completion = inline`
+      ├{ id: 1 },
+      { id: 2 },┤
+    `;
+    const expected = inline`
+      ├{ id: 1 },┤
+    `;
+    await assertFilterResult(filter, context, completion, expected);
+  });
+
+  it("should handle empty content after cursor", async () => {
+    const context = documentContext`
+      const a = ║
+    `;
+    const completion = inline`
+      ├42;┤
+    `;
+    const expected = completion;
+    await assertFilterResult(filter, context, completion, expected);
+  });
+
+  it("should remove duplicated comma items", async () => {
+    const context = documentContext`
+      function example() {
+        const items = [
+          ║
+          4,
+          5,
+          6
+        ];
+      }
+    `;
+    const completion = inline`
+      ├1,
+      2,
+      3,
+      4,┤
+    `;
+    const expected = inline`
+      ├1,
+      2,
+      3,┤
+    `;
+    await assertFilterResult(filter, context, completion, expected);
+  });
+
+  it("should remove duplicate method calls", async () => {
+    const context = documentContext`
+      class Example {
+        constructor() {
+          ║
+          this.setup();
+          this.init()
+        }
+      }
+    `;
+    const completion = inline`
+      ├this.value = 1;
+      this.name = "test";
+      this.items = [];
+      this.setup();┤
+    `;
+    const expected = inline`
+      ├this.value = 1;
+      this.name = "test";
+      this.items = [];┤
+    `;
+    await assertFilterResult(filter, context, completion, expected);
+  });
+
+  it("should remove duplicate object properties", async () => {
+    const context = documentContext`
+      const config = {
+        ║
+        enabled: true,
+        debug: false          
+      };
+    `;
+    const completion = inline`
+      ├name: "test",
+      value: 42,
+      items: [],
+      enabled: true,┤
+    `;
+    const expected = inline`
+      ├name: "test",
+      value: 42,
+      items: [],┤
+    `;
+    await assertFilterResult(filter, context, completion, expected);
+  });
+
+  it("should keep content when no matches", async () => {
+    const context = documentContext`
+      function process() {
+        ║
+        console.log("done");          
+      }
+    `;
+    const completion = inline`
+      ├console.log("processing");┤
+    `;
+    const expected = completion;
+    await assertFilterResult(filter, context, completion, expected);
+  });
+});

clients/tabby-agent/src/codeCompletion/postprocess/removeDuplicateSuffixLines.ts

@@ -0,0 +1,50 @@
+import { PostprocessFilter } from "./base";
+import { CompletionItem } from "../solution";
+import { isBlank } from "../../utils/string";
+
+export function removeDuplicateSuffixLines(): PostprocessFilter {
+  return (item: CompletionItem): CompletionItem => {
+    const text = item?.text;
+    const suffix = item?.context?.suffix;
+
+    if (text == null || suffix == null) {
+      return item;
+    }
+
+    const originalLines = text.split("\n").map((line) => line || "");
+
+    const suffixLines = (suffix || "")
+      .split("\n")
+      .map((line) => (line || "").trim())
+      .filter((line) => !isBlank(line));
+
+    if (suffixLines.length === 0) {
+      return item;
+    }
+
+    const firstSuffixLine = suffixLines[0] || "";
+
+    // iterate through lines from end to find potential match
+    for (let i = originalLines.length - 1; i >= 0; i--) {
+      const currentLine = originalLines[i] || "";
+      if (!isBlank(currentLine) && currentLine === firstSuffixLine) {
+        // check if subsequent lines also match with suffix
+        let isFullMatch = true;
+        for (let j = 0; j < suffixLines.length && i + j < originalLines.length; j++) {
+          const suffixLine = suffixLines[j] || "";
+          const textLine = originalLines[i + j] || "";
+          if (suffixLine !== textLine) {
+            isFullMatch = false;
+            break;
+          }
+        }
+        if (isFullMatch) {
+          const remainingLines = originalLines.slice(0, i);
+          return item.withText(remainingLines.join("\n"));
+        }
+      }
+    }
+
+    return item;
+  };
+}