Fix bitbucket sync (#3198)

* increase number of sets returned to 100 * formatting * add changeset * update bitbucket storage test * update fetchJsonfiles to read more than 100 files * formatting * update bitbucket test * formatting * fix linting issues
tokens-studio · Nov 11, 2024 · 1af628f · 1af628f
1 parent fdfc7be
commit 1af628f
Show file tree

Hide file tree

Showing 3 changed files with 71 additions and 66 deletions.
diff --git a/.changeset/healthy-rabbits-fetch.md b/.changeset/healthy-rabbits-fetch.md
@@ -0,0 +1,5 @@
+---
+"@tokens-studio/figma-plugin": patch
+---
+
+fixed an issue with pulling token sets from Bitbucket when multi-file sync is enabled, wherein all the token sets were not being pulled.
diff --git a/packages/tokens-studio-for-figma/src/storage/BitbucketTokenStorage.ts b/packages/tokens-studio-for-figma/src/storage/BitbucketTokenStorage.ts
@@ -1,6 +1,3 @@
-/* eslint-disable no-else-return */
-/* eslint-disable @typescript-eslint/indent */
-/* eslint "@typescript-eslint/no-unused-vars": off */
 import { Bitbucket, Schema } from 'bitbucket';
 import compact from 'just-compact';
 import {
@@ -145,31 +142,40 @@ export class BitbucketTokenStorage extends GitTokenStorage {
    */
 
   private async fetchJsonFilesFromDirectory(url: string): Promise<FetchJsonResult> {
-    const response = await fetch(url, {
-      headers: {
-        Authorization: `Basic ${btoa(`${this.username}:${this.secret}`)}`,
-      },
-      cache: 'no-cache',
+    let allJsonFiles: any[] = [];
+    let nextPageUrl: string | null = `${url}?pagelen=100`;
+
+    while (nextPageUrl) {
+      const response = await fetch(nextPageUrl, {
+        headers: {
+          Authorization: `Basic ${btoa(`${this.username}:${this.secret}`)}`,
+        },
+        cache: 'no-cache',
     });
 
-    if (!response.ok) {
-      throw new Error(`Failed to read from Bitbucket: ${response.statusText}`);
-    }
+      if (!response.ok) {
+        throw new Error(`Failed to read from Bitbucket: ${response.statusText}`);
+      }
 
-    const data = await response.json();
-    if (data.values && Array.isArray(data.values)) {
-      let jsonFiles = data.values.filter((file: any) => file.path.endsWith('.json'));
+      const data = await response.json();
+      if (data.values && Array.isArray(data.values)) {
+        const jsonFiles = data.values.filter((file: any) => file.path.endsWith('.json'));
+        allJsonFiles = allJsonFiles.concat(jsonFiles);
 
-      const subDirectoryFiles = await Promise.all(
-        data.values
-          .filter((file: any) => file.type === 'commit_directory')
-          .map(async (directory: any) => await this.fetchJsonFilesFromDirectory(directory.links.self.href)),
-      );
+        // Fetch files from subdirectories recursively
+        const subDirectoryFiles = await Promise.all(
+          data.values
+            .filter((file: any) => file.type === 'commit_directory')
+            .map(async (directory: any) => await this.fetchJsonFilesFromDirectory(directory.links.self.href)),
+        );
 
-      jsonFiles = jsonFiles.concat(...subDirectoryFiles);
-      return jsonFiles;
-        }
-      return data;
+        allJsonFiles = allJsonFiles.concat(...subDirectoryFiles);
+      }
+
+      nextPageUrl = data.next || null;
+    }
+
+    return allJsonFiles;
   }
 
   public async read(): Promise<RemoteTokenStorageFile[] | RemoteTokenstorageErrorMessage> {
@@ -180,14 +186,14 @@ export class BitbucketTokenStorage extends GitTokenStorage {
       const jsonFiles = await this.fetchJsonFilesFromDirectory(url);
 
       if (Array.isArray(jsonFiles)) {
-      const jsonFileContents = await Promise.all(
-        jsonFiles.map((file: any) => fetch(file.links.self.href, {
+        const jsonFileContents = await Promise.all(
+          jsonFiles.map((file: any) => fetch(file.links.self.href, {
             headers: {
               Authorization: `Basic ${btoa(`${this.username}:${this.secret}`)}`,
             },
             cache: 'no-cache',
           }).then((rsp) => rsp.text())),
-      );
+        );
         // Process the content of each JSON file
         return jsonFileContents.map((fileContent, index) => {
           const { path } = jsonFiles[index];
@@ -219,7 +225,7 @@ export class BitbucketTokenStorage extends GitTokenStorage {
             data: parsed as AnyTokenSet<false>,
           };
         });
-      } else if (jsonFiles) {
+      } if (jsonFiles) {
         const parsed = jsonFiles as GitSingleFileObject;
         return [
           {
@@ -229,12 +235,12 @@ export class BitbucketTokenStorage extends GitTokenStorage {
           },
           ...(parsed.$metadata
             ? [
-                {
-                  type: 'metadata' as const,
-                  path: this.path,
-                  data: parsed.$metadata,
-                },
-              ]
+              {
+                type: 'metadata' as const,
+                path: this.path,
+                data: parsed.$metadata,
+              },
+            ]
             : []),
           ...(
             Object.entries(parsed).filter(([key]) => !Object.values<string>(SystemFilenames).includes(key)) as [
@@ -290,8 +296,8 @@ export class BitbucketTokenStorage extends GitTokenStorage {
    * const response = await createOrUpdateFiles(params);
    */
   public async createOrUpdateFiles({
- owner, repo, branch, changes,
-}: CreatedOrUpdatedFileType) {
+    owner, repo, branch, changes,
+  }: CreatedOrUpdatedFileType) {
     const { message, files } = changes[0];
 
     const data = new FormData();

diff --git a/packages/tokens-studio-for-figma/src/storage/__tests__/BitbucketTokenStorage.test.ts b/packages/tokens-studio-for-figma/src/storage/__tests__/BitbucketTokenStorage.test.ts
@@ -113,56 +113,50 @@ describe('BitbucketTokenStorage', () => {
   });
 
   it('can read from Git in single file format', async () => {
-    mockFetch.mockImplementationOnce(() => Promise.resolve({
-      ok: true,
-      json: () => Promise.resolve({ global: { red: { name: 'red', type: 'color', value: '#ff0000' } } }),
-    }));
+    mockFetch
+      .mockImplementationOnce(() => Promise.resolve({
+        ok: true,
+        json: () => Promise.resolve({
+          values: [
+            { path: '$themes.json', type: 'commit_file', links: { self: { href: 'https://api.bitbucket.org/file/$themes.json' } } },
+            { path: 'global.json', type: 'commit_file', links: { self: { href: 'https://api.bitbucket.org/file/global.json' } } },
+          ],
+          next: null, // No further pagination needed for this test
+        }),
+      }))
+      .mockImplementationOnce(() => Promise.resolve({
+        ok: true,
+        text: () => Promise.resolve(JSON.stringify({ $themes: [] })),
+      }))
+      .mockImplementationOnce(() => Promise.resolve({
+        ok: true,
+        text: () => Promise.resolve(JSON.stringify({ red: { name: 'red', type: 'color', value: '#ff0000' } })),
+      }));
 
     const result = await storageProvider.read();
-    expect(result).toEqual([
-      {
-        path: '/$themes.json',
-        type: 'themes',
-        data: [],
-      },
-      {
-        path: '/global.json',
-        name: 'global',
-        type: 'tokenSet',
-        data: {
-          red: {
-            name: 'red',
-            type: 'color',
-            value: '#ff0000',
-          },
-        },
-      },
-    ]);
-
-    storageProvider.changePath('data/core.json');
 
     expect(result).toEqual([
       {
-        path: '/$themes.json',
+        path: '$themes.json',
         type: 'themes',
-        data: [],
+        data: { $themes: [] },
       },
       {
-        path: '/global.json',
+        path: 'global.json',
         name: 'global',
         type: 'tokenSet',
         data: {
           red: {
-            type: 'color',
             name: 'red',
+            type: 'color',
             value: '#ff0000',
           },
         },
       },
     ]);
 
-    expect(mockFetch).toBeCalledWith(
-      `https://api.bitbucket.org/2.0/repositories/${storageProvider.owner}/${storageProvider.repository}/src/${storageProvider.branch}/`,
+    expect(mockFetch).toHaveBeenCalledWith(
+      `https://api.bitbucket.org/2.0/repositories/${storageProvider.owner}/${storageProvider.repository}/src/${storageProvider.branch}/?pagelen=100`,
       {
         headers: {
           Authorization: `Basic ${btoa('myusername:mock-secret')}`,