fix(CE): fix discover and table url (#316)

Co-authored-by: TivonB-AI2 <[email protected]>
Multiwoven · Aug 19, 2024 · 467b8bf · 467b8bf
1 parent 28cc8dc
commit 467b8bf
Show file tree

Hide file tree

Showing 6 changed files with 17 additions and 14 deletions.
diff --git a/integrations/Gemfile.lock b/integrations/Gemfile.lock
@@ -7,7 +7,7 @@ GIT
 PATH
   remote: .
   specs:
-    multiwoven-integrations (0.8.2)
+    multiwoven-integrations (0.8.3)
       activesupport
       async-websocket
       aws-sdk-athena

diff --git a/integrations/lib/multiwoven/integrations/core/constants.rb b/integrations/lib/multiwoven/integrations/core/constants.rb
@@ -40,13 +40,12 @@ module Constants
       MS_EXCEL_TABLE_ROW_WRITE_API = "https://graph.microsoft.com/v1.0/drives/%<drive_id>s/items/%<item_id>s/"\
       "workbook/worksheets/%<sheet_name>s/tables/%<table_name>s/rows"
       MS_EXCEL_TABLE_API = "https://graph.microsoft.com/v1.0/drives/%<drive_id>s/items/%<item_id>s/workbook/"\
-      "worksheets/sheet/tables?$select=name"
+      "worksheets/%<sheet_name>s/tables?$select=name"
       MS_EXCEL_FILES_API = "https://graph.microsoft.com/v1.0/drives/%<drive_id>s/root/children"
       MS_EXCEL_WORKSHEETS_API = "https://graph.microsoft.com/v1.0/drives/%<drive_id>s/items/%<item_id>s/"\
       "workbook/worksheets"
       MS_EXCEL_SHEET_RANGE_API = "https://graph.microsoft.com/v1.0/drives/%<drive_id>s/items/%<item_id>s/"\
       "workbook/worksheets/%<sheet_name>s/range(address='A1:Z1')/usedRange?$select=values"
-
       AWS_ACCESS_KEY_ID = ENV["AWS_ACCESS_KEY_ID"]
       AWS_SECRET_ACCESS_KEY = ENV["AWS_SECRET_ACCESS_KEY"]
 

diff --git a/integrations/lib/multiwoven/integrations/destination/microsoft_excel/client.rb b/integrations/lib/multiwoven/integrations/destination/microsoft_excel/client.rb
@@ -49,8 +49,7 @@ def write(sync_config, records, _action = "destination_insert")
         excel_files = get_file(token, drive_id)
         worksheet = excel_files.find { |file| file[:name] == file_name }
         item_id = worksheet[:id]
-
-        table = get_table(token, drive_id, item_id)
+        table = get_table(token, drive_id, item_id, sheet_name)
         write_url = format(MS_EXCEL_TABLE_ROW_WRITE_API, drive_id: drive_id, item_id: item_id, sheet_name: sheet_name,
                                                          table_name: table["name"])
         payload = { values: records.map(&:values) }
@@ -69,8 +68,8 @@ def create_connection(connection_config)
         JSON.parse(response.body)["id"]
       end
 
-      def get_table(token, drive_id, item_id)
-        table_url = format(MS_EXCEL_TABLE_API, drive_id: drive_id, item_id: item_id)
+      def get_table(token, drive_id, item_id, sheet_name)
+        table_url = format(MS_EXCEL_TABLE_API, drive_id: drive_id, item_id: item_id, sheet_name: sheet_name)
         response = Multiwoven::Integrations::Core::HttpClient.request(
           table_url,
           HTTP_GET,
@@ -114,9 +113,14 @@ def get_file_data(token, drive_id, item_id)
             headers: auth_headers(token)
           )
           sheets_data = JSON.parse(sheet_response.body)
+          column_names = if sheets_data.key?("error")
+                           ["Column A"]
+                         else
+                           sheets_data["values"].first
+                         end
           result << {
             sheet_name: sheet_name,
-            column_names: sheets_data["values"].first
+            column_names: column_names
           }
         end
         result

diff --git a/integrations/lib/multiwoven/integrations/destination/microsoft_excel/config/catalog.json b/integrations/lib/multiwoven/integrations/destination/microsoft_excel/config/catalog.json
@@ -1,7 +1,7 @@
 {
   "request_rate_limit": 6000,
   "request_rate_limit_unit": "minute",
-  "request_rate_concurrency": 10,
+  "request_rate_concurrency": 1,
   "streams": []
 }
 
diff --git a/integrations/lib/multiwoven/integrations/rollout.rb b/integrations/lib/multiwoven/integrations/rollout.rb
@@ -2,7 +2,7 @@
 
 module Multiwoven
   module Integrations
-    VERSION = "0.8.2"
+    VERSION = "0.8.3"
 
     ENABLED_SOURCES = %w[
       Snowflake

diff --git a/integrations/spec/multiwoven/integrations/destination/microsoft_excel/client_spec.rb b/integrations/spec/multiwoven/integrations/destination/microsoft_excel/client_spec.rb
@@ -36,7 +36,7 @@
       sync_mode: "incremental",
       destination_sync_mode: "insert",
       stream: {
-        name: "test_table.xlsx",
+        name: "test_table.xlsx, sheet",
         action: "create",
         json_schema: {},
         supported_sync_modes: %w[incremental],
@@ -108,7 +108,7 @@
       expect(catalog).to be_a(Multiwoven::Integrations::Protocol::Catalog)
       expect(catalog.streams.first.request_rate_limit).to eql(6000)
       expect(catalog.streams.first.request_rate_limit_unit).to eql("minute")
-      expect(catalog.streams.first.request_rate_concurrency).to eql(10)
+      expect(catalog.streams.first.request_rate_concurrency).to eql(1)
       expect(catalog.streams.count).to eql(1)
       expect(catalog.streams[0].supported_sync_modes).to eql(%w[incremental])
     end
@@ -144,7 +144,7 @@
           )
 
         stub_request(:post, "https://graph.microsoft.com/v1.0/drives/DRIVE1/items/file1_id/workbook/worksheets/"\
-        "test_table.xlsx/tables/Table1/rows")
+        "sheet/tables/Table1/rows")
           .to_return(status: 201, body: successful_update_response_body, headers: {})
 
         sync_config = Multiwoven::Integrations::Protocol::SyncConfig.from_json(sync_config_json.to_json)
@@ -194,7 +194,7 @@
 
         stub_request(:post,
                      "https://graph.microsoft.com/v1.0/drives/DRIVE1/items/file1_id/workbook/worksheets/"\
-                     "test_table.xlsx/tables/Table1/rows")
+                     "sheet/tables/Table1/rows")
           .to_return(status: 400, body: failed_update_response_body, headers: {})
 
         sync_config = Multiwoven::Integrations::Protocol::SyncConfig.from_json(sync_config_json.to_json)