diff --git a/integrations/Gemfile.lock b/integrations/Gemfile.lock index ef94c94e..79d31293 100644 --- a/integrations/Gemfile.lock +++ b/integrations/Gemfile.lock @@ -7,7 +7,7 @@ GIT PATH remote: . specs: - multiwoven-integrations (0.8.2) + multiwoven-integrations (0.8.3) activesupport async-websocket aws-sdk-athena diff --git a/integrations/lib/multiwoven/integrations/core/constants.rb b/integrations/lib/multiwoven/integrations/core/constants.rb index 9ce33d04..7c8e5248 100644 --- a/integrations/lib/multiwoven/integrations/core/constants.rb +++ b/integrations/lib/multiwoven/integrations/core/constants.rb @@ -40,13 +40,12 @@ module Constants MS_EXCEL_TABLE_ROW_WRITE_API = "https://graph.microsoft.com/v1.0/drives/%s/items/%s/"\ "workbook/worksheets/%s/tables/%s/rows" MS_EXCEL_TABLE_API = "https://graph.microsoft.com/v1.0/drives/%s/items/%s/workbook/"\ - "worksheets/sheet/tables?$select=name" + "worksheets/%s/tables?$select=name" MS_EXCEL_FILES_API = "https://graph.microsoft.com/v1.0/drives/%s/root/children" MS_EXCEL_WORKSHEETS_API = "https://graph.microsoft.com/v1.0/drives/%s/items/%s/"\ "workbook/worksheets" MS_EXCEL_SHEET_RANGE_API = "https://graph.microsoft.com/v1.0/drives/%s/items/%s/"\ "workbook/worksheets/%s/range(address='A1:Z1')/usedRange?$select=values" - AWS_ACCESS_KEY_ID = ENV["AWS_ACCESS_KEY_ID"] AWS_SECRET_ACCESS_KEY = ENV["AWS_SECRET_ACCESS_KEY"] diff --git a/integrations/lib/multiwoven/integrations/destination/microsoft_excel/client.rb b/integrations/lib/multiwoven/integrations/destination/microsoft_excel/client.rb index 7e5805c2..cc120a68 100644 --- a/integrations/lib/multiwoven/integrations/destination/microsoft_excel/client.rb +++ b/integrations/lib/multiwoven/integrations/destination/microsoft_excel/client.rb @@ -49,8 +49,7 @@ def write(sync_config, records, _action = "destination_insert") excel_files = get_file(token, drive_id) worksheet = excel_files.find { |file| file[:name] == file_name } item_id = worksheet[:id] - - table = get_table(token, drive_id, item_id) + table = get_table(token, drive_id, item_id, sheet_name) write_url = format(MS_EXCEL_TABLE_ROW_WRITE_API, drive_id: drive_id, item_id: item_id, sheet_name: sheet_name, table_name: table["name"]) payload = { values: records.map(&:values) } @@ -69,8 +68,8 @@ def create_connection(connection_config) JSON.parse(response.body)["id"] end - def get_table(token, drive_id, item_id) - table_url = format(MS_EXCEL_TABLE_API, drive_id: drive_id, item_id: item_id) + def get_table(token, drive_id, item_id, sheet_name) + table_url = format(MS_EXCEL_TABLE_API, drive_id: drive_id, item_id: item_id, sheet_name: sheet_name) response = Multiwoven::Integrations::Core::HttpClient.request( table_url, HTTP_GET, @@ -114,9 +113,14 @@ def get_file_data(token, drive_id, item_id) headers: auth_headers(token) ) sheets_data = JSON.parse(sheet_response.body) + column_names = if sheets_data.key?("error") + ["Column A"] + else + sheets_data["values"].first + end result << { sheet_name: sheet_name, - column_names: sheets_data["values"].first + column_names: column_names } end result diff --git a/integrations/lib/multiwoven/integrations/destination/microsoft_excel/config/catalog.json b/integrations/lib/multiwoven/integrations/destination/microsoft_excel/config/catalog.json index 9a178849..ace9ffee 100644 --- a/integrations/lib/multiwoven/integrations/destination/microsoft_excel/config/catalog.json +++ b/integrations/lib/multiwoven/integrations/destination/microsoft_excel/config/catalog.json @@ -1,7 +1,7 @@ { "request_rate_limit": 6000, "request_rate_limit_unit": "minute", - "request_rate_concurrency": 10, + "request_rate_concurrency": 1, "streams": [] } diff --git a/integrations/lib/multiwoven/integrations/rollout.rb b/integrations/lib/multiwoven/integrations/rollout.rb index 9fc5944e..25b5219f 100644 --- a/integrations/lib/multiwoven/integrations/rollout.rb +++ b/integrations/lib/multiwoven/integrations/rollout.rb @@ -2,7 +2,7 @@ module Multiwoven module Integrations - VERSION = "0.8.2" + VERSION = "0.8.3" ENABLED_SOURCES = %w[ Snowflake diff --git a/integrations/spec/multiwoven/integrations/destination/microsoft_excel/client_spec.rb b/integrations/spec/multiwoven/integrations/destination/microsoft_excel/client_spec.rb index bf856fbd..46a5280d 100644 --- a/integrations/spec/multiwoven/integrations/destination/microsoft_excel/client_spec.rb +++ b/integrations/spec/multiwoven/integrations/destination/microsoft_excel/client_spec.rb @@ -36,7 +36,7 @@ sync_mode: "incremental", destination_sync_mode: "insert", stream: { - name: "test_table.xlsx", + name: "test_table.xlsx, sheet", action: "create", json_schema: {}, supported_sync_modes: %w[incremental], @@ -108,7 +108,7 @@ expect(catalog).to be_a(Multiwoven::Integrations::Protocol::Catalog) expect(catalog.streams.first.request_rate_limit).to eql(6000) expect(catalog.streams.first.request_rate_limit_unit).to eql("minute") - expect(catalog.streams.first.request_rate_concurrency).to eql(10) + expect(catalog.streams.first.request_rate_concurrency).to eql(1) expect(catalog.streams.count).to eql(1) expect(catalog.streams[0].supported_sync_modes).to eql(%w[incremental]) end @@ -144,7 +144,7 @@ ) stub_request(:post, "https://graph.microsoft.com/v1.0/drives/DRIVE1/items/file1_id/workbook/worksheets/"\ - "test_table.xlsx/tables/Table1/rows") + "sheet/tables/Table1/rows") .to_return(status: 201, body: successful_update_response_body, headers: {}) sync_config = Multiwoven::Integrations::Protocol::SyncConfig.from_json(sync_config_json.to_json) @@ -194,7 +194,7 @@ stub_request(:post, "https://graph.microsoft.com/v1.0/drives/DRIVE1/items/file1_id/workbook/worksheets/"\ - "test_table.xlsx/tables/Table1/rows") + "sheet/tables/Table1/rows") .to_return(status: 400, body: failed_update_response_body, headers: {}) sync_config = Multiwoven::Integrations::Protocol::SyncConfig.from_json(sync_config_json.to_json)