diff --git a/integrations/Gemfile b/integrations/Gemfile index 1b82fbae..757d6502 100644 --- a/integrations/Gemfile +++ b/integrations/Gemfile @@ -67,7 +67,7 @@ gem "mysql2" gem "aws-sdk-sts" -gem "ruby-oci8" +gem "ruby-oci8", "~> 2.2.12" group :development, :test do gem "simplecov", require: false diff --git a/integrations/Gemfile.lock b/integrations/Gemfile.lock index 71ed2acb..eaf02de0 100644 --- a/integrations/Gemfile.lock +++ b/integrations/Gemfile.lock @@ -7,7 +7,7 @@ GIT PATH remote: . specs: - multiwoven-integrations (0.7.9) + multiwoven-integrations (0.8.1) activesupport async-websocket aws-sdk-athena @@ -360,7 +360,7 @@ DEPENDENCIES rspec (~> 3.0) rubocop (~> 1.21) ruby-limiter - ruby-oci8 + ruby-oci8 (~> 2.2.12) ruby-odbc! rubyzip sequel diff --git a/integrations/lib/multiwoven/integrations.rb b/integrations/lib/multiwoven/integrations.rb index 4f80fab4..fbcc5537 100644 --- a/integrations/lib/multiwoven/integrations.rb +++ b/integrations/lib/multiwoven/integrations.rb @@ -81,6 +81,7 @@ require_relative "integrations/destination/maria_db/client" require_relative "integrations/destination/databricks_lakehouse/client" require_relative "integrations/destination/oracle_db/client" +require_relative "integrations/destination/microsoft_excel/client" module Multiwoven module Integrations diff --git a/integrations/lib/multiwoven/integrations/core/base_connector.rb b/integrations/lib/multiwoven/integrations/core/base_connector.rb index b594f517..672e474c 100644 --- a/integrations/lib/multiwoven/integrations/core/base_connector.rb +++ b/integrations/lib/multiwoven/integrations/core/base_connector.rb @@ -63,7 +63,8 @@ def success_status end def failure_status(error) - ConnectionStatus.new(status: ConnectionStatusType["failed"], message: error.message).to_multiwoven_message + message = error&.message || "failed" + ConnectionStatus.new(status: ConnectionStatusType["failed"], message: message).to_multiwoven_message end end end diff --git a/integrations/lib/multiwoven/integrations/core/constants.rb b/integrations/lib/multiwoven/integrations/core/constants.rb index 45b106f0..9ce33d04 100644 --- a/integrations/lib/multiwoven/integrations/core/constants.rb +++ b/integrations/lib/multiwoven/integrations/core/constants.rb @@ -36,6 +36,17 @@ module Constants AIRTABLE_BASES_ENDPOINT = "https://api.airtable.com/v0/meta/bases" AIRTABLE_GET_BASE_SCHEMA_ENDPOINT = "https://api.airtable.com/v0/meta/bases/{baseId}/tables" + MS_EXCEL_AUTH_ENDPOINT = "https://graph.microsoft.com/v1.0/me" + MS_EXCEL_TABLE_ROW_WRITE_API = "https://graph.microsoft.com/v1.0/drives/%s/items/%s/"\ + "workbook/worksheets/%s/tables/%s/rows" + MS_EXCEL_TABLE_API = "https://graph.microsoft.com/v1.0/drives/%s/items/%s/workbook/"\ + "worksheets/sheet/tables?$select=name" + MS_EXCEL_FILES_API = "https://graph.microsoft.com/v1.0/drives/%s/root/children" + MS_EXCEL_WORKSHEETS_API = "https://graph.microsoft.com/v1.0/drives/%s/items/%s/"\ + "workbook/worksheets" + MS_EXCEL_SHEET_RANGE_API = "https://graph.microsoft.com/v1.0/drives/%s/items/%s/"\ + "workbook/worksheets/%s/range(address='A1:Z1')/usedRange?$select=values" + AWS_ACCESS_KEY_ID = ENV["AWS_ACCESS_KEY_ID"] AWS_SECRET_ACCESS_KEY = ENV["AWS_SECRET_ACCESS_KEY"] @@ -44,6 +55,7 @@ module Constants HTTP_POST = "POST" HTTP_PUT = "PUT" HTTP_DELETE = "DELETE" + HTTP_PATCH = "PATCH" # google sheets GOOGLE_SHEETS_SCOPE = "https://www.googleapis.com/auth/drive" diff --git a/integrations/lib/multiwoven/integrations/core/destination_connector.rb b/integrations/lib/multiwoven/integrations/core/destination_connector.rb index f0595a31..da327412 100644 --- a/integrations/lib/multiwoven/integrations/core/destination_connector.rb +++ b/integrations/lib/multiwoven/integrations/core/destination_connector.rb @@ -15,6 +15,14 @@ def tracking_message(success, failure, log_message_array) success: success, failed: failure, logs: log_message_array ).to_multiwoven_message end + + def auth_headers(access_token) + { + "Accept" => "application/json", + "Authorization" => "Bearer #{access_token}", + "Content-Type" => "application/json" + } + end end end end diff --git a/integrations/lib/multiwoven/integrations/core/http_client.rb b/integrations/lib/multiwoven/integrations/core/http_client.rb index c0aa40c9..b119c256 100644 --- a/integrations/lib/multiwoven/integrations/core/http_client.rb +++ b/integrations/lib/multiwoven/integrations/core/http_client.rb @@ -19,13 +19,14 @@ def build_request(method, uri, payload, headers) when Constants::HTTP_GET then Net::HTTP::Get when Constants::HTTP_POST then Net::HTTP::Post when Constants::HTTP_PUT then Net::HTTP::Put + when Constants::HTTP_PATCH then Net::HTTP::Patch when Constants::HTTP_DELETE then Net::HTTP::Delete else raise ArgumentError, "Unsupported HTTP method: #{method}" end request = request_class.new(uri) headers.each { |key, value| request[key] = value } - request.body = payload.to_json if payload && %w[POST PUT].include?(method.upcase) + request.body = payload.to_json if payload && %w[POST PUT PATCH].include?(method.upcase) request end end diff --git a/integrations/lib/multiwoven/integrations/destination/airtable/client.rb b/integrations/lib/multiwoven/integrations/destination/airtable/client.rb index 8d89ff88..d2242fbe 100644 --- a/integrations/lib/multiwoven/integrations/destination/airtable/client.rb +++ b/integrations/lib/multiwoven/integrations/destination/airtable/client.rb @@ -109,14 +109,6 @@ def create_payload(records) } end - def auth_headers(access_token) - { - "Accept" => "application/json", - "Authorization" => "Bearer #{access_token}", - "Content-Type" => "application/json" - } - end - def base_id_exists?(bases, base_id) return if extract_bases(bases).any? { |base| base["id"] == base_id } diff --git a/integrations/lib/multiwoven/integrations/destination/facebook_custom_audience/client.rb b/integrations/lib/multiwoven/integrations/destination/facebook_custom_audience/client.rb index 685cb54f..e2a69e1c 100644 --- a/integrations/lib/multiwoven/integrations/destination/facebook_custom_audience/client.rb +++ b/integrations/lib/multiwoven/integrations/destination/facebook_custom_audience/client.rb @@ -110,14 +110,6 @@ def extract_schema_and_data(records, json_schema) [schema, data] end - def auth_headers(access_token) - { - "Accept" => "application/json", - "Authorization" => "Bearer #{access_token}", - "Content-Type" => "application/json" - } - end - def ad_account_exists?(response, ad_account_id) return if extract_data(response).any? { |ad_account| ad_account["id"] == "act_#{ad_account_id}" } diff --git a/integrations/lib/multiwoven/integrations/destination/microsoft_excel/client.rb b/integrations/lib/multiwoven/integrations/destination/microsoft_excel/client.rb new file mode 100644 index 00000000..7e5805c2 --- /dev/null +++ b/integrations/lib/multiwoven/integrations/destination/microsoft_excel/client.rb @@ -0,0 +1,194 @@ +# frozen_string_literal: true + +module Multiwoven::Integrations::Destination + module MicrosoftExcel + include Multiwoven::Integrations::Core + class Client < DestinationConnector + prepend Multiwoven::Integrations::Core::RateLimiter + def check_connection(connection_config) + connection_config = connection_config.with_indifferent_access + drive_id = create_connection(connection_config) + if drive_id + success_status + else + failure_status(nil) + end + rescue StandardError => e + handle_exception(e, { + context: "MICROSOFT:EXCEL:CHECK_CONNECTION:EXCEPTION", + type: "error" + }) + failure_status(e) + end + + def discover(connection_config) + catalog_json = read_json(CATALOG_SPEC_PATH) + connection_config = connection_config.with_indifferent_access + token = connection_config[:token] + drive_id = create_connection(connection_config) + records = get_file(token, drive_id) + records.each do |record| + file_id = record[:id] + record[:worksheets] = get_file_data(token, drive_id, file_id) + end + catalog = Catalog.new(streams: create_streams(records, catalog_json)) + catalog.to_multiwoven_message + rescue StandardError => e + handle_exception(e, { + context: "MICROSOFT:EXCEL:DISCOVER:EXCEPTION", + type: "error" + }) + end + + def write(sync_config, records, _action = "destination_insert") + connection_config = sync_config.destination.connection_specification.with_indifferent_access + token = connection_config[:token] + file_name = sync_config.stream.name.split(", ").first + sheet_name = sync_config.stream.name.split(", ").last + drive_id = create_connection(connection_config) + excel_files = get_file(token, drive_id) + worksheet = excel_files.find { |file| file[:name] == file_name } + item_id = worksheet[:id] + + table = get_table(token, drive_id, item_id) + write_url = format(MS_EXCEL_TABLE_ROW_WRITE_API, drive_id: drive_id, item_id: item_id, sheet_name: sheet_name, + table_name: table["name"]) + payload = { values: records.map(&:values) } + process_write_request(write_url, payload, token, sync_config) + end + + private + + def create_connection(connection_config) + token = connection_config[:token] + response = Multiwoven::Integrations::Core::HttpClient.request( + MS_EXCEL_AUTH_ENDPOINT, + HTTP_GET, + headers: auth_headers(token) + ) + JSON.parse(response.body)["id"] + end + + def get_table(token, drive_id, item_id) + table_url = format(MS_EXCEL_TABLE_API, drive_id: drive_id, item_id: item_id) + response = Multiwoven::Integrations::Core::HttpClient.request( + table_url, + HTTP_GET, + headers: auth_headers(token) + ) + JSON.parse(response.body)["value"].first + end + + def get_file(token, drive_id) + url = format(MS_EXCEL_FILES_API, drive_id: drive_id) + response = Multiwoven::Integrations::Core::HttpClient.request( + url, + HTTP_GET, + headers: auth_headers(token) + ) + files = JSON.parse(response.body)["value"] + excel_files = files.select { |file| file["name"].match(/\.(xlsx|xls|xlsm)$/) } + excel_files.map { |file| { name: file["name"], id: file["id"] } } + end + + def get_all_sheets(token, drive_id, item_id) + base_url = format(MS_EXCEL_WORKSHEETS_API, drive_id: drive_id, item_id: item_id) + worksheet_response = Multiwoven::Integrations::Core::HttpClient.request( + base_url, + HTTP_GET, + headers: auth_headers(token) + ) + JSON.parse(worksheet_response.body)["value"] + end + + def get_file_data(token, drive_id, item_id) + result = [] + worksheets_data = get_all_sheets(token, drive_id, item_id) + worksheets_data.each do |sheet| + sheet_name = sheet["name"] + sheet_url = format(MS_EXCEL_SHEET_RANGE_API, drive_id: drive_id, item_id: item_id, sheet_name: sheet_name) + + sheet_response = Multiwoven::Integrations::Core::HttpClient.request( + sheet_url, + HTTP_GET, + headers: auth_headers(token) + ) + sheets_data = JSON.parse(sheet_response.body) + result << { + sheet_name: sheet_name, + column_names: sheets_data["values"].first + } + end + result + end + + def create_streams(records, catalog_json) + group_by_table(records).flat_map do |_, record| + record.map do |_, r| + Multiwoven::Integrations::Protocol::Stream.new( + name: r[:workbook], + action: StreamAction["fetch"], + json_schema: convert_to_json_schema(r[:columns]), + request_rate_limit: catalog_json["request_rate_limit"] || 60, + request_rate_limit_unit: catalog_json["request_rate_limit_unit"] || "minute", + request_rate_concurrency: catalog_json["request_rate_concurrency"] || 1 + ) + end + end + end + + def group_by_table(records) + result = {} + + records.each_with_index do |entries, entries_index| + entries[:worksheets].each_with_index do |sheet, entry_index| + workbook_sheet = "#{entries[:name]}, #{sheet[:sheet_name]}" + columns = sheet[:column_names].map do |column_name| + column_name = "empty column" if column_name.empty? + { + column_name: column_name, + data_type: "String", + is_nullable: true + } + end + result[entries_index] ||= {} + result[entries_index][entry_index] = { workbook: workbook_sheet, columns: columns } + end + end + result + end + + def process_write_request(write_url, payload, token, sync_config) + write_success = 0 + write_failure = 0 + log_message_array = [] + + begin + response = Multiwoven::Integrations::Core::HttpClient.request( + write_url, + HTTP_POST, + payload: payload, + headers: auth_headers(token) + ) + if success?(response) + write_success += 1 + else + write_failure += 1 + end + log_message_array << log_request_response("info", [HTTP_POST, write_url, payload], response) + rescue StandardError => e + handle_exception(e, { + context: "MICROSOFT:EXCEL:RECORD:WRITE:EXCEPTION", + type: "error", + sync_id: sync_config.sync_id, + sync_run_id: sync_config.sync_run_id + }) + write_failure += 1 + log_message_array << log_request_response("error", [HTTP_POST, write_url, payload], e.message) + end + + tracking_message(write_success, write_failure, log_message_array) + end + end + end +end diff --git a/integrations/lib/multiwoven/integrations/destination/microsoft_excel/config/catalog.json b/integrations/lib/multiwoven/integrations/destination/microsoft_excel/config/catalog.json new file mode 100644 index 00000000..9a178849 --- /dev/null +++ b/integrations/lib/multiwoven/integrations/destination/microsoft_excel/config/catalog.json @@ -0,0 +1,7 @@ +{ + "request_rate_limit": 6000, + "request_rate_limit_unit": "minute", + "request_rate_concurrency": 10, + "streams": [] +} + diff --git a/integrations/lib/multiwoven/integrations/destination/microsoft_excel/config/meta.json b/integrations/lib/multiwoven/integrations/destination/microsoft_excel/config/meta.json new file mode 100644 index 00000000..49455eac --- /dev/null +++ b/integrations/lib/multiwoven/integrations/destination/microsoft_excel/config/meta.json @@ -0,0 +1,15 @@ +{ + "data": { + "name": "MicrosoftExcel", + "title": "Microsoft Excel", + "connector_type": "destination", + "category": "Database", + "documentation_url": "https://docs.squared.ai/guides/data-integration/destination/microsoft_excel", + "github_issue_label": "destination-microsoft-excel", + "icon": "icon.svg", + "license": "MIT", + "release_stage": "alpha", + "support_level": "community", + "tags": ["language:ruby", "multiwoven"] + } +} diff --git a/integrations/lib/multiwoven/integrations/destination/microsoft_excel/config/spec.json b/integrations/lib/multiwoven/integrations/destination/microsoft_excel/config/spec.json new file mode 100644 index 00000000..d3fe4a99 --- /dev/null +++ b/integrations/lib/multiwoven/integrations/destination/microsoft_excel/config/spec.json @@ -0,0 +1,19 @@ +{ + "documentation_url": "https://docs.squared.ai/guides/data-integration/destination/microsoft_excel", + "stream_type": "dynamic", + "connector_query_type": "raw_sql", + "connection_specification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Microsoft Excel", + "type": "object", + "required": ["token"], + "properties": { + "token": { + "description": "Token from Microsoft Graph.", + "type": "string", + "title": "Token", + "order": 0 + } + } + } +} \ No newline at end of file diff --git a/integrations/lib/multiwoven/integrations/destination/microsoft_excel/icon.svg b/integrations/lib/multiwoven/integrations/destination/microsoft_excel/icon.svg new file mode 100644 index 00000000..3ec1e490 --- /dev/null +++ b/integrations/lib/multiwoven/integrations/destination/microsoft_excel/icon.svg @@ -0,0 +1,18 @@ + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/integrations/lib/multiwoven/integrations/rollout.rb b/integrations/lib/multiwoven/integrations/rollout.rb index 62c6d503..7c91233c 100644 --- a/integrations/lib/multiwoven/integrations/rollout.rb +++ b/integrations/lib/multiwoven/integrations/rollout.rb @@ -2,7 +2,7 @@ module Multiwoven module Integrations - VERSION = "0.7.9" + VERSION = "0.8.1" ENABLED_SOURCES = %w[ Snowflake @@ -36,6 +36,7 @@ module Integrations MariaDB DatabricksLakehouse Oracle + MicrosoftExcel ].freeze end end diff --git a/integrations/spec/multiwoven/integrations/core/http_client_spec.rb b/integrations/spec/multiwoven/integrations/core/http_client_spec.rb index e90e7847..b2d2d328 100644 --- a/integrations/spec/multiwoven/integrations/core/http_client_spec.rb +++ b/integrations/spec/multiwoven/integrations/core/http_client_spec.rb @@ -39,6 +39,13 @@ module Integrations::Core end end + context "when making a PATCH request" do + it "creates a PATCH request" do + described_class.request(url, "PATCH", headers: headers) + expect(a_request(:patch, url).with(headers: headers)).to have_been_made.once + end + end + context "with an unsupported HTTP method" do it "raises an ArgumentError" do expect { described_class.request(url, "INVALID", headers: headers) }.to raise_error(ArgumentError) diff --git a/integrations/spec/multiwoven/integrations/destination/microsoft_excel/client_spec.rb b/integrations/spec/multiwoven/integrations/destination/microsoft_excel/client_spec.rb new file mode 100644 index 00000000..bf856fbd --- /dev/null +++ b/integrations/spec/multiwoven/integrations/destination/microsoft_excel/client_spec.rb @@ -0,0 +1,224 @@ +# frozen_string_literal: true + +RSpec.describe Multiwoven::Integrations::Destination::MicrosoftExcel::Client do + include WebMock::API + + before(:each) do + WebMock.disable_net_connect!(allow_localhost: true) + end + + let(:client) { described_class.new } + let(:connection_config) do + { + token: "test" + } + end + let(:sync_config_json) do + { + source: { + name: "Sample Source Connector", + type: "source", + connection_specification: { + private_api_key: "test_api_key" + } + }, + destination: { + name: "Databricks", + type: "destination", + connection_specification: connection_config + }, + model: { + name: "ExampleModel", + query: "SELECT col1, col2, col3 FROM test_table_1", + query_type: "raw_sql", + primary_key: "col1" + }, + sync_mode: "incremental", + destination_sync_mode: "insert", + stream: { + name: "test_table.xlsx", + action: "create", + json_schema: {}, + supported_sync_modes: %w[incremental], + request_rate_limit: 4, + rate_limit_unit_seconds: 1 + } + } + end + + let(:response_body) { { "id" => "DRIVE1" }.to_json } + let(:successful_update_response_body) { { "values" => [["400", "4.4", "Fourth"]] }.to_json } + let(:failed_update_response_body) { { "values" => [["400", "4.4", "Fourth"]] }.to_json } + + describe "#check_connection" do + context "when the connection is successful" do + it "returns a succeeded connection status" do + stub_request(:get, "https://graph.microsoft.com/v1.0/me") + .to_return(status: 200, body: response_body, headers: {}) + + allow(client).to receive(:create_connection).and_return("DRIVE1") + + message = client.check_connection(sync_config_json[:destination][:connection_specification]) + result = message.connection_status + expect(result.status).to eq("succeeded") + expect(result.message).to be_nil + end + end + + context "when the connection fails" do + it "returns a failed connection status with an error message" do + allow(client).to receive(:create_connection).and_raise(StandardError, "Connection failed") + message = client.check_connection(sync_config_json[:destination][:connection_specification]) + result = message.connection_status + expect(result.status).to eq("failed") + expect(result.message).to include("Connection failed") + end + end + end + + describe "#discover" do + it "discovers schema successfully" do + stub_request(:get, "https://graph.microsoft.com/v1.0/me") + .to_return(status: 200, body: response_body, headers: {}) + stub_request(:get, "https://graph.microsoft.com/v1.0/drives/DRIVE1/root/children") + .to_return( + status: 200, + body: { + "value" => [ + { "id" => "file1_id", "name" => "test_file.xlsx" } + ] + }.to_json, + headers: {} + ) + allow(client).to receive(:get_all_sheets).and_return([ + { "name" => "Sheet1" } + ]) + stub_request(:get, "https://graph.microsoft.com/v1.0/drives/DRIVE1/items/file1_id/workbook/worksheets/Sheet1/"\ + "range(address='A1:Z1')/usedRange?$select=values") + .to_return( + status: 200, + body: { + "values" => [%w[col1 col2 col3]] + }.to_json, + headers: {} + ) + + message = client.discover(connection_config) + catalog = message.catalog + expect(catalog).to be_a(Multiwoven::Integrations::Protocol::Catalog) + expect(catalog.streams.first.request_rate_limit).to eql(6000) + expect(catalog.streams.first.request_rate_limit_unit).to eql("minute") + expect(catalog.streams.first.request_rate_concurrency).to eql(10) + expect(catalog.streams.count).to eql(1) + expect(catalog.streams[0].supported_sync_modes).to eql(%w[incremental]) + end + end + + describe "#write" do + context "when the write operation is successful" do + it "increments the success count" do + stub_request(:get, "https://graph.microsoft.com/v1.0/me") + .to_return(status: 200, body: response_body, headers: {}) + + stub_request(:get, "https://graph.microsoft.com/v1.0/drives/DRIVE1/root/children") + .to_return( + status: 200, + body: { + "value" => [ + { "id" => "file1_id", "name" => "test_table.xlsx" } + ] + }.to_json, + headers: {} + ) + + stub_request(:get, "https://graph.microsoft.com/v1.0/drives/DRIVE1/items/file1_id/workbook/worksheets/sheet/"\ + "tables?$select=name") + .to_return( + status: 200, + body: { + "value" => [ + { "name" => "Table1" } + ] + }.to_json, + headers: {} + ) + + stub_request(:post, "https://graph.microsoft.com/v1.0/drives/DRIVE1/items/file1_id/workbook/worksheets/"\ + "test_table.xlsx/tables/Table1/rows") + .to_return(status: 201, body: successful_update_response_body, headers: {}) + + sync_config = Multiwoven::Integrations::Protocol::SyncConfig.from_json(sync_config_json.to_json) + records = [{ "Col1" => 400, "Col2" => 4.4, "Col3" => "Fourth" }] + + message = client.write(sync_config, records) + tracker = message.tracking + + expect(tracker.success).to eq(records.count) + expect(tracker.failed).to eq(0) + log_message = message.tracking.logs.first + expect(log_message).to be_a(Multiwoven::Integrations::Protocol::LogMessage) + expect(log_message.level).to eql("info") + + expect(log_message.message).to include("request") + expect(log_message.message).to include("response") + end + end + + context "when the write operation fails" do + it "increments the failure count" do + stub_request(:get, "https://graph.microsoft.com/v1.0/me") + .to_return(status: 200, body: response_body, headers: {}) + + stub_request(:get, "https://graph.microsoft.com/v1.0/drives/DRIVE1/root/children") + .to_return( + status: 200, + body: { + "value" => [ + { "id" => "file1_id", "name" => "test_table.xlsx" } + ] + }.to_json, + headers: {} + ) + + stub_request(:get, "https://graph.microsoft.com/v1.0/drives/DRIVE1/items/file1_id/workbook/worksheets/sheet/"\ + "tables?$select=name") + .to_return( + status: 200, + body: { + "value" => [ + { "name" => "Table1" } + ] + }.to_json, + headers: {} + ) + + stub_request(:post, + "https://graph.microsoft.com/v1.0/drives/DRIVE1/items/file1_id/workbook/worksheets/"\ + "test_table.xlsx/tables/Table1/rows") + .to_return(status: 400, body: failed_update_response_body, headers: {}) + + sync_config = Multiwoven::Integrations::Protocol::SyncConfig.from_json(sync_config_json.to_json) + records = [{ "Col1" => 400, "Col2" => 4.4, "Col3" => "Fourth" }] + + message = client.write(sync_config, records) + tracker = message.tracking + expect(tracker.failed).to eq(records.count) + expect(tracker.success).to eq(0) + log_message = message.tracking.logs.first + expect(log_message).to be_a(Multiwoven::Integrations::Protocol::LogMessage) + expect(log_message.level).to eql("info") + + expect(log_message.message).to include("request") + expect(log_message.message).to include("response") + end + end + end + + describe "#meta_data" do + # change this to rollout validation for all connector rolling out + it "client class_name and meta name is same" do + meta_name = client.class.to_s.split("::")[-2] + expect(client.send(:meta_data)[:data][:name]).to eq(meta_name) + end + end +end diff --git a/server/app/controllers/api/v1/connectors_controller.rb b/server/app/controllers/api/v1/connectors_controller.rb index d83c93e3..b26fe46b 100644 --- a/server/app/controllers/api/v1/connectors_controller.rb +++ b/server/app/controllers/api/v1/connectors_controller.rb @@ -95,7 +95,7 @@ def query_source if result.success? @records = result.records.map(&:record).map(&:data) - render json: @records, status: :ok + render json: { data: @records }, status: :ok else render_error( message: result["error"], diff --git a/server/lib/reverse_etl/extractors/base.rb b/server/lib/reverse_etl/extractors/base.rb index 88743d00..04a018dd 100644 --- a/server/lib/reverse_etl/extractors/base.rb +++ b/server/lib/reverse_etl/extractors/base.rb @@ -84,7 +84,8 @@ def find_or_initialize_sync_record(sync_run, primary_key) # there might be a risk of losing either the update or the create due to these concurrent operations. # we can use ActiveRecord::Base.transaction to prevent such scenarios SyncRecord.find_by(sync_id: sync_run.sync_id, primary_key:) || - sync_run.sync_records.new(sync_id: sync_run.sync_id, primary_key:, created_at: DateTime.current) + SyncRecord.new(sync_id: sync_run.sync_id, sync_run_id: sync_run.id, + primary_key:, created_at: DateTime.current) end def new_record?(sync_record, fingerprint) diff --git a/server/spec/requests/api/v1/connectors_controller_spec.rb b/server/spec/requests/api/v1/connectors_controller_spec.rb index 0ed99941..cf970563 100644 --- a/server/spec/requests/api/v1/connectors_controller_spec.rb +++ b/server/spec/requests/api/v1/connectors_controller_spec.rb @@ -438,8 +438,8 @@ post "/api/v1/connectors/#{connectors.second.id}/query_source", params: request_body.to_json, headers: { "Content-Type": "application/json" }.merge(auth_headers(user, workspace_id)) expect(response).to have_http_status(:ok) - response_hash = JSON.parse(response.body) - expect(response_hash).to eq([record1.record.data, record2.record.data]) + response_hash = JSON.parse(response.body).with_indifferent_access + expect(response_hash[:data]).to eq([record1.record.data, record2.record.data]) end it "returns success status for a valid query for member role" do @@ -449,8 +449,8 @@ post "/api/v1/connectors/#{connectors.second.id}/query_source", params: request_body.to_json, headers: { "Content-Type": "application/json" }.merge(auth_headers(user, workspace_id)) expect(response).to have_http_status(:ok) - response_hash = JSON.parse(response.body) - expect(response_hash).to eq([record1.record.data, record2.record.data]) + response_hash = JSON.parse(response.body).with_indifferent_access + expect(response_hash[:data]).to eq([record1.record.data, record2.record.data]) end it "returns success status for a valid query for viewer role" do @@ -460,8 +460,8 @@ post "/api/v1/connectors/#{connectors.second.id}/query_source", params: request_body.to_json, headers: { "Content-Type": "application/json" }.merge(auth_headers(user, workspace_id)) expect(response).to have_http_status(:ok) - response_hash = JSON.parse(response.body) - expect(response_hash).to eq([record1.record.data, record2.record.data]) + response_hash = JSON.parse(response.body).with_indifferent_access + expect(response_hash[:data]).to eq([record1.record.data, record2.record.data]) end it "returns failure status for a invalid query" do