diff --git a/integrations/lib/multiwoven/integrations.rb b/integrations/lib/multiwoven/integrations.rb index 2190b31d..35bb8848 100644 --- a/integrations/lib/multiwoven/integrations.rb +++ b/integrations/lib/multiwoven/integrations.rb @@ -76,6 +76,7 @@ require_relative "integrations/destination/zendesk/client" require_relative "integrations/destination/http/client" require_relative "integrations/destination/iterable/client" +require_relative "integrations/destination/maria_db/client" module Multiwoven module Integrations diff --git a/integrations/lib/multiwoven/integrations/destination/maria_db/client.rb b/integrations/lib/multiwoven/integrations/destination/maria_db/client.rb new file mode 100644 index 00000000..75931b8d --- /dev/null +++ b/integrations/lib/multiwoven/integrations/destination/maria_db/client.rb @@ -0,0 +1,115 @@ +# frozen_string_literal: true + +module Multiwoven::Integrations::Destination + module MariaDB + include Multiwoven::Integrations::Core + class Client < DestinationConnector + def check_connection(connection_config) + connection_config = connection_config.with_indifferent_access + create_connection(connection_config) + ConnectionStatus.new(status: ConnectionStatusType["succeeded"]).to_multiwoven_message + rescue StandardError => e + ConnectionStatus.new(status: ConnectionStatusType["failed"], message: e.message).to_multiwoven_message + end + + def discover(connection_config) + connection_config = connection_config.with_indifferent_access + query = "SELECT table_name, column_name, data_type, is_nullable + FROM information_schema.columns + WHERE table_schema = '#{connection_config[:database]}' + ORDER BY table_name, ordinal_position;" + + db = create_connection(connection_config) + records = db.fetch(query) do |result| + result.map do |row| + row + end + end + catalog = Catalog.new(streams: create_streams(records)) + catalog.to_multiwoven_message + rescue StandardError => e + handle_exception( + "MARIA:DB:DISCOVER:EXCEPTION", + "error", + e + ) + end + + def write(sync_config, records, action = "destination_insert") + connection_config = sync_config.destination.connection_specification.with_indifferent_access + table_name = sync_config.stream.name + primary_key = sync_config.model.primary_key + db = create_connection(connection_config) + + write_success = 0 + write_failure = 0 + + records.each do |record| + query = Multiwoven::Integrations::Core::QueryBuilder.perform(action, table_name, record, primary_key) + logger.debug("MARIA:DB:WRITE:QUERY query = #{query} sync_id = #{sync_config.sync_id} sync_run_id = #{sync_config.sync_run_id}") + begin + db.run(query) + write_success += 1 + rescue StandardError => e + handle_exception(e, { + context: "MARIA:DB:RECORD:WRITE:EXCEPTION", + type: "error", + sync_id: sync_config.sync_id, + sync_run_id: sync_config.sync_run_id + }) + write_failure += 1 + end + end + tracking_message(write_success, write_failure) + rescue StandardError => e + handle_exception(e, { + context: "MARIA:DB:RECORD:WRITE:EXCEPTION", + type: "error", + sync_id: sync_config.sync_id, + sync_run_id: sync_config.sync_run_id + }) + end + + private + + def create_connection(connection_config) + Sequel.connect( + adapter: "mysql2", + host: connection_config[:host], + port: connection_config[:port], + user: connection_config[:username], + password: connection_config[:password], + database: connection_config[:database] + ) + end + + def create_streams(records) + group_by_table(records).map do |_, r| + Multiwoven::Integrations::Protocol::Stream.new(name: r[:tablename], action: StreamAction["fetch"], json_schema: convert_to_json_schema(r[:columns])) + end + end + + def group_by_table(records) + result = {} + records.each_with_index do |entry, index| + table_name = entry[:table_name] + column_data = { + column_name: entry[:column_name], + data_type: entry[:data_type], + is_nullable: entry[:is_nullable] == "YES" + } + result[index] ||= {} + result[index][:tablename] = table_name + result[index][:columns] = [column_data] + end + result + end + + def tracking_message(success, failure) + Multiwoven::Integrations::Protocol::TrackingMessage.new( + success: success, failed: failure + ).to_multiwoven_message + end + end + end +end diff --git a/integrations/lib/multiwoven/integrations/destination/maria_db/config/meta.json b/integrations/lib/multiwoven/integrations/destination/maria_db/config/meta.json new file mode 100644 index 00000000..6c6626a6 --- /dev/null +++ b/integrations/lib/multiwoven/integrations/destination/maria_db/config/meta.json @@ -0,0 +1,15 @@ +{ + "data": { + "name": "MariaDB", + "title": "Maria DB", + "connector_type": "destination", + "category": "Database", + "documentation_url": "https://docs.squared.ai/guides/data-integration/destination/mariadb", + "github_issue_label": "destination-maria-db", + "icon": "icon.svg", + "license": "MIT", + "release_stage": "alpha", + "support_level": "community", + "tags": ["language:ruby", "multiwoven"] + } +} diff --git a/integrations/lib/multiwoven/integrations/destination/maria_db/config/spec.json b/integrations/lib/multiwoven/integrations/destination/maria_db/config/spec.json new file mode 100644 index 00000000..b726ac30 --- /dev/null +++ b/integrations/lib/multiwoven/integrations/destination/maria_db/config/spec.json @@ -0,0 +1,48 @@ +{ + "documentation_url": "https://docs.squared.ai/guides/data-integration/destination/mariadb", + "stream_type": "dynamic", + "connector_query_type": "raw_sql", + "connection_specification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Maria DB", + "type": "object", + "required": ["host", "port", "username", "password", "database"], + "properties": { + "host": { + "description": "The hostname or IP address of the server where the MariaDB database is hosted.", + "examples": ["localhost"], + "type": "string", + "title": "Host", + "order": 0 + }, + "port": { + "description": "The port number on which the MariaDB server is listening for connections.", + "examples": ["3306"], + "type": "string", + "title": "Port", + "order": 1 + }, + "username": { + "description": "The username used to authenticate and connect to the MariaDB database.", + "examples": ["root"], + "type": "string", + "title": "Username", + "order": 2 + }, + "password": { + "description": "The password corresponding to the username used for authentication.", + "type": "string", + "multiwoven_secret": true, + "title": "Password", + "order": 3 + }, + "database": { + "description": "The name of the specific database within the MariaDB server to connect to.", + "examples": ["test"], + "type": "string", + "title": "Database", + "order": 4 + } + } + } +} \ No newline at end of file diff --git a/integrations/lib/multiwoven/integrations/destination/maria_db/icon.svg b/integrations/lib/multiwoven/integrations/destination/maria_db/icon.svg new file mode 100644 index 00000000..2d0c2ee5 --- /dev/null +++ b/integrations/lib/multiwoven/integrations/destination/maria_db/icon.svg @@ -0,0 +1,15 @@ + + + + + + MDB-VLogo_RGB + + + + + + + + + \ No newline at end of file diff --git a/integrations/lib/multiwoven/integrations/rollout.rb b/integrations/lib/multiwoven/integrations/rollout.rb index 39022ca7..d7cd5fd9 100644 --- a/integrations/lib/multiwoven/integrations/rollout.rb +++ b/integrations/lib/multiwoven/integrations/rollout.rb @@ -32,6 +32,7 @@ module Integrations Zendesk Http Iterable + MariaDB ].freeze end end diff --git a/integrations/spec/multiwoven/integrations/destination/maria_db/client_spec.rb b/integrations/spec/multiwoven/integrations/destination/maria_db/client_spec.rb new file mode 100644 index 00000000..270ce34d --- /dev/null +++ b/integrations/spec/multiwoven/integrations/destination/maria_db/client_spec.rb @@ -0,0 +1,149 @@ +# frozen_string_literal: true + +RSpec.describe Multiwoven::Integrations::Destination::MariaDB::Client do + include WebMock::API + + before(:each) do + WebMock.disable_net_connect!(allow_localhost: true) + end + + let(:client) { described_class.new } + let(:connection_config) do + { + host: "127.0.0.1", + port: "3306", + username: "Test_service", + password: ENV["MARIADB_PASSWORD"], + database: "test_database" + } + end + let(:sync_config_json) do + { + source: { + name: "Sample Source Connector", + type: "source", + connection_specification: { + private_api_key: "test_api_key" + } + }, + destination: { + name: "MariaDB", + type: "destination", + connection_specification: connection_config + }, + model: { + name: "ExampleModel", + query: "SELECT col1, col2, col3 FROM test_table_1", + query_type: "raw_sql", + primary_key: "col1" + }, + sync_mode: "incremental", + destination_sync_mode: "insert", + stream: { + name: "Test_Table", + action: "create", + json_schema: { "field1": "type1" }, + supported_sync_modes: %w[full_refresh incremental] + } + } + end + + let(:sequel_client) { instance_double(Sequel::Database) } + let(:table) { double("Table") } + + describe "#check_connection" do + context "when the connection is successful" do + it "returns a succeeded connection status" do + allow_any_instance_of(Multiwoven::Integrations::Destination::MariaDB::Client).to receive(:create_connection).and_return(sequel_client) + message = client.check_connection(sync_config_json[:destination][:connection_specification]) + result = message.connection_status + expect(result.status).to eq("succeeded") + expect(result.message).to be_nil + end + end + + context "when the connection fails" do + it "returns a failed connection status with an error message" do + allow_any_instance_of(Multiwoven::Integrations::Destination::MariaDB::Client).to receive(:create_connection).and_raise(StandardError, "Connection failed") + message = client.check_connection(sync_config_json[:destination][:connection_specification]) + result = message.connection_status + expect(result.status).to eq("failed") + expect(result.message).to include("Connection failed") + end + end + end + + describe "#discover" do + it "discovers schema successfully" do + dataset = [ + { table_name: "test_table", column_name: "col1", data_type: "int", is_nullable: "YES" }, + { table_name: "test_table", column_name: "col2", data_type: "varchar", is_nullable: "YES" }, + { table_name: "test_table", column_name: "col3", data_type: "float", is_nullable: "YES" } + ] + allow(sequel_client).to receive(:fetch).and_return(dataset) + allow(client).to receive(:create_connection).and_return(sequel_client) + + message = client.discover(sync_config_json[:destination][:connection_specification]) + expect(message.catalog).to be_an(Multiwoven::Integrations::Protocol::Catalog) + first_stream = message.catalog.streams.first + expect(first_stream).to be_a(Multiwoven::Integrations::Protocol::Stream) + expect(first_stream.name).to eq("test_table") + expect(first_stream.json_schema).to be_an(Hash) + expect(first_stream.json_schema["type"]).to eq("object") + expect(first_stream.json_schema["properties"]).to eq({ "col1" => { "type" => "string" } }) + end + end + + describe "#write" do + context "when the write operation is successful" do + before do + allow_any_instance_of(Multiwoven::Integrations::Source::MariaDB::Client).to receive(:create_connection).and_return(sequel_client) + end + + it "increments the success count" do + sync_config = Multiwoven::Integrations::Protocol::SyncConfig.from_json( + sync_config_json.to_json + ) + records = [ + { "table_name" => "external_table", "value_attribute" => { "Col1" => 400, "Col2" => 4.4, "Col3" => "Fourth" }.to_json }, + { "table_name" => "external_table", "value_attribute" => { "Col1" => 500, "Col2" => 5.5, "Col3" => "Fifth" }.to_json }, + { "table_name" => "external_table", "value_attribute" => { "Col1" => 600, "Col2" => 6.6, "Col3" => "Sixth" }.to_json } + ] + allow(client).to receive(:create_connection).and_return(sequel_client) + allow(sequel_client).to receive(:run).and_return(nil) + response = client.write(sync_config, records) + expect(response.tracking.success).to eq(records.size) + expect(response.tracking.failed).to eq(0) + end + end + + context "when the write operation fails" do + before do + allow_any_instance_of(Multiwoven::Integrations::Destination::MariaDB::Client).to receive(:create_connection).and_return(sequel_client) + end + it "increments the failure count" do + sync_config = Multiwoven::Integrations::Protocol::SyncConfig.from_json( + sync_config_json.to_json + ) + records = [ + { "table_name" => "external_table", "value_attribute" => { "Col1" => 400, "Col2" => 4.4, "Col3" => "Fourth" } }, + { "table_name" => "external_table", "value_attribute" => { "Col1" => 500, "Col2" => 5.5, "Col3" => "Fifth" } }, + { "table_name" => "external_table", "value_attribute" => { "Col1" => 600, "Col2" => 6.6, "Col3" => "Sixth" } } + ] + allow(client).to receive(:create_connection).and_return(sequel_client) + allow(sequel_client).to receive(:run).and_raise(StandardError) + response = client.write(sync_config, records) + expect(response.tracking.failed).to eq(records.size) + expect(response.tracking.success).to eq(0) + end + end + end + + describe "#meta_data" do + # change this to rollout validation for all connector rolling out + it "client class_name and meta name is same" do + meta_name = client.class.to_s.split("::")[-2] + expect(client.send(:meta_data)[:data][:name]).to eq(meta_name) + end + end +end diff --git a/release-notes.md b/release-notes.md index 9864442c..88652cce 100644 --- a/release-notes.md +++ b/release-notes.md @@ -2,29 +2,30 @@ All notable changes to this project will be documented in this file. -## [0.14.0] - 2024-07-01 +## [0.15.0] - 2024-07-08 ### 🚀 Features -- *(CE)* Separate code climate reports and badges (#203) -- *(CE)* Add mariaDB source connector (#208) -- *(CE)* Lock user login attempts (#182) -- *(CE)* Sync records error log (#211) -- *(CE)* Add table selector as model query type (#243) (#209) -- *(CE)* Force refresh catalog when refresh flag is set true (#248) (#217) -- *(CE)* Add table selector as model query type (#234) (#213) +- *(CE)* Error logs for all sync records +- *(CE)* Added s3 connector ARN support for auth +- *(CE)* Integration changes for sync record log (#223) +- *(CE)* Server changes for save logs to sync record table (#231) +- *(CE)* Added select row support to data table (#232) ### 🐛 Bug Fixes -- *(CE)* Skip verify_authorized in logout (#204) -- *(CE)* Json error field added in sync record (#205) -- *(CE)* Add mariadb-dev in DockerFile (#235) -- *(CE)* Signup error response (#214) +- *(CE)* Sync records table (#257) (#221) +- *(CE)* Handle S3 credentials (#246) (#215) +- *(CE)* Add STS credentials for AWS S3 source connector (#224) +- *(CE)* Fixed issue where sync interval dropdown text was hidden in smaller screens (#252) (#230) +- *(CE)* Added processFormData to process form data before checking connection (#262) (#228) ### ⚙️ Miscellaneous Tasks -- *(CE)* Update the role policies (#206) -- *(CE)* Update server gem (#227) -- *(CE)* Pundit policy at role permissions level (#210) +- *(CE)* Gem update for s3 connector arn support (#254) (#220) +- *(CE)* Update Role Descriptions (#229) +- *(CE)* Config jwt secret from env variable (#269) (#233) +- *(CE)* Update README for MariaDB connectors (#249) +- *(CE)* Update README for MariaDB connectors (#249) diff --git a/server/Gemfile b/server/Gemfile index 0f4fea74..1ef76d41 100644 --- a/server/Gemfile +++ b/server/Gemfile @@ -13,7 +13,7 @@ gem "interactor", "~> 3.0" gem "ruby-odbc", git: "https://github.com/Multiwoven/ruby-odbc.git" -gem "multiwoven-integrations", "~> 0.3.5" +gem "multiwoven-integrations", "~> 0.4.0" gem "temporal-ruby", github: "coinbase/temporal-ruby" diff --git a/server/Gemfile.lock b/server/Gemfile.lock index af27ac53..38121bc5 100644 --- a/server/Gemfile.lock +++ b/server/Gemfile.lock @@ -1892,7 +1892,7 @@ GEM msgpack (1.7.2) multi_json (1.15.0) multipart-post (2.4.1) - multiwoven-integrations (0.3.5) + multiwoven-integrations (0.4.0) activesupport async-websocket aws-sdk-athena @@ -2115,7 +2115,7 @@ GEM faraday-multipart gli hashie - sorbet-runtime (0.5.11463) + sorbet-runtime (0.5.11465) stringio (3.1.0) stripe (12.0.0) strong_migrations (1.8.0) @@ -2189,7 +2189,7 @@ DEPENDENCIES jwt kaminari liquid - multiwoven-integrations (~> 0.3.5) + multiwoven-integrations (~> 0.4.0) mysql2 newrelic_rpm parallel diff --git a/server/app/models/organization.rb b/server/app/models/organization.rb index 81836b79..c9957676 100644 --- a/server/app/models/organization.rb +++ b/server/app/models/organization.rb @@ -13,4 +13,6 @@ class Organization < ApplicationRecord validates :name, presence: true, uniqueness: { case_sensitive: false } has_many :workspaces, dependent: :destroy + has_many :workspace_users, through: :workspaces + has_many :users, through: :workspace_users end diff --git a/server/spec/models/organization_spec.rb b/server/spec/models/organization_spec.rb index cf113ee5..9287c151 100644 --- a/server/spec/models/organization_spec.rb +++ b/server/spec/models/organization_spec.rb @@ -20,6 +20,28 @@ # Test associations describe "associations" do it { should have_many(:workspaces).dependent(:destroy) } - # Add other associations here + it { should have_many(:workspace_users).through(:workspaces) } + it { should have_many(:users).through(:workspace_users) } + end + + describe "association functionality" do + let(:organization) { create(:organization) } + let(:workspace) { create(:workspace, organization:) } + let(:user) { create(:user) } + let(:workspace_user) { create(:workspace_user, workspace:, user:) } + + before do + workspace + user + workspace_user + end + + it "includes the correct workspace_users through workspaces" do + expect(organization.workspace_users).to include(workspace_user) + end + + it "includes the correct users through workspace_users" do + expect(organization.users).to include(user) + end end end