diff --git a/.github/workflows/integrations-ci.yml b/.github/workflows/integrations-ci.yml index 4e8c44d9..1eca6149 100644 --- a/.github/workflows/integrations-ci.yml +++ b/.github/workflows/integrations-ci.yml @@ -31,6 +31,14 @@ jobs: sudo mv libduckdb/libduckdb.so /usr/local/lib sudo ldconfig /usr/local/lib + - name: Download and Install Oracle Instant Client + run: | + sudo apt-get install -y libaio1 alien + wget http://yum.oracle.com/repo/OracleLinux/OL7/oracle/instantclient/x86_64/getPackage/oracle-instantclient19.6-basic-19.6.0.0.0-1.x86_64.rpm + wget http://yum.oracle.com/repo/OracleLinux/OL7/oracle/instantclient/x86_64/getPackage/oracle-instantclient19.6-devel-19.6.0.0.0-1.x86_64.rpm + sudo alien -i --scripts oracle-instantclient*.rpm + rm -f oracle-instantclient*.rpm + - name: Install dependencies run: | gem install bundler diff --git a/.github/workflows/integrations-main.yml b/.github/workflows/integrations-main.yml index d9ecb8ad..fd43f306 100644 --- a/.github/workflows/integrations-main.yml +++ b/.github/workflows/integrations-main.yml @@ -39,6 +39,14 @@ jobs: sudo mv libduckdb/libduckdb.so /usr/local/lib sudo ldconfig /usr/local/lib + - name: Download and Install Oracle Instant Client + run: | + sudo apt-get install -y libaio1 alien + wget http://yum.oracle.com/repo/OracleLinux/OL7/oracle/instantclient/x86_64/getPackage/oracle-instantclient19.6-basic-19.6.0.0.0-1.x86_64.rpm + wget http://yum.oracle.com/repo/OracleLinux/OL7/oracle/instantclient/x86_64/getPackage/oracle-instantclient19.6-devel-19.6.0.0.0-1.x86_64.rpm + sudo alien -i --scripts oracle-instantclient*.rpm + rm -f oracle-instantclient*.rpm + - name: Install dependencies run: bundle install working-directory: ./integrations diff --git a/.github/workflows/server-ci.yml b/.github/workflows/server-ci.yml index 133e7399..176bacc6 100644 --- a/.github/workflows/server-ci.yml +++ b/.github/workflows/server-ci.yml @@ -50,6 +50,14 @@ jobs: sudo mv libduckdb/libduckdb.so /usr/local/lib sudo ldconfig /usr/local/lib + - name: Download and Install Oracle Instant Client + run: | + sudo apt-get install -y libaio1 alien + wget http://yum.oracle.com/repo/OracleLinux/OL7/oracle/instantclient/x86_64/getPackage/oracle-instantclient19.6-basic-19.6.0.0.0-1.x86_64.rpm + wget http://yum.oracle.com/repo/OracleLinux/OL7/oracle/instantclient/x86_64/getPackage/oracle-instantclient19.6-devel-19.6.0.0.0-1.x86_64.rpm + sudo alien -i --scripts oracle-instantclient*.rpm + rm -f oracle-instantclient*.rpm + - name: Bundle Install run: bundle install working-directory: ./server diff --git a/integrations/Gemfile b/integrations/Gemfile index b28b59ce..1b82fbae 100644 --- a/integrations/Gemfile +++ b/integrations/Gemfile @@ -67,6 +67,8 @@ gem "mysql2" gem "aws-sdk-sts" +gem "ruby-oci8" + group :development, :test do gem "simplecov", require: false gem "simplecov_json_formatter", require: false diff --git a/integrations/Gemfile.lock b/integrations/Gemfile.lock index 49bc10da..846de3fe 100644 --- a/integrations/Gemfile.lock +++ b/integrations/Gemfile.lock @@ -7,11 +7,7 @@ GIT PATH remote: . specs: -<<<<<<< HEAD - multiwoven-integrations (0.5.2) -======= multiwoven-integrations (0.7.1) ->>>>>>> c05f9f32 (chore(CE): change name to databricks datawarehouse (#340)) activesupport async-websocket aws-sdk-athena @@ -32,6 +28,7 @@ PATH rake restforce ruby-limiter + ruby-oci8 ruby-odbc rubyzip sequel @@ -279,6 +276,8 @@ GEM rubocop-ast (1.31.3) parser (>= 3.3.1.0) ruby-limiter (2.3.0) + ruby-oci8 (2.2.12) + ruby-oci8 (2.2.12-x64-mingw-ucrt) ruby-progressbar (1.13.0) ruby2_keywords (0.0.5) rubyzip (2.3.2) @@ -361,6 +360,7 @@ DEPENDENCIES rspec (~> 3.0) rubocop (~> 1.21) ruby-limiter + ruby-oci8 ruby-odbc! rubyzip sequel diff --git a/integrations/lib/multiwoven/integrations.rb b/integrations/lib/multiwoven/integrations.rb index d305120f..4f80fab4 100644 --- a/integrations/lib/multiwoven/integrations.rb +++ b/integrations/lib/multiwoven/integrations.rb @@ -31,6 +31,7 @@ require "duckdb" require "iterable-api-client" require "aws-sdk-sts" +require "ruby-oci8" # Service require_relative "integrations/config" @@ -60,6 +61,7 @@ require_relative "integrations/source/clickhouse/client" require_relative "integrations/source/amazon_s3/client" require_relative "integrations/source/maria_db/client" +require_relative "integrations/source/oracle_db/client" # Destination require_relative "integrations/destination/klaviyo/client" @@ -78,6 +80,7 @@ require_relative "integrations/destination/iterable/client" require_relative "integrations/destination/maria_db/client" require_relative "integrations/destination/databricks_lakehouse/client" +require_relative "integrations/destination/oracle_db/client" module Multiwoven module Integrations diff --git a/integrations/lib/multiwoven/integrations/destination/oracle_db/client.rb b/integrations/lib/multiwoven/integrations/destination/oracle_db/client.rb new file mode 100644 index 00000000..da848bf8 --- /dev/null +++ b/integrations/lib/multiwoven/integrations/destination/oracle_db/client.rb @@ -0,0 +1,112 @@ +# frozen_string_literal: true + +module Multiwoven::Integrations::Destination + module Oracle + include Multiwoven::Integrations::Core + class Client < DestinationConnector + def check_connection(connection_config) + connection_config = connection_config.with_indifferent_access + create_connection(connection_config) + ConnectionStatus.new( + status: ConnectionStatusType["succeeded"] + ).to_multiwoven_message + rescue StandardError => e + ConnectionStatus.new( + status: ConnectionStatusType["failed"], message: e.message + ).to_multiwoven_message + end + + def discover(connection_config) + records = [] + connection_config = connection_config.with_indifferent_access + query = "SELECT table_name, column_name, data_type, nullable + FROM all_tab_columns + WHERE owner = '#{connection_config[:username].upcase}' + ORDER BY table_name, column_id" + conn = create_connection(connection_config) + cursor = conn.exec(query) + while (row = cursor.fetch) + records << row + end + catalog = Catalog.new(streams: create_streams(records)) + catalog.to_multiwoven_message + rescue StandardError => e + handle_exception( + "ORACLE:DISCOVER:EXCEPTION", + "error", + e + ) + end + + def write(sync_config, records, action = "destination_insert") + connection_config = sync_config.destination.connection_specification.with_indifferent_access + table_name = sync_config.stream.name + primary_key = sync_config.model.primary_key + conn = create_connection(connection_config) + + write_success = 0 + write_failure = 0 + log_message_array = [] + + records.each do |record| + query = Multiwoven::Integrations::Core::QueryBuilder.perform(action, table_name, record, primary_key) + query = query.gsub(";", "") + logger.debug("ORACLE:WRITE:QUERY query = #{query} sync_id = #{sync_config.sync_id} sync_run_id = #{sync_config.sync_run_id}") + begin + response = conn.exec(query) + conn.exec("COMMIT") + write_success += 1 + log_message_array << log_request_response("info", query, response) + rescue StandardError => e + handle_exception(e, { + context: "ORACLE:RECORD:WRITE:EXCEPTION", + type: "error", + sync_id: sync_config.sync_id, + sync_run_id: sync_config.sync_run_id + }) + write_failure += 1 + log_message_array << log_request_response("error", query, e.message) + end + end + tracking_message(write_success, write_failure, log_message_array) + rescue StandardError => e + handle_exception(e, { + context: "ORACLE:RECORD:WRITE:EXCEPTION", + type: "error", + sync_id: sync_config.sync_id, + sync_run_id: sync_config.sync_run_id + }) + end + + private + + def create_connection(connection_config) + OCI8.new(connection_config[:username], connection_config[:password], "#{connection_config[:host]}:#{connection_config[:port]}/#{connection_config[:sid]}") + end + + def create_streams(records) + group_by_table(records).map do |_, r| + Multiwoven::Integrations::Protocol::Stream.new(name: r[:tablename], action: StreamAction["fetch"], json_schema: convert_to_json_schema(r[:columns])) + end + end + + def group_by_table(records) + result = {} + records.each_with_index do |entry, index| + table_name = entry[0] + column_data = { + column_name: entry[1], + data_type: entry[2], + is_nullable: entry[3] == "Y" + } + result[index] ||= {} + result[index][:tablename] = table_name + result[index][:columns] = [column_data] + end + result.values.group_by { |entry| entry[:tablename] }.transform_values do |entries| + { tablename: entries.first[:tablename], columns: entries.flat_map { |entry| entry[:columns] } } + end + end + end + end +end diff --git a/integrations/lib/multiwoven/integrations/destination/oracle_db/config/meta.json b/integrations/lib/multiwoven/integrations/destination/oracle_db/config/meta.json new file mode 100644 index 00000000..5662698c --- /dev/null +++ b/integrations/lib/multiwoven/integrations/destination/oracle_db/config/meta.json @@ -0,0 +1,15 @@ +{ + "data": { + "name": "Oracle", + "title": "Oracle", + "connector_type": "destination", + "category": "Database", + "documentation_url": "https://docs.squared.ai/guides/data-integration/destination/oracle", + "github_issue_label": "destination-oracle", + "icon": "icon.svg", + "license": "MIT", + "release_stage": "alpha", + "support_level": "community", + "tags": ["language:ruby", "multiwoven"] + } +} diff --git a/integrations/lib/multiwoven/integrations/destination/oracle_db/config/spec.json b/integrations/lib/multiwoven/integrations/destination/oracle_db/config/spec.json new file mode 100644 index 00000000..2eed10f6 --- /dev/null +++ b/integrations/lib/multiwoven/integrations/destination/oracle_db/config/spec.json @@ -0,0 +1,47 @@ +{ + "documentation_url": "https://docs.squared.ai/guides/data-integration/destination/oracle", + "stream_type": "dynamic", + "connector_query_type": "raw_sql", + "connection_specification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Oracle", + "type": "object", + "required": ["host", "port", "sid", "username", "password"], + "properties": { + "host": { + "description": "The Oracle host.", + "examples": ["localhost"], + "type": "string", + "title": "Host", + "order": 0 + }, + "port": { + "description": "The Oracle port number.", + "examples": ["1521"], + "type": "string", + "title": "Port", + "order": 1 + }, + "sid": { + "description": "The name of your service in Oracle.", + "examples": ["ORCLPDB1"], + "type": "string", + "title": "SID", + "order": 2 + }, + "username": { + "description": "The username used to authenticate and connect.", + "type": "string", + "title": "Username", + "order": 3 + }, + "password": { + "description": "The password corresponding to the username used for authentication.", + "type": "string", + "multiwoven_secret": true, + "title": "Password", + "order": 4 + } + } + } +} \ No newline at end of file diff --git a/integrations/lib/multiwoven/integrations/destination/oracle_db/icon.svg b/integrations/lib/multiwoven/integrations/destination/oracle_db/icon.svg new file mode 100644 index 00000000..3f4e051f --- /dev/null +++ b/integrations/lib/multiwoven/integrations/destination/oracle_db/icon.svg @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/integrations/lib/multiwoven/integrations/rollout.rb b/integrations/lib/multiwoven/integrations/rollout.rb index 3c9fdd87..45d2b8b2 100644 --- a/integrations/lib/multiwoven/integrations/rollout.rb +++ b/integrations/lib/multiwoven/integrations/rollout.rb @@ -2,11 +2,7 @@ module Multiwoven module Integrations -<<<<<<< HEAD - VERSION = "0.5.2" -======= - VERSION = "0.7.1" ->>>>>>> c05f9f32 (chore(CE): change name to databricks datawarehouse (#340)) + VERSION = "0.7.0" ENABLED_SOURCES = %w[ Snowflake @@ -19,6 +15,7 @@ module Integrations Clickhouse AmazonS3 MariaDB + Oracle ].freeze ENABLED_DESTINATIONS = %w[ @@ -38,6 +35,7 @@ module Integrations Iterable MariaDB DatabricksLakehouse + Oracle ].freeze end end diff --git a/integrations/lib/multiwoven/integrations/source/oracle_db/client.rb b/integrations/lib/multiwoven/integrations/source/oracle_db/client.rb new file mode 100644 index 00000000..f5d787bc --- /dev/null +++ b/integrations/lib/multiwoven/integrations/source/oracle_db/client.rb @@ -0,0 +1,127 @@ +# frozen_string_literal: true + +module Multiwoven::Integrations::Source + module Oracle + include Multiwoven::Integrations::Core + class Client < SourceConnector + def check_connection(connection_config) + connection_config = connection_config.with_indifferent_access + create_connection(connection_config) + ConnectionStatus.new( + status: ConnectionStatusType["succeeded"] + ).to_multiwoven_message + rescue StandardError => e + ConnectionStatus.new( + status: ConnectionStatusType["failed"], message: e.message + ).to_multiwoven_message + end + + def discover(connection_config) + records = [] + connection_config = connection_config.with_indifferent_access + query = "SELECT table_name, column_name, data_type, nullable + FROM all_tab_columns + WHERE owner = '#{connection_config[:username].upcase}' + ORDER BY table_name, column_id" + conn = create_connection(connection_config) + cursor = conn.exec(query) + while (row = cursor.fetch) + records << row + end + catalog = Catalog.new(streams: create_streams(records)) + catalog.to_multiwoven_message + rescue StandardError => e + handle_exception( + "ORACLE:DISCOVER:EXCEPTION", + "error", + e + ) + end + + def read(sync_config) + connection_config = sync_config.source.connection_specification.with_indifferent_access + query = sync_config.model.query + db = create_connection(connection_config) + query(db, query) + rescue StandardError => e + handle_exception(e, { + context: "ORACLE:READ:EXCEPTION", + type: "error", + sync_id: sync_config.sync_id, + sync_run_id: sync_config.sync_run_id + }) + end + + private + + def create_connection(connection_config) + OCI8.new(connection_config[:username], connection_config[:password], "#{connection_config[:host]}:#{connection_config[:port]}/#{connection_config[:sid]}") + end + + def create_streams(records) + group_by_table(records).map do |_, r| + Multiwoven::Integrations::Protocol::Stream.new(name: r[:tablename], action: StreamAction["fetch"], json_schema: convert_to_json_schema(r[:columns])) + end + end + + def query(connection, query) + records = [] + query = reformat_query(query) + cursor = connection.exec(query) + columns = cursor.get_col_names + while (row = cursor.fetch) + data_hash = columns.zip(row).to_h + records << RecordMessage.new(data: data_hash, emitted_at: Time.now.to_i).to_multiwoven_message + end + records + end + + def group_by_table(records) + result = {} + records.each_with_index do |entry, index| + table_name = entry[0] + column_data = { + column_name: entry[1], + data_type: entry[2], + is_nullable: entry[3] == "Y" + } + result[index] ||= {} + result[index][:tablename] = table_name + result[index][:columns] = [column_data] + end + result.values.group_by { |entry| entry[:tablename] }.transform_values do |entries| + { tablename: entries.first[:tablename], columns: entries.flat_map { |entry| entry[:columns] } } + end + end + + def reformat_query(sql_query) + offset = nil + limit = nil + + sql_query = sql_query.gsub(";", "") + + if sql_query.match?(/LIMIT (\d+)/i) + limit = sql_query.match(/LIMIT (\d+)/i)[1].to_i + sql_query.sub!(/LIMIT \d+/i, "") + end + + if sql_query.match?(/OFFSET (\d+)/i) + offset = sql_query.match(/OFFSET (\d+)/i)[1].to_i + sql_query.sub!(/OFFSET \d+/i, "") + end + + sql_query.strip! + + if offset && limit + "#{sql_query} OFFSET #{offset} ROWS FETCH NEXT #{limit} ROWS ONLY" + elsif offset + "#{sql_query} OFFSET #{offset} ROWS" + elsif limit + "#{sql_query} FETCH NEXT #{limit} ROWS ONLY" + else + sql_query + end + end + end + end +end diff --git a/integrations/lib/multiwoven/integrations/source/oracle_db/config/meta.json b/integrations/lib/multiwoven/integrations/source/oracle_db/config/meta.json new file mode 100644 index 00000000..833d36f0 --- /dev/null +++ b/integrations/lib/multiwoven/integrations/source/oracle_db/config/meta.json @@ -0,0 +1,15 @@ +{ + "data": { + "name": "Oracle", + "title": "Oracle", + "connector_type": "source", + "category": "Database", + "documentation_url": "https://docs.squared.ai/guides/data-integration/source/oracle", + "github_issue_label": "source-oracle", + "icon": "icon.svg", + "license": "MIT", + "release_stage": "alpha", + "support_level": "community", + "tags": ["language:ruby", "multiwoven"] + } +} diff --git a/integrations/lib/multiwoven/integrations/source/oracle_db/config/spec.json b/integrations/lib/multiwoven/integrations/source/oracle_db/config/spec.json new file mode 100644 index 00000000..703791b7 --- /dev/null +++ b/integrations/lib/multiwoven/integrations/source/oracle_db/config/spec.json @@ -0,0 +1,47 @@ +{ + "documentation_url": "https://docs.squared.ai/guides/data-integration/source/oracle", + "stream_type": "dynamic", + "connector_query_type": "raw_sql", + "connection_specification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Oracle", + "type": "object", + "required": ["host", "port", "sid", "username", "password"], + "properties": { + "host": { + "description": "The Oracle host.", + "examples": ["localhost"], + "type": "string", + "title": "Host", + "order": 0 + }, + "port": { + "description": "The Oracle port number.", + "examples": ["1521"], + "type": "string", + "title": "Port", + "order": 1 + }, + "sid": { + "description": "The name of your service in Oracle.", + "examples": ["ORCLPDB1"], + "type": "string", + "title": "SID", + "order": 2 + }, + "username": { + "description": "The username used to authenticate and connect.", + "type": "string", + "title": "Username", + "order": 3 + }, + "password": { + "description": "The password corresponding to the username used for authentication.", + "type": "string", + "multiwoven_secret": true, + "title": "Password", + "order": 4 + } + } + } +} \ No newline at end of file diff --git a/integrations/lib/multiwoven/integrations/source/oracle_db/icon.svg b/integrations/lib/multiwoven/integrations/source/oracle_db/icon.svg new file mode 100644 index 00000000..3f4e051f --- /dev/null +++ b/integrations/lib/multiwoven/integrations/source/oracle_db/icon.svg @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/integrations/multiwoven-integrations.gemspec b/integrations/multiwoven-integrations.gemspec index 34aec384..498a2d19 100644 --- a/integrations/multiwoven-integrations.gemspec +++ b/integrations/multiwoven-integrations.gemspec @@ -53,6 +53,7 @@ Gem::Specification.new do |spec| spec.add_runtime_dependency "rake" spec.add_runtime_dependency "restforce" spec.add_runtime_dependency "ruby-limiter" + spec.add_runtime_dependency "ruby-oci8" spec.add_runtime_dependency "ruby-odbc" spec.add_runtime_dependency "rubyzip" spec.add_runtime_dependency "sequel" diff --git a/integrations/spec/multiwoven/integrations/destination/oracle_db/client_spec.rb b/integrations/spec/multiwoven/integrations/destination/oracle_db/client_spec.rb new file mode 100644 index 00000000..7774e2e5 --- /dev/null +++ b/integrations/spec/multiwoven/integrations/destination/oracle_db/client_spec.rb @@ -0,0 +1,146 @@ +# frozen_string_literal: true + +RSpec.describe Multiwoven::Integrations::Destination::Oracle::Client do + include WebMock::API + + before(:each) do + WebMock.disable_net_connect!(allow_localhost: true) + end + + let(:client) { described_class.new } + let(:connection_config) do + { + host: "localhost", + port: "1521", + servicename: "PDB1", + username: "oracle_user", + password: "oracle_password" + } + end + let(:sync_config_json) do + { + source: { + name: "Sample Source Connector", + type: "source", + connection_specification: { + private_api_key: "test_api_key" + } + }, + destination: { + name: "Databricks", + type: "destination", + connection_specification: connection_config + }, + model: { + name: "ExampleModel", + query: "SELECT col1, col2, col3 FROM test_table_1", + query_type: "raw_sql", + primary_key: "col1" + }, + sync_mode: "incremental", + destination_sync_mode: "insert", + stream: { + name: "table", + action: "create", + json_schema: {}, + supported_sync_modes: %w[incremental] + } + } + end + + let(:oracle_connection) { instance_double(OCI8) } + let(:cursor) { instance_double("OCI8::Cursor") } + + describe "#check_connection" do + context "when the connection is successful" do + it "returns a succeeded connection status" do + allow(OCI8).to receive(:new).and_return(oracle_connection) + allow(oracle_connection).to receive(:exec).and_return(true) + message = client.check_connection(sync_config_json[:destination][:connection_specification]) + result = message.connection_status + expect(result.status).to eq("succeeded") + expect(result.message).to be_nil + end + end + + context "when the connection fails" do + it "returns a failed connection status with an error message" do + allow(client).to receive(:create_connection).and_raise(StandardError, "Connection failed") + message = client.check_connection(sync_config_json[:destination][:connection_specification]) + result = message.connection_status + expect(result.status).to eq("failed") + expect(result.message).to include("Connection failed") + end + end + end + + describe "#discover" do + it "discovers schema successfully" do + response = %w[test_table col1 NUMBER Y] + allow(OCI8).to receive(:new).and_return(oracle_connection) + allow(oracle_connection).to receive(:exec).and_return(cursor) + allow(cursor).to receive(:fetch).and_return(response, nil) + message = client.discover(sync_config_json[:destination][:connection_specification]) + expect(message.catalog).to be_an(Multiwoven::Integrations::Protocol::Catalog) + first_stream = message.catalog.streams.first + expect(first_stream).to be_a(Multiwoven::Integrations::Protocol::Stream) + expect(first_stream.name).to eq("test_table") + expect(first_stream.json_schema).to be_an(Hash) + expect(first_stream.json_schema["type"]).to eq("object") + expect(first_stream.json_schema["properties"]).to eq({ "col1" => { "type" => "string" } }) + end + end + + describe "#write" do + context "when the write operation is successful" do + it "increments the success count" do + sync_config = Multiwoven::Integrations::Protocol::SyncConfig.from_json( + sync_config_json.to_json + ) + record = [ + { "col1" => 1, "col2" => "first", "col3" => 1.1 } + ] + allow(OCI8).to receive(:new).and_return(oracle_connection) + allow(oracle_connection).to receive(:exec).and_return(1) + allow(cursor).to receive(:fetch).and_return(1, nil) + response = client.write(sync_config, record) + expect(response.tracking.success).to eq(record.size) + expect(response.tracking.failed).to eq(0) + log_message = response.tracking.logs.first + expect(log_message).to be_a(Multiwoven::Integrations::Protocol::LogMessage) + expect(log_message.level).to eql("info") + expect(log_message.message).to include("request") + expect(log_message.message).to include("response") + end + end + + context "when the write operation fails" do + it "increments the failure count" do + sync_config = Multiwoven::Integrations::Protocol::SyncConfig.from_json( + sync_config_json.to_json + ) + record = [ + { "col1" => 1, "col2" => "first", "col3" => 1.1 } + ] + allow(OCI8).to receive(:new).and_return(oracle_connection) + allow(oracle_connection).to receive(:exec).and_raise(StandardError, "Test error") + response = client.write(sync_config, record) + expect(response.tracking.failed).to eq(record.size) + expect(response.tracking.success).to eq(0) + log_message = response.tracking.logs.first + expect(log_message).to be_a(Multiwoven::Integrations::Protocol::LogMessage) + expect(log_message.level).to eql("error") + expect(log_message.message).to include("request") + expect(log_message.message).to include("{\"request\":\"INSERT INTO table (col1, col2, col3) VALUES ('1', 'first', '1.1')\",\"response\":\"Test error\",\"level\":\"error\"}") + end + end + end + + describe "#meta_data" do + # change this to rollout validation for all connector rolling out + it "client class_name and meta name is same" do + meta_name = client.class.to_s.split("::")[-2] + expect(client.send(:meta_data)[:data][:name]).to eq(meta_name) + end + end +end diff --git a/integrations/spec/multiwoven/integrations/source/oracle_db/client_spec.rb b/integrations/spec/multiwoven/integrations/source/oracle_db/client_spec.rb new file mode 100644 index 00000000..51b9af14 --- /dev/null +++ b/integrations/spec/multiwoven/integrations/source/oracle_db/client_spec.rb @@ -0,0 +1,153 @@ +# frozen_string_literal: true + +RSpec.describe Multiwoven::Integrations::Source::Oracle::Client do + let(:client) { Multiwoven::Integrations::Source::Oracle::Client.new } + let(:sync_config) do + { + "source": { + "name": "OracleConnector", + "type": "source", + "connection_specification": { + "host": "localhost", + "port": "1521", + "servicename": "PDB1", + "username": "oracle_user", + "password": "oracle_password" + } + }, + "destination": { + "name": "DestinationConnectorName", + "type": "destination", + "connection_specification": { + "example_destination_key": "example_destination_value" + } + }, + "model": { + "name": "OracleDB Model", + "query": "SELECT col1, col2, col3 FROM test_table", + "query_type": "raw_sql", + "primary_key": "id" + }, + "stream": { + "name": "example_stream", "action": "create", + "json_schema": { "field1": "type1" }, + "supported_sync_modes": %w[full_refresh incremental], + "source_defined_cursor": true, + "default_cursor_field": ["field1"], + "source_defined_primary_key": [["field1"], ["field2"]], + "namespace": "exampleNamespace", + "url": "https://api.example.com/data", + "method": "GET" + }, + "sync_mode": "full_refresh", + "cursor_field": "timestamp", + "destination_sync_mode": "upsert", + "sync_id": "1" + } + end + + let(:oracle_connection) { instance_double(OCI8) } + let(:cursor) { instance_double("OCI8::Cursor") } + + describe "#check_connection" do + context "when the connection is successful" do + it "returns a succeeded connection status" do + allow(OCI8).to receive(:new).and_return(oracle_connection) + allow(oracle_connection).to receive(:exec).and_return(true) + message = client.check_connection(sync_config[:source][:connection_specification]) + result = message.connection_status + expect(result.status).to eq("succeeded") + expect(result.message).to be_nil + end + end + + context "when the connection fails" do + it "returns a failed connection status with an error message" do + allow(client).to receive(:create_connection).and_raise(StandardError, "Connection failed") + message = client.check_connection(sync_config[:source][:connection_specification]) + result = message.connection_status + expect(result.status).to eq("failed") + expect(result.message).to include("Connection failed") + end + end + end + + # read and #discover tests for MariaDB + describe "#read" do + it "reads records successfully" do + s_config = Multiwoven::Integrations::Protocol::SyncConfig.from_json(sync_config.to_json) + columns = %w[col1 col2 col3] + response = %w[1 First Row Text First Row Additional Text] + allow(OCI8).to receive(:new).and_return(oracle_connection) + allow(oracle_connection).to receive(:exec).and_return(cursor) + allow(cursor).to receive(:get_col_names).and_return(columns, nil) + allow(cursor).to receive(:fetch).and_return(response, nil) + records = client.read(s_config) + expect(records).to be_an(Array) + expect(records).not_to be_empty + expect(records.first).to be_a(Multiwoven::Integrations::Protocol::MultiwovenMessage) + end + + it "reads records successfully with limit" do + s_config = Multiwoven::Integrations::Protocol::SyncConfig.from_json(sync_config.to_json) + s_config.limit = 100 + s_config.offset = 1 + columns = %w[col1 col2 col3] + response = %w[1 First Row Text First Row Additional Text] + allow(OCI8).to receive(:new).and_return(oracle_connection) + allow(oracle_connection).to receive(:exec).and_return(cursor) + allow(cursor).to receive(:get_col_names).and_return(columns, nil) + allow(cursor).to receive(:fetch).and_return(response, nil) + records = client.read(s_config) + expect(records).to be_an(Array) + expect(records).not_to be_empty + expect(records.first).to be_a(Multiwoven::Integrations::Protocol::MultiwovenMessage) + end + + it "read records failure" do + s_config = Multiwoven::Integrations::Protocol::SyncConfig.from_json(sync_config.to_json) + s_config.sync_run_id = "2" + allow(client).to receive(:create_connection).and_raise(StandardError, "test error") + expect(client).to receive(:handle_exception).with( + an_instance_of(StandardError), { + context: "ORACLE:READ:EXCEPTION", + type: "error", + sync_id: "1", + sync_run_id: "2" + } + ) + client.read(s_config) + end + end + + describe "#discover" do + it "discovers schema successfully" do + response = %w[test_table col1 NUMBER Y] + allow(OCI8).to receive(:new).and_return(oracle_connection) + allow(oracle_connection).to receive(:exec).and_return(cursor) + allow(cursor).to receive(:fetch).and_return(response, nil) + message = client.discover(sync_config[:source][:connection_specification]) + expect(message.catalog).to be_an(Multiwoven::Integrations::Protocol::Catalog) + first_stream = message.catalog.streams.first + expect(first_stream).to be_a(Multiwoven::Integrations::Protocol::Stream) + expect(first_stream.name).to eq("test_table") + expect(first_stream.json_schema).to be_an(Hash) + expect(first_stream.json_schema["type"]).to eq("object") + expect(first_stream.json_schema["properties"]).to eq({ "col1" => { "type" => "string" } }) + end + end + + describe "#meta_data" do + # change this to rollout validation for all connector rolling out + it "client class_name and meta name is same" do + meta_name = client.class.to_s.split("::")[-2] + expect(client.send(:meta_data)[:data][:name]).to eq(meta_name) + end + end + + describe "method definition" do + it "defines a private #query method" do + expect(described_class.private_instance_methods).to include(:query) + end + end +end diff --git a/server/Dockerfile b/server/Dockerfile index 3f9b9c4b..1615d389 100644 --- a/server/Dockerfile +++ b/server/Dockerfile @@ -20,6 +20,10 @@ FROM base as build RUN apt-get update -qq && \ apt-get install --no-install-recommends -y build-essential autoconf automake libtool git libpq-dev libvips pkg-config m4 perl libltdl-dev curl git wget unzip default-libmysqlclient-dev +# Copy and run the Oracle Instant Client installation script +COPY getoracleinstantclient.sh . +RUN chmod +x getoracleinstantclient.sh && ./getoracleinstantclient.sh + COPY getduckdb.sh . COPY gethttpfsextension.sh . @@ -72,17 +76,16 @@ ENV LD_LIBRARY_PATH=/usr/local/lib:${LD_LIBRARY_PATH} ARG TARGETARCH=amd64 RUN if [ "$TARGETARCH" = "amd64" ] || [ "$TARGETARCH" = "x86_64" ]; then \ - wget https://sfc-repo.snowflakecomputing.com/odbc/linux/latest/snowflake-odbc-3.2.0.x86_64.deb -O snowflake-odbc.deb && \ - dpkg -i snowflake-odbc.deb || apt-get -y -f install; \ + wget https://sfc-repo.snowflakecomputing.com/odbc/linux/latest/snowflake-odbc-3.2.0.x86_64.deb -O snowflake-odbc.deb && \ + dpkg -i snowflake-odbc.deb || apt-get -y -f install; \ elif [ "$TARGETARCH" = "arm64" ] || [ "$TARGETARCH" = "aarch64" ]; then \ - wget https://sfc-repo.snowflakecomputing.com/odbc/linuxaarch64/3.2.0/snowflake-odbc-3.2.0.aarch64.deb -O snowflake-odbc.deb && \ - dpkg -i snowflake-odbc.deb || apt-get -y -f install; \ + wget https://sfc-repo.snowflakecomputing.com/odbc/linuxaarch64/3.2.0/snowflake-odbc-3.2.0.aarch64.deb -O snowflake-odbc.deb && \ + dpkg -i snowflake-odbc.deb || apt-get -y -f install; \ else \ - echo "Unsupported architecture: $TARGETARCH" >&2; \ - exit 1; \ + echo "Unsupported architecture: $TARGETARCH" >&2; \ + exit 1; \ fi - RUN apt-get update -qq && \ apt-get install -y unzip @@ -90,14 +93,13 @@ RUN apt-get update -qq && \ apt-get install -y libsasl2-modules-gssapi-mit RUN if [ "$TARGETARCH" = "amd64" ] || [ "$TARGETARCH" = "x86_64" ]; then \ - wget --quiet https://databricks-bi-artifacts.s3.us-east-2.amazonaws.com/simbaspark-drivers/odbc/2.7.7/SimbaSparkODBC-2.7.7.1016-Debian-64bit.zip -O /tmp/databricks_odbc.zip && \ - unzip /tmp/databricks_odbc.zip -d /tmp && \ - dpkg -i /tmp/simbaspark_*.deb && \ - rm -rf /tmp/*; \ + wget --quiet https://databricks-bi-artifacts.s3.us-east-2.amazonaws.com/simbaspark-drivers/odbc/2.7.7/SimbaSparkODBC-2.7.7.1016-Debian-64bit.zip -O /tmp/databricks_odbc.zip && \ + unzip /tmp/databricks_odbc.zip -d /tmp && \ + dpkg -i /tmp/simbaspark_*.deb && \ + rm -rf /tmp/*; \ fi # ARM64 version of the Simba Spark ODBC driver is not currently available - # Change back to the root directory before copying the Rails app # Rails app lives here WORKDIR /rails diff --git a/server/Dockerfile.dev b/server/Dockerfile.dev index ae2e9ac1..c3d7fe36 100644 --- a/server/Dockerfile.dev +++ b/server/Dockerfile.dev @@ -9,6 +9,10 @@ FROM registry.docker.com/library/ruby:$RUBY_VERSION-slim as base RUN apt-get update -qq && \ apt-get install --no-install-recommends -y build-essential autoconf automake libtool git libpq-dev libvips pkg-config m4 perl libltdl-dev curl git wget unzip default-libmysqlclient-dev +# Copy and run the Oracle Instant Client installation script +COPY getoracleinstantclient.sh . +RUN chmod +x getoracleinstantclient.sh && ./getoracleinstantclient.sh + COPY getduckdb.sh . COPY gethttpfsextension.sh . @@ -78,14 +82,14 @@ ENV LD_LIBRARY_PATH=/usr/local/lib:${LD_LIBRARY_PATH} ARG TARGETARCH=amd64 RUN if [ "$TARGETARCH" = "amd64" ] || [ "$TARGETARCH" = "x86_64" ]; then \ - wget https://sfc-repo.snowflakecomputing.com/odbc/linux/latest/snowflake-odbc-3.2.0.x86_64.deb -O snowflake-odbc.deb && \ - dpkg -i snowflake-odbc.deb || apt-get -y -f install; \ + wget https://sfc-repo.snowflakecomputing.com/odbc/linux/latest/snowflake-odbc-3.2.0.x86_64.deb -O snowflake-odbc.deb && \ + dpkg -i snowflake-odbc.deb || apt-get -y -f install; \ elif [ "$TARGETARCH" = "arm64" ] || [ "$TARGETARCH" = "aarch64" ]; then \ - wget https://sfc-repo.snowflakecomputing.com/odbc/linuxaarch64/3.2.0/snowflake-odbc-3.2.0.aarch64.deb -O snowflake-odbc.deb && \ - dpkg -i snowflake-odbc.deb || apt-get -y -f install; \ + wget https://sfc-repo.snowflakecomputing.com/odbc/linuxaarch64/3.2.0/snowflake-odbc-3.2.0.aarch64.deb -O snowflake-odbc.deb && \ + dpkg -i snowflake-odbc.deb || apt-get -y -f install; \ else \ - echo "Unsupported architecture: $TARGETARCH" >&2; \ - exit 1; \ + echo "Unsupported architecture: $TARGETARCH" >&2; \ + exit 1; \ fi RUN apt-get update -qq && \ @@ -95,10 +99,10 @@ RUN apt-get update -qq && \ apt-get install -y libsasl2-modules-gssapi-mit RUN if [ "$TARGETARCH" = "amd64" ] || [ "$TARGETARCH" = "x86_64" ]; then \ - wget --quiet https://databricks-bi-artifacts.s3.us-east-2.amazonaws.com/simbaspark-drivers/odbc/2.7.7/SimbaSparkODBC-2.7.7.1016-Debian-64bit.zip -O /tmp/databricks_odbc.zip && \ - unzip /tmp/databricks_odbc.zip -d /tmp && \ - dpkg -i /tmp/simbaspark_*.deb && \ - rm -rf /tmp/*; \ + wget --quiet https://databricks-bi-artifacts.s3.us-east-2.amazonaws.com/simbaspark-drivers/odbc/2.7.7/SimbaSparkODBC-2.7.7.1016-Debian-64bit.zip -O /tmp/databricks_odbc.zip && \ + unzip /tmp/databricks_odbc.zip -d /tmp && \ + dpkg -i /tmp/simbaspark_*.deb && \ + rm -rf /tmp/*; \ fi #ARM64 version of the Simba Spark ODBC driver is not currently available, diff --git a/server/Gemfile b/server/Gemfile index 61071a5a..6f715f6c 100644 --- a/server/Gemfile +++ b/server/Gemfile @@ -13,7 +13,7 @@ gem "interactor", "~> 3.0" gem "ruby-odbc", git: "https://github.com/Multiwoven/ruby-odbc.git" -gem "multiwoven-integrations", "~> 0.5.1" +gem "multiwoven-integrations", "~> 0.6.0" gem "temporal-ruby", github: "coinbase/temporal-ruby" diff --git a/server/Gemfile.lock b/server/Gemfile.lock index 3a3a9225..4cf79a94 100644 --- a/server/Gemfile.lock +++ b/server/Gemfile.lock @@ -108,7 +108,7 @@ GEM appsignal (3.7.5) rack ast (2.4.2) - async (2.14.1) + async (2.14.2) console (~> 1.25, >= 1.25.2) fiber-annotation io-event (~> 1.6, >= 1.6.5) @@ -123,8 +123,9 @@ GEM traces (>= 0.10) async-pool (0.7.0) async (>= 1.25) - async-websocket (0.27.0) + async-websocket (0.28.0) async-http (~> 0.54) + protocol-http (>= 0.28.1) protocol-rack (~> 0.5) protocol-websocket (~> 0.15) aws-eventstream (1.3.0) @@ -1661,7 +1662,7 @@ GEM activesupport concurrent-ruby (1.2.3) connection_pool (2.4.1) - console (1.25.2) + console (1.27.0) fiber-annotation fiber-local (~> 1.1) json @@ -1749,7 +1750,7 @@ GEM railties (>= 5.0.0) faker (3.2.3) i18n (>= 1.8.11, < 2) - faraday (2.10.0) + faraday (2.10.1) faraday-net_http (>= 2.0, < 3.2) logger faraday-follow_redirects (0.3.0) @@ -1759,9 +1760,12 @@ GEM hashie faraday-multipart (1.0.4) multipart-post (~> 2) - faraday-net_http (3.1.0) + faraday-net_http (3.1.1) net-http + ffi (1.17.0-aarch64-linux-gnu) ffi (1.17.0-arm64-darwin) + ffi (1.17.0-x86_64-darwin) + ffi (1.17.0-x86_64-linux-gnu) fiber-annotation (0.2.0) fiber-local (1.1.0) fiber-storage @@ -1774,20 +1778,20 @@ GEM addressable (~> 2.8) process_executer (~> 1.1) rchardet (~> 1.8) - gli (2.21.1) + gli (2.21.5) globalid (1.2.1) activesupport (>= 6.1) - google-apis-bigquery_v2 (0.72.0) + google-apis-bigquery_v2 (0.73.0) google-apis-core (>= 0.15.0, < 2.a) - google-apis-core (0.15.0) + google-apis-core (0.15.1) addressable (~> 2.5, >= 2.5.1) googleauth (~> 1.9) - httpclient (>= 2.8.1, < 3.a) + httpclient (>= 2.8.3, < 3.a) mini_mime (~> 1.0) + mutex_m representable (~> 3.0) retriable (>= 2.0, < 4.a) - rexml - google-apis-sheets_v4 (0.32.0) + google-apis-sheets_v4 (0.33.0) google-apis-core (>= 0.15.0, < 2.a) google-cloud-bigquery (1.50.0) bigdecimal (~> 3.0) @@ -1894,7 +1898,7 @@ GEM msgpack (1.7.2) multi_json (1.15.0) multipart-post (2.4.1) - multiwoven-integrations (0.5.1) + multiwoven-integrations (0.6.0) activesupport async-websocket aws-sdk-athena @@ -1915,6 +1919,7 @@ GEM rake restforce ruby-limiter + ruby-oci8 ruby-odbc rubyzip sequel @@ -1967,8 +1972,8 @@ GEM net-smtp premailer (~> 1.7, >= 1.7.9) process_executer (1.1.0) - protocol-hpack (1.4.3) - protocol-http (0.27.0) + protocol-hpack (1.5.0) + protocol-http (0.28.1) protocol-http1 (0.19.1) protocol-http (~> 0.22) protocol-http2 (0.18.0) @@ -2091,10 +2096,11 @@ GEM rubocop (>= 1.33.0, < 2.0) rubocop-ast (>= 1.30.0, < 2.0) ruby-limiter (2.3.0) + ruby-oci8 (2.2.12) ruby-progressbar (1.13.0) ruby2_keywords (0.0.5) rubyzip (2.3.2) - sequel (5.82.0) + sequel (5.83.0) bigdecimal shoulda-matchers (5.3.0) activesupport (>= 5.2.0) @@ -2117,9 +2123,9 @@ GEM faraday-multipart gli hashie - sorbet-runtime (0.5.11481) + sorbet-runtime (0.5.11504) stringio (3.1.0) - stripe (12.2.0) + stripe (12.4.0) strong_migrations (1.8.0) activerecord (>= 5.2) thor (1.3.0) @@ -2191,7 +2197,7 @@ DEPENDENCIES jwt kaminari liquid - multiwoven-integrations (~> 0.5.1) + multiwoven-integrations (~> 0.6.0) mysql2 newrelic_rpm parallel diff --git a/server/getoracleinstantclient.sh b/server/getoracleinstantclient.sh new file mode 100644 index 00000000..a7b693a6 --- /dev/null +++ b/server/getoracleinstantclient.sh @@ -0,0 +1,29 @@ +#!/bin/sh + +MACHINE=`uname -m` + +case "$MACHINE" in + "x86_64" ) ARC=x86_64 ;; + "aarch64" ) ARC=aarch64 ;; + * ) echo "Unsupported architecture: $MACHINE" >&2; exit 1 ;; +esac + +# Download basic package +if ! wget http://yum.oracle.com/repo/OracleLinux/OL7/oracle/instantclient/$ARC/getPackage/oracle-instantclient19.10-basic-19.10.0.0.0-1.$ARC.rpm; then + echo "Failed to download oracle-instantclient19.10-basic.rpm" >&2 + exit 1 +fi + +# Download devel package (repeat for devel package) +if ! wget http://yum.oracle.com/repo/OracleLinux/OL7/oracle/instantclient/$ARC/getPackage/oracle-instantclient19.10-devel-19.10.0.0.0-1.$ARC.rpm; then + echo "Failed to download oracle-instantclient19.10-devel.rpm" >&2 + exit 1 +fi + +# Install packages +apt-get update -qq && \ + apt-get install -y libaio1 alien && \ + alien -i --scripts oracle-instantclient19.10-basic-19.10.0.0.0-1.$ARC.rpm && \ + alien -i --scripts oracle-instantclient19.10-devel-19.10.0.0.0-1.$ARC.rpm && \ + rm -f oracle-instantclient19.10-basic-19.10.0.0.0-1.$ARC.rpm && \ + rm -f oracle-instantclient19.10-devel-19.10.0.0.0-1.$ARC.rpm \ No newline at end of file