From c30838279e9bf5249df92cb19d7fc33264873ccb Mon Sep 17 00:00:00 2001
From: TivonB-AI2 <124182151+TivonB-AI2@users.noreply.github.com>
Date: Fri, 2 Aug 2024 04:56:40 -0400
Subject: [PATCH 1/4] chore(CE): update server gem 0.5.2 (#275)
---
server/Gemfile | 2 +-
server/Gemfile.lock | 6 +++---
2 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/server/Gemfile b/server/Gemfile
index 61071a5a..35703cb6 100644
--- a/server/Gemfile
+++ b/server/Gemfile
@@ -13,7 +13,7 @@ gem "interactor", "~> 3.0"
gem "ruby-odbc", git: "https://github.com/Multiwoven/ruby-odbc.git"
-gem "multiwoven-integrations", "~> 0.5.1"
+gem "multiwoven-integrations", "~> 0.5.2"
gem "temporal-ruby", github: "coinbase/temporal-ruby"
diff --git a/server/Gemfile.lock b/server/Gemfile.lock
index 3a3a9225..f95ef77b 100644
--- a/server/Gemfile.lock
+++ b/server/Gemfile.lock
@@ -1774,7 +1774,7 @@ GEM
addressable (~> 2.8)
process_executer (~> 1.1)
rchardet (~> 1.8)
- gli (2.21.1)
+ gli (2.21.3)
globalid (1.2.1)
activesupport (>= 6.1)
google-apis-bigquery_v2 (0.72.0)
@@ -1894,7 +1894,7 @@ GEM
msgpack (1.7.2)
multi_json (1.15.0)
multipart-post (2.4.1)
- multiwoven-integrations (0.5.1)
+ multiwoven-integrations (0.5.2)
activesupport
async-websocket
aws-sdk-athena
@@ -2191,7 +2191,7 @@ DEPENDENCIES
jwt
kaminari
liquid
- multiwoven-integrations (~> 0.5.1)
+ multiwoven-integrations (~> 0.5.2)
mysql2
newrelic_rpm
parallel
From 68dd88dbbffbb52c232165e74d19ecf03482ffec Mon Sep 17 00:00:00 2001
From: TivonB-AI2 <124182151+TivonB-AI2@users.noreply.github.com>
Date: Fri, 2 Aug 2024 05:02:31 -0400
Subject: [PATCH 2/4] feat(CE): add oracle db destination connector (#277)
---
.github/workflows/integrations-ci.yml | 8 +
.github/workflows/integrations-main.yml | 8 +
integrations/Gemfile | 2 +
integrations/Gemfile.lock | 6 +-
integrations/lib/multiwoven/integrations.rb | 2 +
.../destination/oracle_db/client.rb | 112 ++++++++++++++
.../destination/oracle_db/config/meta.json | 15 ++
.../destination/oracle_db/config/spec.json | 47 ++++++
.../destination/oracle_db/icon.svg | 4 +
.../lib/multiwoven/integrations/rollout.rb | 3 +-
integrations/multiwoven-integrations.gemspec | 1 +
.../destination/oracle_db/client_spec.rb | 146 ++++++++++++++++++
12 files changed, 352 insertions(+), 2 deletions(-)
create mode 100644 integrations/lib/multiwoven/integrations/destination/oracle_db/client.rb
create mode 100644 integrations/lib/multiwoven/integrations/destination/oracle_db/config/meta.json
create mode 100644 integrations/lib/multiwoven/integrations/destination/oracle_db/config/spec.json
create mode 100644 integrations/lib/multiwoven/integrations/destination/oracle_db/icon.svg
create mode 100644 integrations/spec/multiwoven/integrations/destination/oracle_db/client_spec.rb
diff --git a/.github/workflows/integrations-ci.yml b/.github/workflows/integrations-ci.yml
index 4e8c44d9..1eca6149 100644
--- a/.github/workflows/integrations-ci.yml
+++ b/.github/workflows/integrations-ci.yml
@@ -31,6 +31,14 @@ jobs:
sudo mv libduckdb/libduckdb.so /usr/local/lib
sudo ldconfig /usr/local/lib
+ - name: Download and Install Oracle Instant Client
+ run: |
+ sudo apt-get install -y libaio1 alien
+ wget http://yum.oracle.com/repo/OracleLinux/OL7/oracle/instantclient/x86_64/getPackage/oracle-instantclient19.6-basic-19.6.0.0.0-1.x86_64.rpm
+ wget http://yum.oracle.com/repo/OracleLinux/OL7/oracle/instantclient/x86_64/getPackage/oracle-instantclient19.6-devel-19.6.0.0.0-1.x86_64.rpm
+ sudo alien -i --scripts oracle-instantclient*.rpm
+ rm -f oracle-instantclient*.rpm
+
- name: Install dependencies
run: |
gem install bundler
diff --git a/.github/workflows/integrations-main.yml b/.github/workflows/integrations-main.yml
index d9ecb8ad..fd43f306 100644
--- a/.github/workflows/integrations-main.yml
+++ b/.github/workflows/integrations-main.yml
@@ -39,6 +39,14 @@ jobs:
sudo mv libduckdb/libduckdb.so /usr/local/lib
sudo ldconfig /usr/local/lib
+ - name: Download and Install Oracle Instant Client
+ run: |
+ sudo apt-get install -y libaio1 alien
+ wget http://yum.oracle.com/repo/OracleLinux/OL7/oracle/instantclient/x86_64/getPackage/oracle-instantclient19.6-basic-19.6.0.0.0-1.x86_64.rpm
+ wget http://yum.oracle.com/repo/OracleLinux/OL7/oracle/instantclient/x86_64/getPackage/oracle-instantclient19.6-devel-19.6.0.0.0-1.x86_64.rpm
+ sudo alien -i --scripts oracle-instantclient*.rpm
+ rm -f oracle-instantclient*.rpm
+
- name: Install dependencies
run: bundle install
working-directory: ./integrations
diff --git a/integrations/Gemfile b/integrations/Gemfile
index b28b59ce..1b82fbae 100644
--- a/integrations/Gemfile
+++ b/integrations/Gemfile
@@ -67,6 +67,8 @@ gem "mysql2"
gem "aws-sdk-sts"
+gem "ruby-oci8"
+
group :development, :test do
gem "simplecov", require: false
gem "simplecov_json_formatter", require: false
diff --git a/integrations/Gemfile.lock b/integrations/Gemfile.lock
index fb0b8ee2..5843cd3a 100644
--- a/integrations/Gemfile.lock
+++ b/integrations/Gemfile.lock
@@ -7,7 +7,7 @@ GIT
PATH
remote: .
specs:
- multiwoven-integrations (0.5.2)
+ multiwoven-integrations (0.6.0)
activesupport
async-websocket
aws-sdk-athena
@@ -28,6 +28,7 @@ PATH
rake
restforce
ruby-limiter
+ ruby-oci8
ruby-odbc
rubyzip
sequel
@@ -275,6 +276,8 @@ GEM
rubocop-ast (1.31.3)
parser (>= 3.3.1.0)
ruby-limiter (2.3.0)
+ ruby-oci8 (2.2.12)
+ ruby-oci8 (2.2.12-x64-mingw-ucrt)
ruby-progressbar (1.13.0)
ruby2_keywords (0.0.5)
rubyzip (2.3.2)
@@ -357,6 +360,7 @@ DEPENDENCIES
rspec (~> 3.0)
rubocop (~> 1.21)
ruby-limiter
+ ruby-oci8
ruby-odbc!
rubyzip
sequel
diff --git a/integrations/lib/multiwoven/integrations.rb b/integrations/lib/multiwoven/integrations.rb
index d305120f..dd8636cb 100644
--- a/integrations/lib/multiwoven/integrations.rb
+++ b/integrations/lib/multiwoven/integrations.rb
@@ -31,6 +31,7 @@
require "duckdb"
require "iterable-api-client"
require "aws-sdk-sts"
+require "ruby-oci8"
# Service
require_relative "integrations/config"
@@ -78,6 +79,7 @@
require_relative "integrations/destination/iterable/client"
require_relative "integrations/destination/maria_db/client"
require_relative "integrations/destination/databricks_lakehouse/client"
+require_relative "integrations/destination/oracle_db/client"
module Multiwoven
module Integrations
diff --git a/integrations/lib/multiwoven/integrations/destination/oracle_db/client.rb b/integrations/lib/multiwoven/integrations/destination/oracle_db/client.rb
new file mode 100644
index 00000000..da848bf8
--- /dev/null
+++ b/integrations/lib/multiwoven/integrations/destination/oracle_db/client.rb
@@ -0,0 +1,112 @@
+# frozen_string_literal: true
+
+module Multiwoven::Integrations::Destination
+ module Oracle
+ include Multiwoven::Integrations::Core
+ class Client < DestinationConnector
+ def check_connection(connection_config)
+ connection_config = connection_config.with_indifferent_access
+ create_connection(connection_config)
+ ConnectionStatus.new(
+ status: ConnectionStatusType["succeeded"]
+ ).to_multiwoven_message
+ rescue StandardError => e
+ ConnectionStatus.new(
+ status: ConnectionStatusType["failed"], message: e.message
+ ).to_multiwoven_message
+ end
+
+ def discover(connection_config)
+ records = []
+ connection_config = connection_config.with_indifferent_access
+ query = "SELECT table_name, column_name, data_type, nullable
+ FROM all_tab_columns
+ WHERE owner = '#{connection_config[:username].upcase}'
+ ORDER BY table_name, column_id"
+ conn = create_connection(connection_config)
+ cursor = conn.exec(query)
+ while (row = cursor.fetch)
+ records << row
+ end
+ catalog = Catalog.new(streams: create_streams(records))
+ catalog.to_multiwoven_message
+ rescue StandardError => e
+ handle_exception(
+ "ORACLE:DISCOVER:EXCEPTION",
+ "error",
+ e
+ )
+ end
+
+ def write(sync_config, records, action = "destination_insert")
+ connection_config = sync_config.destination.connection_specification.with_indifferent_access
+ table_name = sync_config.stream.name
+ primary_key = sync_config.model.primary_key
+ conn = create_connection(connection_config)
+
+ write_success = 0
+ write_failure = 0
+ log_message_array = []
+
+ records.each do |record|
+ query = Multiwoven::Integrations::Core::QueryBuilder.perform(action, table_name, record, primary_key)
+ query = query.gsub(";", "")
+ logger.debug("ORACLE:WRITE:QUERY query = #{query} sync_id = #{sync_config.sync_id} sync_run_id = #{sync_config.sync_run_id}")
+ begin
+ response = conn.exec(query)
+ conn.exec("COMMIT")
+ write_success += 1
+ log_message_array << log_request_response("info", query, response)
+ rescue StandardError => e
+ handle_exception(e, {
+ context: "ORACLE:RECORD:WRITE:EXCEPTION",
+ type: "error",
+ sync_id: sync_config.sync_id,
+ sync_run_id: sync_config.sync_run_id
+ })
+ write_failure += 1
+ log_message_array << log_request_response("error", query, e.message)
+ end
+ end
+ tracking_message(write_success, write_failure, log_message_array)
+ rescue StandardError => e
+ handle_exception(e, {
+ context: "ORACLE:RECORD:WRITE:EXCEPTION",
+ type: "error",
+ sync_id: sync_config.sync_id,
+ sync_run_id: sync_config.sync_run_id
+ })
+ end
+
+ private
+
+ def create_connection(connection_config)
+ OCI8.new(connection_config[:username], connection_config[:password], "#{connection_config[:host]}:#{connection_config[:port]}/#{connection_config[:sid]}")
+ end
+
+ def create_streams(records)
+ group_by_table(records).map do |_, r|
+ Multiwoven::Integrations::Protocol::Stream.new(name: r[:tablename], action: StreamAction["fetch"], json_schema: convert_to_json_schema(r[:columns]))
+ end
+ end
+
+ def group_by_table(records)
+ result = {}
+ records.each_with_index do |entry, index|
+ table_name = entry[0]
+ column_data = {
+ column_name: entry[1],
+ data_type: entry[2],
+ is_nullable: entry[3] == "Y"
+ }
+ result[index] ||= {}
+ result[index][:tablename] = table_name
+ result[index][:columns] = [column_data]
+ end
+ result.values.group_by { |entry| entry[:tablename] }.transform_values do |entries|
+ { tablename: entries.first[:tablename], columns: entries.flat_map { |entry| entry[:columns] } }
+ end
+ end
+ end
+ end
+end
diff --git a/integrations/lib/multiwoven/integrations/destination/oracle_db/config/meta.json b/integrations/lib/multiwoven/integrations/destination/oracle_db/config/meta.json
new file mode 100644
index 00000000..5662698c
--- /dev/null
+++ b/integrations/lib/multiwoven/integrations/destination/oracle_db/config/meta.json
@@ -0,0 +1,15 @@
+{
+ "data": {
+ "name": "Oracle",
+ "title": "Oracle",
+ "connector_type": "destination",
+ "category": "Database",
+ "documentation_url": "https://docs.squared.ai/guides/data-integration/destination/oracle",
+ "github_issue_label": "destination-oracle",
+ "icon": "icon.svg",
+ "license": "MIT",
+ "release_stage": "alpha",
+ "support_level": "community",
+ "tags": ["language:ruby", "multiwoven"]
+ }
+}
diff --git a/integrations/lib/multiwoven/integrations/destination/oracle_db/config/spec.json b/integrations/lib/multiwoven/integrations/destination/oracle_db/config/spec.json
new file mode 100644
index 00000000..2eed10f6
--- /dev/null
+++ b/integrations/lib/multiwoven/integrations/destination/oracle_db/config/spec.json
@@ -0,0 +1,47 @@
+{
+ "documentation_url": "https://docs.squared.ai/guides/data-integration/destination/oracle",
+ "stream_type": "dynamic",
+ "connector_query_type": "raw_sql",
+ "connection_specification": {
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "title": "Oracle",
+ "type": "object",
+ "required": ["host", "port", "sid", "username", "password"],
+ "properties": {
+ "host": {
+ "description": "The Oracle host.",
+ "examples": ["localhost"],
+ "type": "string",
+ "title": "Host",
+ "order": 0
+ },
+ "port": {
+ "description": "The Oracle port number.",
+ "examples": ["1521"],
+ "type": "string",
+ "title": "Port",
+ "order": 1
+ },
+ "sid": {
+ "description": "The name of your service in Oracle.",
+ "examples": ["ORCLPDB1"],
+ "type": "string",
+ "title": "SID",
+ "order": 2
+ },
+ "username": {
+ "description": "The username used to authenticate and connect.",
+ "type": "string",
+ "title": "Username",
+ "order": 3
+ },
+ "password": {
+ "description": "The password corresponding to the username used for authentication.",
+ "type": "string",
+ "multiwoven_secret": true,
+ "title": "Password",
+ "order": 4
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/integrations/lib/multiwoven/integrations/destination/oracle_db/icon.svg b/integrations/lib/multiwoven/integrations/destination/oracle_db/icon.svg
new file mode 100644
index 00000000..3f4e051f
--- /dev/null
+++ b/integrations/lib/multiwoven/integrations/destination/oracle_db/icon.svg
@@ -0,0 +1,4 @@
+
+
\ No newline at end of file
diff --git a/integrations/lib/multiwoven/integrations/rollout.rb b/integrations/lib/multiwoven/integrations/rollout.rb
index 76ece7b2..03ec5a0a 100644
--- a/integrations/lib/multiwoven/integrations/rollout.rb
+++ b/integrations/lib/multiwoven/integrations/rollout.rb
@@ -2,7 +2,7 @@
module Multiwoven
module Integrations
- VERSION = "0.5.2"
+ VERSION = "0.6.0"
ENABLED_SOURCES = %w[
Snowflake
@@ -34,6 +34,7 @@ module Integrations
Iterable
MariaDB
DatabricksLakehouse
+ Oracle
].freeze
end
end
diff --git a/integrations/multiwoven-integrations.gemspec b/integrations/multiwoven-integrations.gemspec
index 34aec384..498a2d19 100644
--- a/integrations/multiwoven-integrations.gemspec
+++ b/integrations/multiwoven-integrations.gemspec
@@ -53,6 +53,7 @@ Gem::Specification.new do |spec|
spec.add_runtime_dependency "rake"
spec.add_runtime_dependency "restforce"
spec.add_runtime_dependency "ruby-limiter"
+ spec.add_runtime_dependency "ruby-oci8"
spec.add_runtime_dependency "ruby-odbc"
spec.add_runtime_dependency "rubyzip"
spec.add_runtime_dependency "sequel"
diff --git a/integrations/spec/multiwoven/integrations/destination/oracle_db/client_spec.rb b/integrations/spec/multiwoven/integrations/destination/oracle_db/client_spec.rb
new file mode 100644
index 00000000..7774e2e5
--- /dev/null
+++ b/integrations/spec/multiwoven/integrations/destination/oracle_db/client_spec.rb
@@ -0,0 +1,146 @@
+# frozen_string_literal: true
+
+RSpec.describe Multiwoven::Integrations::Destination::Oracle::Client do
+ include WebMock::API
+
+ before(:each) do
+ WebMock.disable_net_connect!(allow_localhost: true)
+ end
+
+ let(:client) { described_class.new }
+ let(:connection_config) do
+ {
+ host: "localhost",
+ port: "1521",
+ servicename: "PDB1",
+ username: "oracle_user",
+ password: "oracle_password"
+ }
+ end
+ let(:sync_config_json) do
+ {
+ source: {
+ name: "Sample Source Connector",
+ type: "source",
+ connection_specification: {
+ private_api_key: "test_api_key"
+ }
+ },
+ destination: {
+ name: "Databricks",
+ type: "destination",
+ connection_specification: connection_config
+ },
+ model: {
+ name: "ExampleModel",
+ query: "SELECT col1, col2, col3 FROM test_table_1",
+ query_type: "raw_sql",
+ primary_key: "col1"
+ },
+ sync_mode: "incremental",
+ destination_sync_mode: "insert",
+ stream: {
+ name: "table",
+ action: "create",
+ json_schema: {},
+ supported_sync_modes: %w[incremental]
+ }
+ }
+ end
+
+ let(:oracle_connection) { instance_double(OCI8) }
+ let(:cursor) { instance_double("OCI8::Cursor") }
+
+ describe "#check_connection" do
+ context "when the connection is successful" do
+ it "returns a succeeded connection status" do
+ allow(OCI8).to receive(:new).and_return(oracle_connection)
+ allow(oracle_connection).to receive(:exec).and_return(true)
+ message = client.check_connection(sync_config_json[:destination][:connection_specification])
+ result = message.connection_status
+ expect(result.status).to eq("succeeded")
+ expect(result.message).to be_nil
+ end
+ end
+
+ context "when the connection fails" do
+ it "returns a failed connection status with an error message" do
+ allow(client).to receive(:create_connection).and_raise(StandardError, "Connection failed")
+ message = client.check_connection(sync_config_json[:destination][:connection_specification])
+ result = message.connection_status
+ expect(result.status).to eq("failed")
+ expect(result.message).to include("Connection failed")
+ end
+ end
+ end
+
+ describe "#discover" do
+ it "discovers schema successfully" do
+ response = %w[test_table col1 NUMBER Y]
+ allow(OCI8).to receive(:new).and_return(oracle_connection)
+ allow(oracle_connection).to receive(:exec).and_return(cursor)
+ allow(cursor).to receive(:fetch).and_return(response, nil)
+ message = client.discover(sync_config_json[:destination][:connection_specification])
+ expect(message.catalog).to be_an(Multiwoven::Integrations::Protocol::Catalog)
+ first_stream = message.catalog.streams.first
+ expect(first_stream).to be_a(Multiwoven::Integrations::Protocol::Stream)
+ expect(first_stream.name).to eq("test_table")
+ expect(first_stream.json_schema).to be_an(Hash)
+ expect(first_stream.json_schema["type"]).to eq("object")
+ expect(first_stream.json_schema["properties"]).to eq({ "col1" => { "type" => "string" } })
+ end
+ end
+
+ describe "#write" do
+ context "when the write operation is successful" do
+ it "increments the success count" do
+ sync_config = Multiwoven::Integrations::Protocol::SyncConfig.from_json(
+ sync_config_json.to_json
+ )
+ record = [
+ { "col1" => 1, "col2" => "first", "col3" => 1.1 }
+ ]
+ allow(OCI8).to receive(:new).and_return(oracle_connection)
+ allow(oracle_connection).to receive(:exec).and_return(1)
+ allow(cursor).to receive(:fetch).and_return(1, nil)
+ response = client.write(sync_config, record)
+ expect(response.tracking.success).to eq(record.size)
+ expect(response.tracking.failed).to eq(0)
+ log_message = response.tracking.logs.first
+ expect(log_message).to be_a(Multiwoven::Integrations::Protocol::LogMessage)
+ expect(log_message.level).to eql("info")
+ expect(log_message.message).to include("request")
+ expect(log_message.message).to include("response")
+ end
+ end
+
+ context "when the write operation fails" do
+ it "increments the failure count" do
+ sync_config = Multiwoven::Integrations::Protocol::SyncConfig.from_json(
+ sync_config_json.to_json
+ )
+ record = [
+ { "col1" => 1, "col2" => "first", "col3" => 1.1 }
+ ]
+ allow(OCI8).to receive(:new).and_return(oracle_connection)
+ allow(oracle_connection).to receive(:exec).and_raise(StandardError, "Test error")
+ response = client.write(sync_config, record)
+ expect(response.tracking.failed).to eq(record.size)
+ expect(response.tracking.success).to eq(0)
+ log_message = response.tracking.logs.first
+ expect(log_message).to be_a(Multiwoven::Integrations::Protocol::LogMessage)
+ expect(log_message.level).to eql("error")
+ expect(log_message.message).to include("request")
+ expect(log_message.message).to include("{\"request\":\"INSERT INTO table (col1, col2, col3) VALUES ('1', 'first', '1.1')\",\"response\":\"Test error\",\"level\":\"error\"}")
+ end
+ end
+ end
+
+ describe "#meta_data" do
+ # change this to rollout validation for all connector rolling out
+ it "client class_name and meta name is same" do
+ meta_name = client.class.to_s.split("::")[-2]
+ expect(client.send(:meta_data)[:data][:name]).to eq(meta_name)
+ end
+ end
+end
From 108cc8208562e2b922933541a4878a186f8cd81e Mon Sep 17 00:00:00 2001
From: TivonB-AI2 <124182151+TivonB-AI2@users.noreply.github.com>
Date: Fri, 2 Aug 2024 05:25:51 -0400
Subject: [PATCH 3/4] chore(CE): update gem 0.6.0 (#278)
---
.github/workflows/server-ci.yml | 8 ++++++
server/Dockerfile | 26 +++++++++++---------
server/Dockerfile.dev | 24 ++++++++++--------
server/Gemfile | 2 +-
server/Gemfile.lock | 42 ++++++++++++++++++--------------
server/getoracleinstantclient.sh | 29 ++++++++++++++++++++++
6 files changed, 90 insertions(+), 41 deletions(-)
create mode 100644 server/getoracleinstantclient.sh
diff --git a/.github/workflows/server-ci.yml b/.github/workflows/server-ci.yml
index 133e7399..176bacc6 100644
--- a/.github/workflows/server-ci.yml
+++ b/.github/workflows/server-ci.yml
@@ -50,6 +50,14 @@ jobs:
sudo mv libduckdb/libduckdb.so /usr/local/lib
sudo ldconfig /usr/local/lib
+ - name: Download and Install Oracle Instant Client
+ run: |
+ sudo apt-get install -y libaio1 alien
+ wget http://yum.oracle.com/repo/OracleLinux/OL7/oracle/instantclient/x86_64/getPackage/oracle-instantclient19.6-basic-19.6.0.0.0-1.x86_64.rpm
+ wget http://yum.oracle.com/repo/OracleLinux/OL7/oracle/instantclient/x86_64/getPackage/oracle-instantclient19.6-devel-19.6.0.0.0-1.x86_64.rpm
+ sudo alien -i --scripts oracle-instantclient*.rpm
+ rm -f oracle-instantclient*.rpm
+
- name: Bundle Install
run: bundle install
working-directory: ./server
diff --git a/server/Dockerfile b/server/Dockerfile
index 3f9b9c4b..1615d389 100644
--- a/server/Dockerfile
+++ b/server/Dockerfile
@@ -20,6 +20,10 @@ FROM base as build
RUN apt-get update -qq && \
apt-get install --no-install-recommends -y build-essential autoconf automake libtool git libpq-dev libvips pkg-config m4 perl libltdl-dev curl git wget unzip default-libmysqlclient-dev
+# Copy and run the Oracle Instant Client installation script
+COPY getoracleinstantclient.sh .
+RUN chmod +x getoracleinstantclient.sh && ./getoracleinstantclient.sh
+
COPY getduckdb.sh .
COPY gethttpfsextension.sh .
@@ -72,17 +76,16 @@ ENV LD_LIBRARY_PATH=/usr/local/lib:${LD_LIBRARY_PATH}
ARG TARGETARCH=amd64
RUN if [ "$TARGETARCH" = "amd64" ] || [ "$TARGETARCH" = "x86_64" ]; then \
- wget https://sfc-repo.snowflakecomputing.com/odbc/linux/latest/snowflake-odbc-3.2.0.x86_64.deb -O snowflake-odbc.deb && \
- dpkg -i snowflake-odbc.deb || apt-get -y -f install; \
+ wget https://sfc-repo.snowflakecomputing.com/odbc/linux/latest/snowflake-odbc-3.2.0.x86_64.deb -O snowflake-odbc.deb && \
+ dpkg -i snowflake-odbc.deb || apt-get -y -f install; \
elif [ "$TARGETARCH" = "arm64" ] || [ "$TARGETARCH" = "aarch64" ]; then \
- wget https://sfc-repo.snowflakecomputing.com/odbc/linuxaarch64/3.2.0/snowflake-odbc-3.2.0.aarch64.deb -O snowflake-odbc.deb && \
- dpkg -i snowflake-odbc.deb || apt-get -y -f install; \
+ wget https://sfc-repo.snowflakecomputing.com/odbc/linuxaarch64/3.2.0/snowflake-odbc-3.2.0.aarch64.deb -O snowflake-odbc.deb && \
+ dpkg -i snowflake-odbc.deb || apt-get -y -f install; \
else \
- echo "Unsupported architecture: $TARGETARCH" >&2; \
- exit 1; \
+ echo "Unsupported architecture: $TARGETARCH" >&2; \
+ exit 1; \
fi
-
RUN apt-get update -qq && \
apt-get install -y unzip
@@ -90,14 +93,13 @@ RUN apt-get update -qq && \
apt-get install -y libsasl2-modules-gssapi-mit
RUN if [ "$TARGETARCH" = "amd64" ] || [ "$TARGETARCH" = "x86_64" ]; then \
- wget --quiet https://databricks-bi-artifacts.s3.us-east-2.amazonaws.com/simbaspark-drivers/odbc/2.7.7/SimbaSparkODBC-2.7.7.1016-Debian-64bit.zip -O /tmp/databricks_odbc.zip && \
- unzip /tmp/databricks_odbc.zip -d /tmp && \
- dpkg -i /tmp/simbaspark_*.deb && \
- rm -rf /tmp/*; \
+ wget --quiet https://databricks-bi-artifacts.s3.us-east-2.amazonaws.com/simbaspark-drivers/odbc/2.7.7/SimbaSparkODBC-2.7.7.1016-Debian-64bit.zip -O /tmp/databricks_odbc.zip && \
+ unzip /tmp/databricks_odbc.zip -d /tmp && \
+ dpkg -i /tmp/simbaspark_*.deb && \
+ rm -rf /tmp/*; \
fi
# ARM64 version of the Simba Spark ODBC driver is not currently available
-
# Change back to the root directory before copying the Rails app
# Rails app lives here
WORKDIR /rails
diff --git a/server/Dockerfile.dev b/server/Dockerfile.dev
index ae2e9ac1..c3d7fe36 100644
--- a/server/Dockerfile.dev
+++ b/server/Dockerfile.dev
@@ -9,6 +9,10 @@ FROM registry.docker.com/library/ruby:$RUBY_VERSION-slim as base
RUN apt-get update -qq && \
apt-get install --no-install-recommends -y build-essential autoconf automake libtool git libpq-dev libvips pkg-config m4 perl libltdl-dev curl git wget unzip default-libmysqlclient-dev
+# Copy and run the Oracle Instant Client installation script
+COPY getoracleinstantclient.sh .
+RUN chmod +x getoracleinstantclient.sh && ./getoracleinstantclient.sh
+
COPY getduckdb.sh .
COPY gethttpfsextension.sh .
@@ -78,14 +82,14 @@ ENV LD_LIBRARY_PATH=/usr/local/lib:${LD_LIBRARY_PATH}
ARG TARGETARCH=amd64
RUN if [ "$TARGETARCH" = "amd64" ] || [ "$TARGETARCH" = "x86_64" ]; then \
- wget https://sfc-repo.snowflakecomputing.com/odbc/linux/latest/snowflake-odbc-3.2.0.x86_64.deb -O snowflake-odbc.deb && \
- dpkg -i snowflake-odbc.deb || apt-get -y -f install; \
+ wget https://sfc-repo.snowflakecomputing.com/odbc/linux/latest/snowflake-odbc-3.2.0.x86_64.deb -O snowflake-odbc.deb && \
+ dpkg -i snowflake-odbc.deb || apt-get -y -f install; \
elif [ "$TARGETARCH" = "arm64" ] || [ "$TARGETARCH" = "aarch64" ]; then \
- wget https://sfc-repo.snowflakecomputing.com/odbc/linuxaarch64/3.2.0/snowflake-odbc-3.2.0.aarch64.deb -O snowflake-odbc.deb && \
- dpkg -i snowflake-odbc.deb || apt-get -y -f install; \
+ wget https://sfc-repo.snowflakecomputing.com/odbc/linuxaarch64/3.2.0/snowflake-odbc-3.2.0.aarch64.deb -O snowflake-odbc.deb && \
+ dpkg -i snowflake-odbc.deb || apt-get -y -f install; \
else \
- echo "Unsupported architecture: $TARGETARCH" >&2; \
- exit 1; \
+ echo "Unsupported architecture: $TARGETARCH" >&2; \
+ exit 1; \
fi
RUN apt-get update -qq && \
@@ -95,10 +99,10 @@ RUN apt-get update -qq && \
apt-get install -y libsasl2-modules-gssapi-mit
RUN if [ "$TARGETARCH" = "amd64" ] || [ "$TARGETARCH" = "x86_64" ]; then \
- wget --quiet https://databricks-bi-artifacts.s3.us-east-2.amazonaws.com/simbaspark-drivers/odbc/2.7.7/SimbaSparkODBC-2.7.7.1016-Debian-64bit.zip -O /tmp/databricks_odbc.zip && \
- unzip /tmp/databricks_odbc.zip -d /tmp && \
- dpkg -i /tmp/simbaspark_*.deb && \
- rm -rf /tmp/*; \
+ wget --quiet https://databricks-bi-artifacts.s3.us-east-2.amazonaws.com/simbaspark-drivers/odbc/2.7.7/SimbaSparkODBC-2.7.7.1016-Debian-64bit.zip -O /tmp/databricks_odbc.zip && \
+ unzip /tmp/databricks_odbc.zip -d /tmp && \
+ dpkg -i /tmp/simbaspark_*.deb && \
+ rm -rf /tmp/*; \
fi
#ARM64 version of the Simba Spark ODBC driver is not currently available,
diff --git a/server/Gemfile b/server/Gemfile
index 35703cb6..6f715f6c 100644
--- a/server/Gemfile
+++ b/server/Gemfile
@@ -13,7 +13,7 @@ gem "interactor", "~> 3.0"
gem "ruby-odbc", git: "https://github.com/Multiwoven/ruby-odbc.git"
-gem "multiwoven-integrations", "~> 0.5.2"
+gem "multiwoven-integrations", "~> 0.6.0"
gem "temporal-ruby", github: "coinbase/temporal-ruby"
diff --git a/server/Gemfile.lock b/server/Gemfile.lock
index f95ef77b..4cf79a94 100644
--- a/server/Gemfile.lock
+++ b/server/Gemfile.lock
@@ -108,7 +108,7 @@ GEM
appsignal (3.7.5)
rack
ast (2.4.2)
- async (2.14.1)
+ async (2.14.2)
console (~> 1.25, >= 1.25.2)
fiber-annotation
io-event (~> 1.6, >= 1.6.5)
@@ -123,8 +123,9 @@ GEM
traces (>= 0.10)
async-pool (0.7.0)
async (>= 1.25)
- async-websocket (0.27.0)
+ async-websocket (0.28.0)
async-http (~> 0.54)
+ protocol-http (>= 0.28.1)
protocol-rack (~> 0.5)
protocol-websocket (~> 0.15)
aws-eventstream (1.3.0)
@@ -1661,7 +1662,7 @@ GEM
activesupport
concurrent-ruby (1.2.3)
connection_pool (2.4.1)
- console (1.25.2)
+ console (1.27.0)
fiber-annotation
fiber-local (~> 1.1)
json
@@ -1749,7 +1750,7 @@ GEM
railties (>= 5.0.0)
faker (3.2.3)
i18n (>= 1.8.11, < 2)
- faraday (2.10.0)
+ faraday (2.10.1)
faraday-net_http (>= 2.0, < 3.2)
logger
faraday-follow_redirects (0.3.0)
@@ -1759,9 +1760,12 @@ GEM
hashie
faraday-multipart (1.0.4)
multipart-post (~> 2)
- faraday-net_http (3.1.0)
+ faraday-net_http (3.1.1)
net-http
+ ffi (1.17.0-aarch64-linux-gnu)
ffi (1.17.0-arm64-darwin)
+ ffi (1.17.0-x86_64-darwin)
+ ffi (1.17.0-x86_64-linux-gnu)
fiber-annotation (0.2.0)
fiber-local (1.1.0)
fiber-storage
@@ -1774,20 +1778,20 @@ GEM
addressable (~> 2.8)
process_executer (~> 1.1)
rchardet (~> 1.8)
- gli (2.21.3)
+ gli (2.21.5)
globalid (1.2.1)
activesupport (>= 6.1)
- google-apis-bigquery_v2 (0.72.0)
+ google-apis-bigquery_v2 (0.73.0)
google-apis-core (>= 0.15.0, < 2.a)
- google-apis-core (0.15.0)
+ google-apis-core (0.15.1)
addressable (~> 2.5, >= 2.5.1)
googleauth (~> 1.9)
- httpclient (>= 2.8.1, < 3.a)
+ httpclient (>= 2.8.3, < 3.a)
mini_mime (~> 1.0)
+ mutex_m
representable (~> 3.0)
retriable (>= 2.0, < 4.a)
- rexml
- google-apis-sheets_v4 (0.32.0)
+ google-apis-sheets_v4 (0.33.0)
google-apis-core (>= 0.15.0, < 2.a)
google-cloud-bigquery (1.50.0)
bigdecimal (~> 3.0)
@@ -1894,7 +1898,7 @@ GEM
msgpack (1.7.2)
multi_json (1.15.0)
multipart-post (2.4.1)
- multiwoven-integrations (0.5.2)
+ multiwoven-integrations (0.6.0)
activesupport
async-websocket
aws-sdk-athena
@@ -1915,6 +1919,7 @@ GEM
rake
restforce
ruby-limiter
+ ruby-oci8
ruby-odbc
rubyzip
sequel
@@ -1967,8 +1972,8 @@ GEM
net-smtp
premailer (~> 1.7, >= 1.7.9)
process_executer (1.1.0)
- protocol-hpack (1.4.3)
- protocol-http (0.27.0)
+ protocol-hpack (1.5.0)
+ protocol-http (0.28.1)
protocol-http1 (0.19.1)
protocol-http (~> 0.22)
protocol-http2 (0.18.0)
@@ -2091,10 +2096,11 @@ GEM
rubocop (>= 1.33.0, < 2.0)
rubocop-ast (>= 1.30.0, < 2.0)
ruby-limiter (2.3.0)
+ ruby-oci8 (2.2.12)
ruby-progressbar (1.13.0)
ruby2_keywords (0.0.5)
rubyzip (2.3.2)
- sequel (5.82.0)
+ sequel (5.83.0)
bigdecimal
shoulda-matchers (5.3.0)
activesupport (>= 5.2.0)
@@ -2117,9 +2123,9 @@ GEM
faraday-multipart
gli
hashie
- sorbet-runtime (0.5.11481)
+ sorbet-runtime (0.5.11504)
stringio (3.1.0)
- stripe (12.2.0)
+ stripe (12.4.0)
strong_migrations (1.8.0)
activerecord (>= 5.2)
thor (1.3.0)
@@ -2191,7 +2197,7 @@ DEPENDENCIES
jwt
kaminari
liquid
- multiwoven-integrations (~> 0.5.2)
+ multiwoven-integrations (~> 0.6.0)
mysql2
newrelic_rpm
parallel
diff --git a/server/getoracleinstantclient.sh b/server/getoracleinstantclient.sh
new file mode 100644
index 00000000..a7b693a6
--- /dev/null
+++ b/server/getoracleinstantclient.sh
@@ -0,0 +1,29 @@
+#!/bin/sh
+
+MACHINE=`uname -m`
+
+case "$MACHINE" in
+ "x86_64" ) ARC=x86_64 ;;
+ "aarch64" ) ARC=aarch64 ;;
+ * ) echo "Unsupported architecture: $MACHINE" >&2; exit 1 ;;
+esac
+
+# Download basic package
+if ! wget http://yum.oracle.com/repo/OracleLinux/OL7/oracle/instantclient/$ARC/getPackage/oracle-instantclient19.10-basic-19.10.0.0.0-1.$ARC.rpm; then
+ echo "Failed to download oracle-instantclient19.10-basic.rpm" >&2
+ exit 1
+fi
+
+# Download devel package (repeat for devel package)
+if ! wget http://yum.oracle.com/repo/OracleLinux/OL7/oracle/instantclient/$ARC/getPackage/oracle-instantclient19.10-devel-19.10.0.0.0-1.$ARC.rpm; then
+ echo "Failed to download oracle-instantclient19.10-devel.rpm" >&2
+ exit 1
+fi
+
+# Install packages
+apt-get update -qq && \
+ apt-get install -y libaio1 alien && \
+ alien -i --scripts oracle-instantclient19.10-basic-19.10.0.0.0-1.$ARC.rpm && \
+ alien -i --scripts oracle-instantclient19.10-devel-19.10.0.0.0-1.$ARC.rpm && \
+ rm -f oracle-instantclient19.10-basic-19.10.0.0.0-1.$ARC.rpm && \
+ rm -f oracle-instantclient19.10-devel-19.10.0.0.0-1.$ARC.rpm
\ No newline at end of file
From a74d8411fa66862babc15dbef3f399eb3fe01b14 Mon Sep 17 00:00:00 2001
From: "github-actions[bot]"
<41898282+github-actions[bot]@users.noreply.github.com>
Date: Fri, 2 Aug 2024 14:56:37 +0530
Subject: [PATCH 4/4] feat(CE): add oracle db source connector (#274)
Co-authored-by: TivonB-AI2 <124182151+TivonB-AI2@users.noreply.github.com>
---
integrations/Gemfile.lock | 2 +-
integrations/lib/multiwoven/integrations.rb | 1 +
.../lib/multiwoven/integrations/rollout.rb | 3 +-
.../integrations/source/oracle_db/client.rb | 127 +++++++++++++++
.../source/oracle_db/config/meta.json | 15 ++
.../source/oracle_db/config/spec.json | 47 ++++++
.../integrations/source/oracle_db/icon.svg | 4 +
.../source/oracle_db/client_spec.rb | 153 ++++++++++++++++++
8 files changed, 350 insertions(+), 2 deletions(-)
create mode 100644 integrations/lib/multiwoven/integrations/source/oracle_db/client.rb
create mode 100644 integrations/lib/multiwoven/integrations/source/oracle_db/config/meta.json
create mode 100644 integrations/lib/multiwoven/integrations/source/oracle_db/config/spec.json
create mode 100644 integrations/lib/multiwoven/integrations/source/oracle_db/icon.svg
create mode 100644 integrations/spec/multiwoven/integrations/source/oracle_db/client_spec.rb
diff --git a/integrations/Gemfile.lock b/integrations/Gemfile.lock
index 5843cd3a..23ac9eee 100644
--- a/integrations/Gemfile.lock
+++ b/integrations/Gemfile.lock
@@ -7,7 +7,7 @@ GIT
PATH
remote: .
specs:
- multiwoven-integrations (0.6.0)
+ multiwoven-integrations (0.7.0)
activesupport
async-websocket
aws-sdk-athena
diff --git a/integrations/lib/multiwoven/integrations.rb b/integrations/lib/multiwoven/integrations.rb
index dd8636cb..4f80fab4 100644
--- a/integrations/lib/multiwoven/integrations.rb
+++ b/integrations/lib/multiwoven/integrations.rb
@@ -61,6 +61,7 @@
require_relative "integrations/source/clickhouse/client"
require_relative "integrations/source/amazon_s3/client"
require_relative "integrations/source/maria_db/client"
+require_relative "integrations/source/oracle_db/client"
# Destination
require_relative "integrations/destination/klaviyo/client"
diff --git a/integrations/lib/multiwoven/integrations/rollout.rb b/integrations/lib/multiwoven/integrations/rollout.rb
index 03ec5a0a..45d2b8b2 100644
--- a/integrations/lib/multiwoven/integrations/rollout.rb
+++ b/integrations/lib/multiwoven/integrations/rollout.rb
@@ -2,7 +2,7 @@
module Multiwoven
module Integrations
- VERSION = "0.6.0"
+ VERSION = "0.7.0"
ENABLED_SOURCES = %w[
Snowflake
@@ -15,6 +15,7 @@ module Integrations
Clickhouse
AmazonS3
MariaDB
+ Oracle
].freeze
ENABLED_DESTINATIONS = %w[
diff --git a/integrations/lib/multiwoven/integrations/source/oracle_db/client.rb b/integrations/lib/multiwoven/integrations/source/oracle_db/client.rb
new file mode 100644
index 00000000..f5d787bc
--- /dev/null
+++ b/integrations/lib/multiwoven/integrations/source/oracle_db/client.rb
@@ -0,0 +1,127 @@
+# frozen_string_literal: true
+
+module Multiwoven::Integrations::Source
+ module Oracle
+ include Multiwoven::Integrations::Core
+ class Client < SourceConnector
+ def check_connection(connection_config)
+ connection_config = connection_config.with_indifferent_access
+ create_connection(connection_config)
+ ConnectionStatus.new(
+ status: ConnectionStatusType["succeeded"]
+ ).to_multiwoven_message
+ rescue StandardError => e
+ ConnectionStatus.new(
+ status: ConnectionStatusType["failed"], message: e.message
+ ).to_multiwoven_message
+ end
+
+ def discover(connection_config)
+ records = []
+ connection_config = connection_config.with_indifferent_access
+ query = "SELECT table_name, column_name, data_type, nullable
+ FROM all_tab_columns
+ WHERE owner = '#{connection_config[:username].upcase}'
+ ORDER BY table_name, column_id"
+ conn = create_connection(connection_config)
+ cursor = conn.exec(query)
+ while (row = cursor.fetch)
+ records << row
+ end
+ catalog = Catalog.new(streams: create_streams(records))
+ catalog.to_multiwoven_message
+ rescue StandardError => e
+ handle_exception(
+ "ORACLE:DISCOVER:EXCEPTION",
+ "error",
+ e
+ )
+ end
+
+ def read(sync_config)
+ connection_config = sync_config.source.connection_specification.with_indifferent_access
+ query = sync_config.model.query
+ db = create_connection(connection_config)
+ query(db, query)
+ rescue StandardError => e
+ handle_exception(e, {
+ context: "ORACLE:READ:EXCEPTION",
+ type: "error",
+ sync_id: sync_config.sync_id,
+ sync_run_id: sync_config.sync_run_id
+ })
+ end
+
+ private
+
+ def create_connection(connection_config)
+ OCI8.new(connection_config[:username], connection_config[:password], "#{connection_config[:host]}:#{connection_config[:port]}/#{connection_config[:sid]}")
+ end
+
+ def create_streams(records)
+ group_by_table(records).map do |_, r|
+ Multiwoven::Integrations::Protocol::Stream.new(name: r[:tablename], action: StreamAction["fetch"], json_schema: convert_to_json_schema(r[:columns]))
+ end
+ end
+
+ def query(connection, query)
+ records = []
+ query = reformat_query(query)
+ cursor = connection.exec(query)
+ columns = cursor.get_col_names
+ while (row = cursor.fetch)
+ data_hash = columns.zip(row).to_h
+ records << RecordMessage.new(data: data_hash, emitted_at: Time.now.to_i).to_multiwoven_message
+ end
+ records
+ end
+
+ def group_by_table(records)
+ result = {}
+ records.each_with_index do |entry, index|
+ table_name = entry[0]
+ column_data = {
+ column_name: entry[1],
+ data_type: entry[2],
+ is_nullable: entry[3] == "Y"
+ }
+ result[index] ||= {}
+ result[index][:tablename] = table_name
+ result[index][:columns] = [column_data]
+ end
+ result.values.group_by { |entry| entry[:tablename] }.transform_values do |entries|
+ { tablename: entries.first[:tablename], columns: entries.flat_map { |entry| entry[:columns] } }
+ end
+ end
+
+ def reformat_query(sql_query)
+ offset = nil
+ limit = nil
+
+ sql_query = sql_query.gsub(";", "")
+
+ if sql_query.match?(/LIMIT (\d+)/i)
+ limit = sql_query.match(/LIMIT (\d+)/i)[1].to_i
+ sql_query.sub!(/LIMIT \d+/i, "")
+ end
+
+ if sql_query.match?(/OFFSET (\d+)/i)
+ offset = sql_query.match(/OFFSET (\d+)/i)[1].to_i
+ sql_query.sub!(/OFFSET \d+/i, "")
+ end
+
+ sql_query.strip!
+
+ if offset && limit
+ "#{sql_query} OFFSET #{offset} ROWS FETCH NEXT #{limit} ROWS ONLY"
+ elsif offset
+ "#{sql_query} OFFSET #{offset} ROWS"
+ elsif limit
+ "#{sql_query} FETCH NEXT #{limit} ROWS ONLY"
+ else
+ sql_query
+ end
+ end
+ end
+ end
+end
diff --git a/integrations/lib/multiwoven/integrations/source/oracle_db/config/meta.json b/integrations/lib/multiwoven/integrations/source/oracle_db/config/meta.json
new file mode 100644
index 00000000..833d36f0
--- /dev/null
+++ b/integrations/lib/multiwoven/integrations/source/oracle_db/config/meta.json
@@ -0,0 +1,15 @@
+{
+ "data": {
+ "name": "Oracle",
+ "title": "Oracle",
+ "connector_type": "source",
+ "category": "Database",
+ "documentation_url": "https://docs.squared.ai/guides/data-integration/source/oracle",
+ "github_issue_label": "source-oracle",
+ "icon": "icon.svg",
+ "license": "MIT",
+ "release_stage": "alpha",
+ "support_level": "community",
+ "tags": ["language:ruby", "multiwoven"]
+ }
+}
diff --git a/integrations/lib/multiwoven/integrations/source/oracle_db/config/spec.json b/integrations/lib/multiwoven/integrations/source/oracle_db/config/spec.json
new file mode 100644
index 00000000..703791b7
--- /dev/null
+++ b/integrations/lib/multiwoven/integrations/source/oracle_db/config/spec.json
@@ -0,0 +1,47 @@
+{
+ "documentation_url": "https://docs.squared.ai/guides/data-integration/source/oracle",
+ "stream_type": "dynamic",
+ "connector_query_type": "raw_sql",
+ "connection_specification": {
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "title": "Oracle",
+ "type": "object",
+ "required": ["host", "port", "sid", "username", "password"],
+ "properties": {
+ "host": {
+ "description": "The Oracle host.",
+ "examples": ["localhost"],
+ "type": "string",
+ "title": "Host",
+ "order": 0
+ },
+ "port": {
+ "description": "The Oracle port number.",
+ "examples": ["1521"],
+ "type": "string",
+ "title": "Port",
+ "order": 1
+ },
+ "sid": {
+ "description": "The name of your service in Oracle.",
+ "examples": ["ORCLPDB1"],
+ "type": "string",
+ "title": "SID",
+ "order": 2
+ },
+ "username": {
+ "description": "The username used to authenticate and connect.",
+ "type": "string",
+ "title": "Username",
+ "order": 3
+ },
+ "password": {
+ "description": "The password corresponding to the username used for authentication.",
+ "type": "string",
+ "multiwoven_secret": true,
+ "title": "Password",
+ "order": 4
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/integrations/lib/multiwoven/integrations/source/oracle_db/icon.svg b/integrations/lib/multiwoven/integrations/source/oracle_db/icon.svg
new file mode 100644
index 00000000..3f4e051f
--- /dev/null
+++ b/integrations/lib/multiwoven/integrations/source/oracle_db/icon.svg
@@ -0,0 +1,4 @@
+
+
\ No newline at end of file
diff --git a/integrations/spec/multiwoven/integrations/source/oracle_db/client_spec.rb b/integrations/spec/multiwoven/integrations/source/oracle_db/client_spec.rb
new file mode 100644
index 00000000..51b9af14
--- /dev/null
+++ b/integrations/spec/multiwoven/integrations/source/oracle_db/client_spec.rb
@@ -0,0 +1,153 @@
+# frozen_string_literal: true
+
+RSpec.describe Multiwoven::Integrations::Source::Oracle::Client do
+ let(:client) { Multiwoven::Integrations::Source::Oracle::Client.new }
+ let(:sync_config) do
+ {
+ "source": {
+ "name": "OracleConnector",
+ "type": "source",
+ "connection_specification": {
+ "host": "localhost",
+ "port": "1521",
+ "servicename": "PDB1",
+ "username": "oracle_user",
+ "password": "oracle_password"
+ }
+ },
+ "destination": {
+ "name": "DestinationConnectorName",
+ "type": "destination",
+ "connection_specification": {
+ "example_destination_key": "example_destination_value"
+ }
+ },
+ "model": {
+ "name": "OracleDB Model",
+ "query": "SELECT col1, col2, col3 FROM test_table",
+ "query_type": "raw_sql",
+ "primary_key": "id"
+ },
+ "stream": {
+ "name": "example_stream", "action": "create",
+ "json_schema": { "field1": "type1" },
+ "supported_sync_modes": %w[full_refresh incremental],
+ "source_defined_cursor": true,
+ "default_cursor_field": ["field1"],
+ "source_defined_primary_key": [["field1"], ["field2"]],
+ "namespace": "exampleNamespace",
+ "url": "https://api.example.com/data",
+ "method": "GET"
+ },
+ "sync_mode": "full_refresh",
+ "cursor_field": "timestamp",
+ "destination_sync_mode": "upsert",
+ "sync_id": "1"
+ }
+ end
+
+ let(:oracle_connection) { instance_double(OCI8) }
+ let(:cursor) { instance_double("OCI8::Cursor") }
+
+ describe "#check_connection" do
+ context "when the connection is successful" do
+ it "returns a succeeded connection status" do
+ allow(OCI8).to receive(:new).and_return(oracle_connection)
+ allow(oracle_connection).to receive(:exec).and_return(true)
+ message = client.check_connection(sync_config[:source][:connection_specification])
+ result = message.connection_status
+ expect(result.status).to eq("succeeded")
+ expect(result.message).to be_nil
+ end
+ end
+
+ context "when the connection fails" do
+ it "returns a failed connection status with an error message" do
+ allow(client).to receive(:create_connection).and_raise(StandardError, "Connection failed")
+ message = client.check_connection(sync_config[:source][:connection_specification])
+ result = message.connection_status
+ expect(result.status).to eq("failed")
+ expect(result.message).to include("Connection failed")
+ end
+ end
+ end
+
+ # read and #discover tests for MariaDB
+ describe "#read" do
+ it "reads records successfully" do
+ s_config = Multiwoven::Integrations::Protocol::SyncConfig.from_json(sync_config.to_json)
+ columns = %w[col1 col2 col3]
+ response = %w[1 First Row Text First Row Additional Text]
+ allow(OCI8).to receive(:new).and_return(oracle_connection)
+ allow(oracle_connection).to receive(:exec).and_return(cursor)
+ allow(cursor).to receive(:get_col_names).and_return(columns, nil)
+ allow(cursor).to receive(:fetch).and_return(response, nil)
+ records = client.read(s_config)
+ expect(records).to be_an(Array)
+ expect(records).not_to be_empty
+ expect(records.first).to be_a(Multiwoven::Integrations::Protocol::MultiwovenMessage)
+ end
+
+ it "reads records successfully with limit" do
+ s_config = Multiwoven::Integrations::Protocol::SyncConfig.from_json(sync_config.to_json)
+ s_config.limit = 100
+ s_config.offset = 1
+ columns = %w[col1 col2 col3]
+ response = %w[1 First Row Text First Row Additional Text]
+ allow(OCI8).to receive(:new).and_return(oracle_connection)
+ allow(oracle_connection).to receive(:exec).and_return(cursor)
+ allow(cursor).to receive(:get_col_names).and_return(columns, nil)
+ allow(cursor).to receive(:fetch).and_return(response, nil)
+ records = client.read(s_config)
+ expect(records).to be_an(Array)
+ expect(records).not_to be_empty
+ expect(records.first).to be_a(Multiwoven::Integrations::Protocol::MultiwovenMessage)
+ end
+
+ it "read records failure" do
+ s_config = Multiwoven::Integrations::Protocol::SyncConfig.from_json(sync_config.to_json)
+ s_config.sync_run_id = "2"
+ allow(client).to receive(:create_connection).and_raise(StandardError, "test error")
+ expect(client).to receive(:handle_exception).with(
+ an_instance_of(StandardError), {
+ context: "ORACLE:READ:EXCEPTION",
+ type: "error",
+ sync_id: "1",
+ sync_run_id: "2"
+ }
+ )
+ client.read(s_config)
+ end
+ end
+
+ describe "#discover" do
+ it "discovers schema successfully" do
+ response = %w[test_table col1 NUMBER Y]
+ allow(OCI8).to receive(:new).and_return(oracle_connection)
+ allow(oracle_connection).to receive(:exec).and_return(cursor)
+ allow(cursor).to receive(:fetch).and_return(response, nil)
+ message = client.discover(sync_config[:source][:connection_specification])
+ expect(message.catalog).to be_an(Multiwoven::Integrations::Protocol::Catalog)
+ first_stream = message.catalog.streams.first
+ expect(first_stream).to be_a(Multiwoven::Integrations::Protocol::Stream)
+ expect(first_stream.name).to eq("test_table")
+ expect(first_stream.json_schema).to be_an(Hash)
+ expect(first_stream.json_schema["type"]).to eq("object")
+ expect(first_stream.json_schema["properties"]).to eq({ "col1" => { "type" => "string" } })
+ end
+ end
+
+ describe "#meta_data" do
+ # change this to rollout validation for all connector rolling out
+ it "client class_name and meta name is same" do
+ meta_name = client.class.to_s.split("::")[-2]
+ expect(client.send(:meta_data)[:data][:name]).to eq(meta_name)
+ end
+ end
+
+ describe "method definition" do
+ it "defines a private #query method" do
+ expect(described_class.private_instance_methods).to include(:query)
+ end
+ end
+end