-
Notifications
You must be signed in to change notification settings - Fork 66
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat(CE): add oracle db destination connector (#277)
- Loading branch information
1 parent
c308382
commit 68dd88d
Showing
12 changed files
with
352 additions
and
2 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
112 changes: 112 additions & 0 deletions
112
integrations/lib/multiwoven/integrations/destination/oracle_db/client.rb
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,112 @@ | ||
| # frozen_string_literal: true | ||
|
|
||
| module Multiwoven::Integrations::Destination | ||
| module Oracle | ||
| include Multiwoven::Integrations::Core | ||
| class Client < DestinationConnector | ||
| def check_connection(connection_config) | ||
| connection_config = connection_config.with_indifferent_access | ||
| create_connection(connection_config) | ||
| ConnectionStatus.new( | ||
| status: ConnectionStatusType["succeeded"] | ||
| ).to_multiwoven_message | ||
| rescue StandardError => e | ||
| ConnectionStatus.new( | ||
| status: ConnectionStatusType["failed"], message: e.message | ||
| ).to_multiwoven_message | ||
| end | ||
|
|
||
| def discover(connection_config) | ||
| records = [] | ||
| connection_config = connection_config.with_indifferent_access | ||
| query = "SELECT table_name, column_name, data_type, nullable | ||
| FROM all_tab_columns | ||
| WHERE owner = '#{connection_config[:username].upcase}' | ||
| ORDER BY table_name, column_id" | ||
| conn = create_connection(connection_config) | ||
| cursor = conn.exec(query) | ||
| while (row = cursor.fetch) | ||
| records << row | ||
| end | ||
| catalog = Catalog.new(streams: create_streams(records)) | ||
| catalog.to_multiwoven_message | ||
| rescue StandardError => e | ||
| handle_exception( | ||
| "ORACLE:DISCOVER:EXCEPTION", | ||
| "error", | ||
| e | ||
| ) | ||
| end | ||
|
|
||
| def write(sync_config, records, action = "destination_insert") | ||
| connection_config = sync_config.destination.connection_specification.with_indifferent_access | ||
| table_name = sync_config.stream.name | ||
| primary_key = sync_config.model.primary_key | ||
| conn = create_connection(connection_config) | ||
|
|
||
| write_success = 0 | ||
| write_failure = 0 | ||
| log_message_array = [] | ||
|
|
||
| records.each do |record| | ||
| query = Multiwoven::Integrations::Core::QueryBuilder.perform(action, table_name, record, primary_key) | ||
| query = query.gsub(";", "") | ||
| logger.debug("ORACLE:WRITE:QUERY query = #{query} sync_id = #{sync_config.sync_id} sync_run_id = #{sync_config.sync_run_id}") | ||
| begin | ||
| response = conn.exec(query) | ||
| conn.exec("COMMIT") | ||
| write_success += 1 | ||
| log_message_array << log_request_response("info", query, response) | ||
| rescue StandardError => e | ||
| handle_exception(e, { | ||
| context: "ORACLE:RECORD:WRITE:EXCEPTION", | ||
| type: "error", | ||
| sync_id: sync_config.sync_id, | ||
| sync_run_id: sync_config.sync_run_id | ||
| }) | ||
| write_failure += 1 | ||
| log_message_array << log_request_response("error", query, e.message) | ||
| end | ||
| end | ||
| tracking_message(write_success, write_failure, log_message_array) | ||
| rescue StandardError => e | ||
| handle_exception(e, { | ||
| context: "ORACLE:RECORD:WRITE:EXCEPTION", | ||
| type: "error", | ||
| sync_id: sync_config.sync_id, | ||
| sync_run_id: sync_config.sync_run_id | ||
| }) | ||
| end | ||
|
|
||
| private | ||
|
|
||
| def create_connection(connection_config) | ||
| OCI8.new(connection_config[:username], connection_config[:password], "#{connection_config[:host]}:#{connection_config[:port]}/#{connection_config[:sid]}") | ||
| end | ||
|
|
||
| def create_streams(records) | ||
| group_by_table(records).map do |_, r| | ||
| Multiwoven::Integrations::Protocol::Stream.new(name: r[:tablename], action: StreamAction["fetch"], json_schema: convert_to_json_schema(r[:columns])) | ||
| end | ||
| end | ||
|
|
||
| def group_by_table(records) | ||
| result = {} | ||
| records.each_with_index do |entry, index| | ||
| table_name = entry[0] | ||
| column_data = { | ||
| column_name: entry[1], | ||
| data_type: entry[2], | ||
| is_nullable: entry[3] == "Y" | ||
| } | ||
| result[index] ||= {} | ||
| result[index][:tablename] = table_name | ||
| result[index][:columns] = [column_data] | ||
| end | ||
| result.values.group_by { |entry| entry[:tablename] }.transform_values do |entries| | ||
| { tablename: entries.first[:tablename], columns: entries.flat_map { |entry| entry[:columns] } } | ||
| end | ||
| end | ||
| end | ||
| end | ||
| end |
15 changes: 15 additions & 0 deletions
15
integrations/lib/multiwoven/integrations/destination/oracle_db/config/meta.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,15 @@ | ||
| { | ||
| "data": { | ||
| "name": "Oracle", | ||
| "title": "Oracle", | ||
| "connector_type": "destination", | ||
| "category": "Database", | ||
| "documentation_url": "https://docs.squared.ai/guides/data-integration/destination/oracle", | ||
| "github_issue_label": "destination-oracle", | ||
| "icon": "icon.svg", | ||
| "license": "MIT", | ||
| "release_stage": "alpha", | ||
| "support_level": "community", | ||
| "tags": ["language:ruby", "multiwoven"] | ||
| } | ||
| } |
47 changes: 47 additions & 0 deletions
47
integrations/lib/multiwoven/integrations/destination/oracle_db/config/spec.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,47 @@ | ||
| { | ||
| "documentation_url": "https://docs.squared.ai/guides/data-integration/destination/oracle", | ||
| "stream_type": "dynamic", | ||
| "connector_query_type": "raw_sql", | ||
| "connection_specification": { | ||
| "$schema": "http://json-schema.org/draft-07/schema#", | ||
| "title": "Oracle", | ||
| "type": "object", | ||
| "required": ["host", "port", "sid", "username", "password"], | ||
| "properties": { | ||
| "host": { | ||
| "description": "The Oracle host.", | ||
| "examples": ["localhost"], | ||
| "type": "string", | ||
| "title": "Host", | ||
| "order": 0 | ||
| }, | ||
| "port": { | ||
| "description": "The Oracle port number.", | ||
| "examples": ["1521"], | ||
| "type": "string", | ||
| "title": "Port", | ||
| "order": 1 | ||
| }, | ||
| "sid": { | ||
| "description": "The name of your service in Oracle.", | ||
| "examples": ["ORCLPDB1"], | ||
| "type": "string", | ||
| "title": "SID", | ||
| "order": 2 | ||
| }, | ||
| "username": { | ||
| "description": "The username used to authenticate and connect.", | ||
| "type": "string", | ||
| "title": "Username", | ||
| "order": 3 | ||
| }, | ||
| "password": { | ||
| "description": "The password corresponding to the username used for authentication.", | ||
| "type": "string", | ||
| "multiwoven_secret": true, | ||
| "title": "Password", | ||
| "order": 4 | ||
| } | ||
| } | ||
| } | ||
| } |
4 changes: 4 additions & 0 deletions
4
integrations/lib/multiwoven/integrations/destination/oracle_db/icon.svg
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.