Merge branch 'main' into feat/add-mixpanel-destination-connector

.github/workflows/integrations-ci.yml

@@ -33,11 +33,25 @@ jobs:
 
       - name: Download and Install Oracle Instant Client
         run: |
+<<<<<<< HEAD
           sudo apt-get install -y libaio1 alien
           wget http://yum.oracle.com/repo/OracleLinux/OL7/oracle/instantclient/x86_64/getPackage/oracle-instantclient19.6-basic-19.6.0.0.0-1.x86_64.rpm
           wget http://yum.oracle.com/repo/OracleLinux/OL7/oracle/instantclient/x86_64/getPackage/oracle-instantclient19.6-devel-19.6.0.0.0-1.x86_64.rpm
           sudo alien -i --scripts oracle-instantclient*.rpm
           rm -f oracle-instantclient*.rpm
+=======
+            wget http://ftp.debian.org/debian/pool/main/liba/libaio/libaio1_0.3.113-4_amd64.deb
+            sudo dpkg -i libaio1_0.3.113-4_amd64.deb
+            sudo apt-get install -f
+            sudo apt-get install -y alien unixodbc-dev
+            wget http://yum.oracle.com/repo/OracleLinux/OL7/oracle/instantclient/x86_64/getPackage/oracle-instantclient19.6-basic-19.6.0.0.0-1.x86_64.rpm
+            wget http://yum.oracle.com/repo/OracleLinux/OL7/oracle/instantclient/x86_64/getPackage/oracle-instantclient19.6-devel-19.6.0.0.0-1.x86_64.rpm
+            sudo alien -i --scripts oracle-instantclient*.rpm
+            rm -f oracle-instantclient*.rpm
+            echo "export LD_LIBRARY_PATH=/usr/lib/oracle/19.6/client64/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV
+            echo "export C_INCLUDE_PATH=/usr/include/oracle/19.6/client64:$C_INCLUDE_PATH" >> $GITHUB_ENV
+            echo "export CPLUS_INCLUDE_PATH=/usr/include/oracle/19.6/client64:$CPLUS_INCLUDE_PATH" >> $GITHUB_ENV
+>>>>>>> 9879f3ea (chore(CE): Update Oracle Instant Client Installation (#723))
 
       - name: Install FreeTDS for TinyTDS gem
         run: |

.github/workflows/integrations-main.yml

@@ -41,11 +41,26 @@ jobs:
 
       - name: Download and Install Oracle Instant Client
         run: |
+<<<<<<< HEAD
           sudo apt-get install -y libaio1 alien
           wget http://yum.oracle.com/repo/OracleLinux/OL7/oracle/instantclient/x86_64/getPackage/oracle-instantclient19.6-basic-19.6.0.0.0-1.x86_64.rpm
           wget http://yum.oracle.com/repo/OracleLinux/OL7/oracle/instantclient/x86_64/getPackage/oracle-instantclient19.6-devel-19.6.0.0.0-1.x86_64.rpm
           sudo alien -i --scripts oracle-instantclient*.rpm
           rm -f oracle-instantclient*.rpm
+=======
+            wget http://ftp.debian.org/debian/pool/main/liba/libaio/libaio1_0.3.113-4_amd64.deb
+            sudo dpkg -i libaio1_0.3.113-4_amd64.deb
+            sudo apt-get install -f
+            sudo apt-get install -y alien unixodbc-dev
+            wget http://yum.oracle.com/repo/OracleLinux/OL7/oracle/instantclient/x86_64/getPackage/oracle-instantclient19.6-basic-19.6.0.0.0-1.x86_64.rpm
+            wget http://yum.oracle.com/repo/OracleLinux/OL7/oracle/instantclient/x86_64/getPackage/oracle-instantclient19.6-devel-19.6.0.0.0-1.x86_64.rpm
+            sudo alien -i --scripts oracle-instantclient*.rpm
+            rm -f oracle-instantclient*.rpm
+            echo "export LD_LIBRARY_PATH=/usr/lib/oracle/19.6/client64/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV
+            echo "export C_INCLUDE_PATH=/usr/include/oracle/19.6/client64:$C_INCLUDE_PATH" >> $GITHUB_ENV
+            echo "export CPLUS_INCLUDE_PATH=/usr/include/oracle/19.6/client64:$CPLUS_INCLUDE_PATH" >> $GITHUB_ENV
+
+>>>>>>> 9879f3ea (chore(CE): Update Oracle Instant Client Installation (#723))
 
       - name: Install FreeTDS for TinyTDS gem
         run: |

.github/workflows/server-ci.yml

@@ -52,12 +52,23 @@ jobs:
 
       - name: Download and Install Oracle Instant Client
         run: |
-          sudo apt-get install -y libaio1 alien
+          wget http://ftp.debian.org/debian/pool/main/liba/libaio/libaio1_0.3.113-4_amd64.deb
+          sudo dpkg -i libaio1_0.3.113-4_amd64.deb
+          sudo apt-get install -f
+          sudo apt-get install -y alien unixodbc-dev
           wget http://yum.oracle.com/repo/OracleLinux/OL7/oracle/instantclient/x86_64/getPackage/oracle-instantclient19.6-basic-19.6.0.0.0-1.x86_64.rpm
           wget http://yum.oracle.com/repo/OracleLinux/OL7/oracle/instantclient/x86_64/getPackage/oracle-instantclient19.6-devel-19.6.0.0.0-1.x86_64.rpm
           sudo alien -i --scripts oracle-instantclient*.rpm
           rm -f oracle-instantclient*.rpm
+<<<<<<< HEAD
 
+=======
+          echo "export LD_LIBRARY_PATH=/usr/lib/oracle/19.6/client64/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV
+          echo "export C_INCLUDE_PATH=/usr/include/oracle/19.6/client64:$C_INCLUDE_PATH" >> $GITHUB_ENV
+          echo "export CPLUS_INCLUDE_PATH=/usr/include/oracle/19.6/client64:$CPLUS_INCLUDE_PATH" >> $GITHUB_ENV
+
+
+>>>>>>> 9879f3ea (chore(CE): Update Oracle Instant Client Installation (#723))
       - name: Install FreeTDS for TinyTDS gem
         run: |
           sudo apt-get install libc6-dev

integrations/Gemfile.lock

@@ -7,7 +7,7 @@ GIT
 PATH
   remote: .
   specs:
-    multiwoven-integrations (0.15.10)
+    multiwoven-integrations (0.16.2)
       MailchimpMarketing
       activesupport
       async-websocket

integrations/lib/multiwoven/integrations.rb

@@ -52,7 +52,9 @@
 require_relative "integrations/core/base_connector"
 require_relative "integrations/core/source_connector"
 require_relative "integrations/core/destination_connector"
+require_relative "integrations/core/http_helper"
 require_relative "integrations/core/http_client"
+require_relative "integrations/core/streaming_http_client"
 require_relative "integrations/core/query_builder"
 
 # Source
@@ -71,6 +73,7 @@
 require_relative "integrations/source/aws_sagemaker_model/client"
 require_relative "integrations/source/google_vertex_model/client"
 require_relative "integrations/source/http_model/client"
+require_relative "integrations/source/open_ai/client"
 
 # Destination
 require_relative "integrations/destination/klaviyo/client"

integrations/lib/multiwoven/integrations/core/constants.rb

@@ -67,6 +67,7 @@ module Constants
       MIXPANEL_BASE_URL = "https://api.mixpanel.com"
       MIXPANEL_AUTH_ENDPOINT = "https://api.mixpanel.com/track"
 
+      OPEN_AI_URL = "https://api.openai.com/v1/chat/completions"
     end
   end
 end

integrations/lib/multiwoven/integrations/core/http_client.rb

@@ -3,40 +3,14 @@
 module Multiwoven
   module Integrations::Core
     class HttpClient
+      extend HttpHelper
       class << self
         def request(url, method, payload: nil, headers: {}, config: {})
           uri = URI(url)
-          http = Net::HTTP.new(uri.host, uri.port)
-          http.use_ssl = (uri.scheme == "https")
-
-          # Set timeout if provided
-          if config[:timeout]
-            timeout_value = config[:timeout].to_f
-            http.open_timeout = timeout_value
-            http.read_timeout = timeout_value
-          end
-
+          http = configure_http(uri, config)
           request = build_request(method, uri, payload, headers)
           http.request(request)
         end
-
-        private
-
-        def build_request(method, uri, payload, headers)
-          request_class = case method.upcase
-                          when Constants::HTTP_GET then Net::HTTP::Get
-                          when Constants::HTTP_POST then Net::HTTP::Post
-                          when Constants::HTTP_PUT then Net::HTTP::Put
-                          when Constants::HTTP_PATCH then Net::HTTP::Patch
-                          when Constants::HTTP_DELETE then Net::HTTP::Delete
-                          else raise ArgumentError, "Unsupported HTTP method: #{method}"
-                          end
-
-          request = request_class.new(uri)
-          headers.each { |key, value| request[key] = value }
-          request.body = payload.to_json if payload && %w[POST PUT PATCH].include?(method.upcase)
-          request
-        end
       end
     end
   end

integrations/lib/multiwoven/integrations/core/http_helper.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Multiwoven
+  module Integrations::Core
+    module HttpHelper
+      def build_request(method, uri, payload, headers)
+        request_class = case method.upcase
+                        when Constants::HTTP_GET then Net::HTTP::Get
+                        when Constants::HTTP_POST then Net::HTTP::Post
+                        when Constants::HTTP_PUT then Net::HTTP::Put
+                        when Constants::HTTP_PATCH then Net::HTTP::Patch
+                        when Constants::HTTP_DELETE then Net::HTTP::Delete
+                        else raise ArgumentError, "Unsupported HTTP method: #{method}"
+                        end
+
+        request = request_class.new(uri)
+        headers.each { |key, value| request[key] = value }
+        request.body = payload.to_json if payload && %w[POST PUT PATCH].include?(method.upcase)
+        request
+      end
+
+      def configure_http(uri, config)
+        http = Net::HTTP.new(uri.host, uri.port)
+        http.use_ssl = (uri.scheme == "https")
+
+        if config[:timeout]
+          timeout_value = config[:timeout].to_f
+          http.open_timeout = timeout_value
+          http.read_timeout = timeout_value
+        end
+
+        http
+      end
+    end
+  end
+end

integrations/lib/multiwoven/integrations/core/source_connector.rb

@@ -39,6 +39,28 @@ def batched_query(sql_query, limit, offset)
         # Appending the LIMIT and OFFSET clauses to the SQL query
         "#{sql_query} LIMIT #{limit} OFFSET #{offset}"
       end
+
+      def send_request(options = {})
+        Multiwoven::Integrations::Core::HttpClient.request(
+          options[:url],
+          options[:http_method],
+          payload: options[:payload],
+          headers: options[:headers],
+          config: options[:config]
+        )
+      end
+
+      def send_streaming_request(options = {})
+        Multiwoven::Integrations::Core::StreamingHttpClient.request(
+          options[:url],
+          options[:http_method],
+          payload: options[:payload],
+          headers: options[:headers],
+          config: options[:config]
+        ) do |chunk|
+          yield chunk if block_given? # Pass each chunk for processing (streaming response)
+        end
+      end
     end
   end
 end

integrations/lib/multiwoven/integrations/core/streaming_http_client.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Multiwoven
+  module Integrations::Core
+    class StreamingHttpClient
+      extend HttpHelper
+      class << self
+        def request(url, method, payload: nil, headers: {}, config: {})
+          uri = URI(url)
+          http = configure_http(uri, config)
+          request = build_request(method, uri, payload, headers)
+          http.request(request) do |response|
+            response.read_body do |chunk|
+              yield chunk if block_given? # Pass each response chunk
+            end
+          end
+        end
+      end
+    end
+  end
+end

integrations/lib/multiwoven/integrations/rollout.rb

@@ -2,7 +2,7 @@
 
 module Multiwoven
   module Integrations
-    VERSION = "0.15.10"
+    VERSION = "0.16.2"
 
     ENABLED_SOURCES = %w[
       Snowflake
@@ -20,6 +20,7 @@ module Integrations
       AwsSagemakerModel
       VertexModel
       HttpModel
+      OpenAI
     ].freeze
 
     ENABLED_DESTINATIONS = %w[

integrations/lib/multiwoven/integrations/source/databrics_model/client.rb

@@ -62,8 +62,12 @@ def run_model(connection_config, payload)
 
       def process_response(response)
         if success?(response)
-          data = JSON.parse(response.body)
-          [RecordMessage.new(data: data, emitted_at: Time.now.to_i).to_multiwoven_message]
+          begin
+            data = JSON.parse(response.body)
+            [RecordMessage.new(data: data, emitted_at: Time.now.to_i).to_multiwoven_message]
+          rescue JSON::ParserError
+            create_log_message("DATABRICKS MODEL:RUN_MODEL", "error", "parsing failed: please send a valid payload")
+          end
         else
           create_log_message("DATABRICKS MODEL:RUN_MODEL", "error", "request failed: #{response.body}")
         end