Merge branch 'main' of https://github.com/Multiwoven/multiwoven into …

…cherry-pick-ce-commit-95c7852c8bd33e9aff951dfd3e1709a54d35000a
Multiwoven · Jul 19, 2024 · d5a22dd · d5a22dd
2 parents b7ce633 + ebf8158
commit d5a22dd
Show file tree

Hide file tree

Showing 4 changed files with 6 additions and 2 deletions.
diff --git a/integrations/Gemfile.lock b/integrations/Gemfile.lock
@@ -7,7 +7,7 @@ GIT
 PATH
   remote: .
   specs:
-    multiwoven-integrations (0.5.0)
+    multiwoven-integrations (0.5.1)
       activesupport
       async-websocket
       aws-sdk-athena

diff --git a/integrations/lib/multiwoven/integrations/core/constants.rb b/integrations/lib/multiwoven/integrations/core/constants.rb
@@ -16,6 +16,8 @@ module Constants
       JSON_SCHEMA_URL = "https://json-schema.org/draft-07/schema#"
 
       # CONNECTORS
+      INSTALL_HTTPFS_QUERY = ENV["INSTALL_HTTPFS_QUERY"] || "INSTALL HTTPFS; LOAD HTTPFS;"
+
       KLAVIYO_AUTH_ENDPOINT = "https://a.klaviyo.com/api/lists/"
       KLAVIYO_AUTH_PAYLOAD = {
         data: {

diff --git a/integrations/lib/multiwoven/integrations/rollout.rb b/integrations/lib/multiwoven/integrations/rollout.rb
@@ -2,7 +2,7 @@
 
 module Multiwoven
   module Integrations
-    VERSION = "0.5.0"
+    VERSION = "0.5.1"
 
     ENABLED_SOURCES = %w[
       Snowflake

diff --git a/integrations/lib/multiwoven/integrations/source/amazon_s3/client.rb b/integrations/lib/multiwoven/integrations/source/amazon_s3/client.rb
@@ -74,6 +74,8 @@ def create_connection(connection_config)
         @session_name = "preview-#{connection_config[:region]}-#{connection_config[:bucket]}" if @session_name.to_s.empty?
         auth_data = get_auth_data(connection_config)
         conn = DuckDB::Database.open.connect
+        # Install and/or Load the HTTPFS extension
+        conn.execute(INSTALL_HTTPFS_QUERY)
         # Set up S3 configuration
         secret_query = "
               CREATE SECRET amazons3_source (