resolve merge conflicts

.github/workflows/deploy-main.yml

@@ -0,0 +1,35 @@
+name: Deploy main
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - "ui/**"
+      - "server/**"
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Set up SSH key
+        env:
+          SSH_KEY_BASE64: ${{ secrets.SSH_KEY_BASE64 }}
+        run: |
+          echo "$SSH_KEY_BASE64" | base64 --decode > ${HOME}/multiwoven-deployments.pem
+          chmod 600 ${HOME}/multiwoven-deployments.pem
+
+      - name: Deploy main
+        env:
+          SSH_HOST: ${{ secrets.SSH_HOST }}
+          SSH_USER: ${{ secrets.SSH_USER }}
+        run: |
+          ssh -o StrictHostKeyChecking=no -i ${HOME}/multiwoven-deployments.pem $SSH_USER@$SSH_HOST << 'EOF'
+            cd multiwoven
+            git pull origin main
+            docker-compose down
+            docker-compose up -d --build
+          EOF

.gitignore

@@ -1,2 +1,3 @@
 # Ignore dotenv file
-.env
+.env
+/.history

integrations/Gemfile.lock

@@ -7,7 +7,7 @@ GIT
 PATH
   remote: .
   specs:
-    multiwoven-integrations (0.1.57)
+    multiwoven-integrations (0.1.59)
       activesupport
       async-websocket
       csv

integrations/lib/multiwoven/integrations/protocol/protocol.rb

@@ -71,6 +71,7 @@ class Connector < ProtocolModel
       attribute :name, Types::String
       attribute :type, ConnectorType
       attribute :connection_specification, Types::Hash
+      attribute :query_type, Types::String.default("raw_sql").enum(*ConnectorQueryType.values)
     end
 
     class LogMessage < ProtocolModel
@@ -114,7 +115,7 @@ class Stream < ProtocolModel
       attribute? :supported_sync_modes, Types::Array.of(SyncMode).optional.default(["incremental"])
 
       # Applicable for database streams
-      attribute? :source_defined_cursor, Types::Bool.default(false).optional
+      attribute :source_defined_cursor, Types::Bool.default(false)
       attribute? :default_cursor_field, Types::Array.of(Types::String).optional
       attribute? :source_defined_primary_key, Types::Array.of(Types::Array.of(Types::String)).optional
       attribute? :namespace, Types::String.optional

integrations/lib/multiwoven/integrations/rollout.rb

@@ -2,7 +2,7 @@
 
 module Multiwoven
   module Integrations
-    VERSION = "0.1.57"
+    VERSION = "0.1.59"
 
     ENABLED_SOURCES = %w[
       Snowflake

integrations/lib/multiwoven/integrations/source/salesforce_consumer_goods_cloud/client.rb

@@ -40,12 +40,6 @@ def discover(connection_config)
           def read(sync_config)
             connection_config = sync_config.source.connection_specification.with_indifferent_access
             initialize_client(connection_config)
-            return [] if sync_config.offset&.> 2000
-
-            # TODO: Salesforce imposes a limit on the use of OFFSET in SOQL queries, where you cannot skip(offset) more than 2000 records.
-            # This limitation can hinder the retrieval of large datasets in a single query.
-            # To overcome this, we need a cursor-based pagination strategy instead of relying on OFFSET.
-            # query = batched_query(query, sync_config.limit, sync_config.offset) unless sync_config.limit.nil? && sync_config.offset.nil?
             query = sync_config.model.query
             query = batched_query(query, sync_config.limit, sync_config.offset) unless sync_config.limit.nil? && sync_config.offset.nil?
             queried_data = @client.query(query)

integrations/lib/multiwoven/integrations/source/salesforce_consumer_goods_cloud/schema_helper.rb

@@ -118,9 +118,9 @@ def create_json_schema_for_object(metadata)
               "json_schema": json_schema,
               "required": required,
               "supported_sync_modes": %w[incremental],
-              "source_defined_cursor": true,
-              "default_cursor_field": ["updated"],
-              "source_defined_primary_key": [primary_key]
+              "source_defined_primary_key": [primary_key],
+              "source_defined_cursor": false,
+              "default_cursor_field": nil
             }
           end
         end

integrations/spec/multiwoven/integrations/destination/salesforce_consumer_goods_cloud/schema_helper_spec.rb

@@ -47,8 +47,6 @@
       expect(result[:batch_size]).to eq(0)
       expect(result[:required]).to contain_exactly("Field1")
       expect(result[:supported_sync_modes]).to contain_exactly("incremental")
-      expect(result[:source_defined_cursor]).to be true
-      expect(result[:default_cursor_field]).to contain_exactly("updated")
     end
   end
 end

integrations/spec/multiwoven/integrations/protocol/protocol_spec.rb

@@ -371,6 +371,7 @@ module Integrations::Protocol
           expect(connector).to be_a(described_class)
           expect(connector.name).to eq("example_connector")
           expect(connector.type).to eq("source")
+          expect(connector.query_type).to eq("raw_sql")
           expect(connector.connection_specification).to eq(key: "value")
         end
       end

integrations/spec/multiwoven/integrations/source/salesforce_consumer_goods/schema_helper_spec.rb

@@ -45,8 +45,8 @@
       expect(result[:json_schema]).to be_a(Hash)
       expect(result[:required]).to contain_exactly("Field1")
       expect(result[:supported_sync_modes]).to contain_exactly("incremental")
-      expect(result[:source_defined_cursor]).to be true
-      expect(result[:default_cursor_field]).to contain_exactly("updated")
+      expect(result[:source_defined_cursor]).to eq(false)
+      expect(result[:default_cursor_field]).to eq(nil)
     end
   end
 end

release-notes.md

@@ -2,184 +2,29 @@
 
 All notable changes to this project will be documented in this file.
 
-## [0.2.0] - 2024-04-08
+## [0.3.0] - 2024-04-15
 
 ### 🚀 Features
 
-- Add Filtering by Connector Type to 'Get All Connectors' API
-- Add batch support for rETL
-- Add default scope for Connector, Model and Sync
-- Strong type validation for API
-- Enhance DestinationSelector and Source and changed Dashboard UI
-- Implemented delete sync
-- Start with default custom object template from chakra package
-- Try simple layouting
-- Add support for specifying colspans
-- Add title field for overriding titles in form
-- Use overridden title field instead of default one
-- Add BaseInputTemplate
-- Add title field template
-- Add title to all the meta json in all connector
-- Add postgresql connector
-- Rate limiting
-- Configure rate limit for destinations
-- Add hubspot destination connector
-- Add databricks source connector
-- Reporting dashboard api
-- Standardization of output
-- Implement static, variable and template catalog field mapping
-- Enable rate limiting
-- Add finite state machine for sync states
-- Sync run and sync record controller
-- Terminate workflow on sync deletion
-- Delete source
-- Edit destinations screen and delete destinations
-- Add pull request template
-- Add env example file
-- Configure retry policy for terminate workflows
-- Add databricks odbc driver
-- Sync run api by sync id
-- Support full refresh sync
-- Custom liquid helper to_datetime
-- Sync mode changes for destination connectors
-- Destination/google sheets
-- *(destination)* Add airtable
-- *(destination)* Add Stripe destination connector
-- Support full refresh sync
-- Add Salesforce Consumer Goods destination connector
-- Add Salesforce Consumer Goods source connector 
-- Add postgresql destination connector
-- Release process automation 
-- Move integrations gem github actions to root
-- Move server and ui github action ci to root
-- Added sync records
-- Add soql support query_type
-- Update salesforce consumer goods authentication
-- Flatten salesforce records attributes
-- Add connector images to github to serve icons
-- Adding batch_support flag and batch_size in catalog for sync
-- Add batch support for facebook custom audience connector 
-- The volume mount paths don't clash with existing volumes and .env file added
-- Create sync run when pending sync run is absent
-- Duplicate sync record with primary key
+- Beautify sql queries
+- Add query type support for SOQL
+- Add sync run mailer on success or failure (#24)
+- Updated Dockerfile and env to run docker-compose on production mode. (#27)
+- Added custom dropdown for destination columns (#30)
+- Add code climate to github actions (#31)
+- Automate integrations gem release (#37)
+- Add protocol changes for cursor based (#38)
+- Pull request template for monorepo (#39)
+- Remove changes associated with multiwoven-integrations repository (#41)
+- Added hover text for columns (#36)
+- Add github action to deploy main (#44)
 
 ### 🐛 Bug Fixes
 
-- Snowflake odbc
-- Increment sync run offsets after each batch query
-- Model preview queries
-- IncrementalDelta primary key downcase issue
-- Input background not white when put in not white containers
-- Add empty line at the end
-- Add types for connector schema response from backend
-- Persist configure screen data on navigation
-- Edit syncs dropdown values
-- Conditionally render footer elements in create sync
-- Pre-render components during edit sync
-- Redirect routes for adding models and connectors
-- Footer buttons missing in source config form
-- Spinner style mismatch
-- Updated edit model header to include breadcrumbs
-- Added breadcrumbs and ui fix in edit query screen
-- Multiple mapping bug during create sync
-- Update chakra theme with design system font tokens
-- Design mismatch between form component and design
-- Spacing issues between the input and label
-- Style mismatch of description in source config form
-- Clicking on add source goes to destinations
-- Description spanning fully by adding a max width
-- Add connector button hardcoded to navigate to sources page always
-- Update text for no connectors found
-- Add batch params to stream
-- Release github action to use official gem action
-- Slack and salesforce svg
-- Avoid numbers in path created by gem in server
-- Change keytransform from underscore to unaltered
-- Update link to resources in readme
-- Prevent sync without connector
-- Bypass cc-test-reporter steps
-- Return complete slices in reporting
-- Update condition for domain name
-- Update issues link in the readme
-- Handle error during listing sync
-- Docker file env update for api host
-- Soft delete for sync run
-- Password policy
-- Pagination link invalid
-- Update login timeout
-- Handle nil for to_datetime filter in liquid
-- Databricks connector specification
-- Spec password 
-- Update query and create connection
-- Sftp server integration
-- Schema_mode changes to enum
-- Sftp full refresh
-- Add batch support fields in catalog streams
-- Batch size in salesforce consumer goods cloud
-- Destination path fix in check connection
-- Sftp connection spec filename added
-- Build and lint errors
-- Lint and build errors
-- Add primary key validation for extractor
-
-### 🚜 Refactor
-
-- Heartbeat timeout for loader and extractor activity
-- Remove unwanted code added for debugging
-- Change ui schemas to an object so that it only gets applied on pages where the ui schema is to be used
-- Extract query method for model preview
-- Support multiple connectors in reporting
-- Minor changes
-- Moved tab filters to new component
-
-### 📚 Documentation
-
-- Add comments to explain layout schema
-- Add github template for multiwoven server
-- Update readme for ui
-- Update readme link for contribution guideline
-- Update contribution guidelines
-
-### 🎨 Styling
-
-- Fix the exit modal as per designs
-- Update styling for step 1 for syncs page
-- Update the brand color in chakra config
-- Fix the designs of the top bar and model list table
-- Update table coloumns styling
-- Update styles for step 3 in syncs page
-- Fix the alignment of the footer container
-- Fix footer button alignments
-- Update styling for finale sync page
-- Update heading text
-- Update the color to use from tokens
-- Fix weaving soon design
-- Ui improvements for edit sync page
-- Make status tag consistent
-- Update styling for edit model details modal
-- Update edit model modal buttons
-- Update styling for delete modal
-- Update styling for final model submit step
-- Update background color for syncs page
-- Update bg color for the list screens
-- Update all destinations pill
-- Update connect your destination form styling
-- Update test destination screen
-- Fix the padding of table
-- Update background colors for edit sync box
-- Update padding of the box
-- Update the font size
-- Update the border color for disabled state
-- Update designs for selecting data source
-- Update style for testing source screen
-- Update design for final source step
-- Update top bar for edit source
-- Update form design for edit source
-- Update side nav as per figma
-- Update styling for logout modal
-- Align the breadcrumbs on top bar
-- Update copy changes for syncs screen
-- Update copy changes for models
-- Update copy for destinations screen
+- Add connector_query_type in spec (#23)
+- Remove disable condition for adding more mapping (#26)
+- Update sync run mailer to trigger on failure only
+- Code climate reporter path not found
+- Update back button navigation on syncs configuration (#35)
 
 <!-- generated by git-cliff -->

server/Gemfile

@@ -12,7 +12,7 @@ gem "interactor", "~> 3.0"
 
 gem "ruby-odbc", git: "https://github.com/Multiwoven/ruby-odbc.git"
 
-gem "multiwoven-integrations", "~> 0.1.55"
+gem "multiwoven-integrations", "~> 0.1.59"
 
 gem "temporal-ruby", github: "coinbase/temporal-ruby"
 

server/Gemfile.lock

@@ -106,7 +106,7 @@ GEM
       activerecord (>= 3.2, < 8.0)
       rake (>= 10.4, < 14.0)
     ast (2.4.2)
-    async (2.10.1)
+    async (2.10.2)
       console (~> 1.10)
       fiber-annotation
       io-event (~> 1.5, >= 1.5.1)
@@ -119,7 +119,7 @@ GEM
       protocol-http1 (~> 0.19.0)
       protocol-http2 (~> 0.16.0)
       traces (>= 0.10.0)
-    async-io (1.42.0)
+    async-io (1.42.1)
       async
     async-pool (0.4.0)
       async (>= 1.25)
@@ -1862,7 +1862,7 @@ GEM
     msgpack (1.7.2)
     multi_json (1.15.0)
     multipart-post (2.4.0)
-    multiwoven-integrations (0.1.55)
+    multiwoven-integrations (0.1.59)
       activesupport
       async-websocket
       csv
@@ -1925,7 +1925,7 @@ GEM
       net-smtp
       premailer (~> 1.7, >= 1.7.9)
     protocol-hpack (1.4.3)
-    protocol-http (0.26.2)
+    protocol-http (0.26.4)
     protocol-http1 (0.19.0)
       protocol-http (~> 0.22)
     protocol-http2 (0.16.0)
@@ -2071,7 +2071,7 @@ GEM
       gli
       hashie
     stringio (3.1.0)
-    stripe (10.14.0)
+    stripe (11.1.0)
     thor (1.3.0)
     timecop (0.9.8)
     timeout (0.4.1)
@@ -2130,7 +2130,7 @@ DEPENDENCIES
   jwt
   kaminari
   liquid
-  multiwoven-integrations (~> 0.1.55)
+  multiwoven-integrations (~> 0.1.59)
   newrelic_rpm
   parallel
   pg (~> 1.1)