From 8c984cd64bf468fb90624fd89c93af4f2d967de6 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 11 Dec 2024 08:54:42 -0500 Subject: [PATCH 01/28] chore(CE): Update server gem 0.15.10 (#525) Co-authored-by: TivonB-AI2 <124182151+TivonB-AI2@users.noreply.github.com> --- server/Gemfile | 2 +- server/Gemfile.lock | 18 ++++++++++-------- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/server/Gemfile b/server/Gemfile index c9c26efa..107eb5cc 100644 --- a/server/Gemfile +++ b/server/Gemfile @@ -13,7 +13,7 @@ gem "interactor", "~> 3.0" gem "ruby-odbc", git: "https://github.com/Multiwoven/ruby-odbc.git" -gem "multiwoven-integrations", "~> 0.15.8" +gem "multiwoven-integrations", "~> 0.15.10" gem "temporal-ruby", github: "coinbase/temporal-ruby" diff --git a/server/Gemfile.lock b/server/Gemfile.lock index 7a326694..fb29b8c8 100644 --- a/server/Gemfile.lock +++ b/server/Gemfile.lock @@ -1667,7 +1667,7 @@ GEM activesupport concurrent-ruby (1.2.3) connection_pool (2.4.1) - console (1.29.0) + console (1.29.2) fiber-annotation fiber-local (~> 1.1) json @@ -1800,7 +1800,7 @@ GEM gli (2.22.0) globalid (1.2.1) activesupport (>= 6.1) - google-apis-bigquery_v2 (0.81.0) + google-apis-bigquery_v2 (0.82.0) google-apis-core (>= 0.15.0, < 2.a) google-apis-core (0.15.1) addressable (~> 2.5, >= 2.5.1) @@ -1812,7 +1812,7 @@ GEM retriable (>= 2.0, < 4.a) google-apis-sheets_v4 (0.38.0) google-apis-core (>= 0.15.0, < 2.a) - google-cloud-ai_platform-v1 (0.59.0) + google-cloud-ai_platform-v1 (0.60.0) gapic-common (>= 0.21.1, < 2.a) google-cloud-errors (~> 1.0) google-cloud-location (>= 0.7, < 2.a) @@ -1838,6 +1838,7 @@ GEM gapic-common (>= 0.21.1, < 2.a) google-cloud-errors (~> 1.0) grpc-google-iam-v1 (~> 1.1) + google-logging-utils (0.1.0) google-protobuf (3.25.2-aarch64-linux) google-protobuf (3.25.2-arm64-darwin) google-protobuf (3.25.2-x86_64-darwin) @@ -1848,9 +1849,10 @@ GEM grpc (~> 1.41) googleapis-common-protos-types (1.11.0) google-protobuf (~> 3.18) - googleauth (1.11.2) + googleauth (1.12.0) faraday (>= 1.0, < 3.a) - google-cloud-env (~> 2.1) + google-cloud-env (~> 2.2) + google-logging-utils (~> 0.1) jwt (>= 1.4, < 3.0) multi_json (~> 1.11) os (>= 0.9, < 2.0) @@ -1938,7 +1940,7 @@ GEM msgpack (1.7.2) multi_json (1.15.0) multipart-post (2.4.1) - multiwoven-integrations (0.15.8) + multiwoven-integrations (0.15.10) MailchimpMarketing activesupport async-websocket @@ -2169,7 +2171,7 @@ GEM faraday-multipart gli hashie - sorbet-runtime (0.5.11681) + sorbet-runtime (0.5.11690) stringio (3.1.0) stripe (13.2.0) strong_migrations (1.8.0) @@ -2245,7 +2247,7 @@ DEPENDENCIES jwt kaminari liquid - multiwoven-integrations (~> 0.15.8) + multiwoven-integrations (~> 0.15.10) mysql2 newrelic_rpm parallel From 969b25b2fdb8f52f3d1e17b64d06ef01cd98921b Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 11 Dec 2024 12:28:54 -0500 Subject: [PATCH 02/28] chore(CE): add resource link to audit logs (#523) * Resolve conflict in cherry-pick of 0781c55347d059789d49e69c0609aee1724afaa0 and change the commit message * chore(CE): Remove Enterprise --------- Co-authored-by: TivonB-AI2 <124182151+TivonB-AI2@users.noreply.github.com> Co-authored-by: TivonB-AI2 --- .../app/controllers/concerns/audit_logger.rb | 19 +++-- ...6211928_add_resource_link_to_audit_logs.rb | 5 ++ server/db/schema.rb | 3 +- .../api/v1/audit_logs_controller_spec.rb | 69 ------------------- server/spec/factories/audit_logs.rb | 1 + 5 files changed, 21 insertions(+), 76 deletions(-) create mode 100644 server/db/migrate/20241206211928_add_resource_link_to_audit_logs.rb delete mode 100644 server/spec/enterprise/requests/api/v1/audit_logs_controller_spec.rb diff --git a/server/app/controllers/concerns/audit_logger.rb b/server/app/controllers/concerns/audit_logger.rb index d32b435f..45c3ab33 100644 --- a/server/app/controllers/concerns/audit_logger.rb +++ b/server/app/controllers/concerns/audit_logger.rb @@ -2,11 +2,16 @@ module AuditLogger extend ActiveSupport::Concern - def audit!(action: nil, user: nil, resource_type: nil, resource_id: nil, resource: nil, workspace: nil, payload: {}) # rubocop:disable Metrics/ParameterLists - action ||= action_name - resource_type ||= controller_name.singularize.capitalize - user ||= current_user - workspace ||= current_workspace + # rubocop:disable Metrics/CyclomaticComplexity + def audit!(options = {}) # rubocop:disable Metrics/PerceivedComplexity + action = options[:action] || action_name + resource_type = options[:resource_type] || controller_name.singularize.capitalize + resource_id = options[:resource_id] || nil + resource = options[:resource] || nil + user = options[:user] || current_user + workspace = options[:workspace] || current_workspace + payload = options[:payload] || {} + resource_link = options[:resource_link] || nil begin AuditLog.create( @@ -16,7 +21,8 @@ def audit!(action: nil, user: nil, resource_type: nil, resource_id: nil, resourc resource_id:, resource:, workspace:, - metadata: payload.try(:to_unsafe_h) || payload + metadata: payload.try(:to_unsafe_h) || payload, + resource_link: ) rescue StandardError => e Rails.logger.error({ @@ -25,4 +31,5 @@ def audit!(action: nil, user: nil, resource_type: nil, resource_id: nil, resourc }.to_s) end end + # rubocop:enable Metrics/CyclomaticComplexity end diff --git a/server/db/migrate/20241206211928_add_resource_link_to_audit_logs.rb b/server/db/migrate/20241206211928_add_resource_link_to_audit_logs.rb new file mode 100644 index 00000000..a4a3cf33 --- /dev/null +++ b/server/db/migrate/20241206211928_add_resource_link_to_audit_logs.rb @@ -0,0 +1,5 @@ +class AddResourceLinkToAuditLogs < ActiveRecord::Migration[7.1] + def change + add_column :audit_logs, :resource_link, :string + end +end diff --git a/server/db/schema.rb b/server/db/schema.rb index 0129bd85..3c7e33ba 100644 --- a/server/db/schema.rb +++ b/server/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.1].define(version: 2024_11_26_104103) do +ActiveRecord::Schema[7.1].define(version: 2024_12_06_211928) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -52,6 +52,7 @@ t.json "metadata" t.datetime "created_at", null: false t.datetime "updated_at", null: false + t.string "resource_link" end create_table "catalogs", force: :cascade do |t| diff --git a/server/spec/enterprise/requests/api/v1/audit_logs_controller_spec.rb b/server/spec/enterprise/requests/api/v1/audit_logs_controller_spec.rb deleted file mode 100644 index 198a3da1..00000000 --- a/server/spec/enterprise/requests/api/v1/audit_logs_controller_spec.rb +++ /dev/null @@ -1,69 +0,0 @@ -# frozen_string_literal: true - -require "rails_helper" - -RSpec.describe Enterprise::Api::V1::AuditLogsController, type: :controller do - let(:workspace) { create(:workspace) } - let!(:workspace_id) { workspace.id } - let(:user) { workspace.workspace_users.first.user } - let!(:audit_logs) do - [ - create(:audit_log, created_at: 2.days.ago, updated_at: 2.days.ago, workspace:, user:), - create(:audit_log, workspace:, user:), - create(:audit_log, created_at: 1.day.ago, updated_at: 1.day.ago, workspace:, user:) - ] - end - let(:mock_params) do - { - start_date: (Time.current - 2.days).strftime("%Y-%m-%d"), - end_date: (Time.current + 1.day).strftime("%Y-%m-%d"), - user_id: audit_logs.first.user_id, - resource_type: audit_logs.first.resource_type, - resource: audit_logs.first.resource, - page: 1 - } - end - let(:member_role) { create(:role, :member) } - - before do - user.update!(confirmed_at: Time.current) - end - - describe "GET /enterprise/api/v1/audit_logs" do - context "when it is an unauthenticated user" do - it "returns unauthorized" do - get :index - expect(response).to have_http_status(:unauthorized) - end - end - - context "when it is an authenticated user" do - it "returns success and fetch all audit log" do - request.headers.merge!(auth_headers(user, workspace_id)) - workspace.workspace_users.first.update(role: member_role) - get :index, params: mock_params - expect(response).to have_http_status(:ok) - - response_hash = JSON.parse(response.body).with_indifferent_access - first_row_date = response_hash["data"].first["attributes"]["created_at"] - last_row_date = response_hash["data"].last["attributes"]["created_at"] - expect(first_row_date).to be > last_row_date - - expect(response_hash["data"].size).to eq(3) - expect(response_hash["data"].first["id"]).to eql(audit_logs.second.id.to_s) - expect(response_hash["data"].first["attributes"]["user_id"]).to eql(audit_logs.second.user_id) - expect(response_hash["data"].first["attributes"]["user_name"]).to eql(user.name) - expect(response_hash["data"].first["attributes"]["action"]).to eql(audit_logs.second.action) - expect(response_hash["data"].first["attributes"]["resource_type"]).to eql(audit_logs.second.resource_type) - expect(response_hash["data"].first["attributes"]["resource_id"]).to eql(audit_logs.second.resource_id) - expect(response_hash["data"].first["attributes"]["resource"]).to eql(audit_logs.second.resource) - expect(response_hash["data"].first["attributes"]["workspace_id"]).to eql(workspace_id) - expect(response_hash["data"].first["attributes"]["metadata"]).to eql(audit_logs.second.metadata) - expect(response_hash.dig(:links, :first)) - .to include("/enterprise/api/v1/audit_logs?") - expect(response_hash.dig(:links, :first)) - .to include("page=1&per_page=10") - end - end - end -end diff --git a/server/spec/factories/audit_logs.rb b/server/spec/factories/audit_logs.rb index e63ceced..8a3bc9f0 100644 --- a/server/spec/factories/audit_logs.rb +++ b/server/spec/factories/audit_logs.rb @@ -9,6 +9,7 @@ metadata { nil } created_at { Time.current } updated_at { Time.current } + resource_link { "api/test_link" } association :workspace association :user From 00794973f0bd7ad7064824b53e6c7032cfc5197d Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 13 Dec 2024 18:24:51 +0530 Subject: [PATCH 03/28] feat(CE): added title change based on route --- ui/package.json | 2 +- ui/src/constants/index.ts | 1 + ui/src/utils/__tests__/getPageTitle.test.ts | 27 +++++++++++++++++++++ ui/src/utils/getPageTitle.ts | 26 ++++++++++++++++++++ ui/src/views/MainLayout/MainLayout.tsx | 21 +++++++++++----- 5 files changed, 70 insertions(+), 7 deletions(-) create mode 100644 ui/src/constants/index.ts create mode 100644 ui/src/utils/__tests__/getPageTitle.test.ts create mode 100644 ui/src/utils/getPageTitle.ts diff --git a/ui/package.json b/ui/package.json index a25868f8..c9d5e7aa 100644 --- a/ui/package.json +++ b/ui/package.json @@ -1,5 +1,5 @@ { - "name": "vite-project", + "name": "multiwoven", "private": true, "version": "0.0.0", "type": "module", diff --git a/ui/src/constants/index.ts b/ui/src/constants/index.ts new file mode 100644 index 00000000..c79fad0f --- /dev/null +++ b/ui/src/constants/index.ts @@ -0,0 +1 @@ +export const BRAND_NAME = 'Multiwoven'; diff --git a/ui/src/utils/__tests__/getPageTitle.test.ts b/ui/src/utils/__tests__/getPageTitle.test.ts new file mode 100644 index 00000000..ea390f04 --- /dev/null +++ b/ui/src/utils/__tests__/getPageTitle.test.ts @@ -0,0 +1,27 @@ +import { expect } from '@jest/globals'; +import '@testing-library/jest-dom/jest-globals'; +import '@testing-library/jest-dom'; + +import getTitle from '@/utils/getPageTitle'; + +describe('getTitle', () => { + it('should return the correct title for a known route', () => { + expect(getTitle('/')).toBe('Dashboard | Multiwoven'); + expect(getTitle('/settings')).toBe('Settings | Multiwoven'); + expect(getTitle('/activate/syncs')).toBe('Syncs | Multiwoven'); + expect(getTitle('/setup/sources')).toBe('Sources | Multiwoven'); + expect(getTitle('/define/models')).toBe('Models | Multiwoven'); + expect(getTitle('/setup/destinations')).toBe('Destinations | Multiwoven'); + }); + + it('should return "Multiwoven" for an unknown route', () => { + expect(getTitle('/unknown')).toBe('Multiwoven'); + expect(getTitle('/another/unknown/path')).toBe('Multiwoven'); + }); + + it('should return the correct title for a route with additional path segments', () => { + expect(getTitle('/')).toBe('Dashboard | Multiwoven'); + expect(getTitle('/define/models/ai/28')).toBe('Models | Multiwoven'); + expect(getTitle('/define/models/28')).toBe('Models | Multiwoven'); + }); +}); diff --git a/ui/src/utils/getPageTitle.ts b/ui/src/utils/getPageTitle.ts new file mode 100644 index 00000000..0038a715 --- /dev/null +++ b/ui/src/utils/getPageTitle.ts @@ -0,0 +1,26 @@ +import { BRAND_NAME } from '@/constants'; + +const ROUTE_MAP = { + '/settings': 'Settings', + '/activate/syncs': 'Syncs', + '/setup/sources': 'Sources', + '/define/models': 'Models', + '/setup/destinations': 'Destinations', +}; + +const getTitle = (pathname: string) => { + if (pathname === '/') { + return 'Dashboard | ' + BRAND_NAME; + } + const normalizedPath = pathname.replace(/\/+$/, ''); + const matchingRoute = Object.keys(ROUTE_MAP).find((route) => normalizedPath.startsWith(route)); + + const title = matchingRoute ? ROUTE_MAP[matchingRoute as keyof typeof ROUTE_MAP] : null; + + if (title) { + return title + ' | ' + BRAND_NAME; + } + return BRAND_NAME; +}; + +export default getTitle; diff --git a/ui/src/views/MainLayout/MainLayout.tsx b/ui/src/views/MainLayout/MainLayout.tsx index 6a4d5992..a6dbb35c 100644 --- a/ui/src/views/MainLayout/MainLayout.tsx +++ b/ui/src/views/MainLayout/MainLayout.tsx @@ -1,19 +1,28 @@ +import { Outlet, useLocation } from 'react-router-dom'; +import { useState, useEffect } from 'react'; + +import Loader from '@/components/Loader'; import { useUiConfig } from '@/utils/hooks'; import Sidebar from '@/views/Sidebar/Sidebar'; +import useCustomToast from '@/hooks/useCustomToast'; +import { CustomToastStatus } from '@/components/Toast'; +import ServerError from '../ServerError'; +import getTitle from '@/utils/getPageTitle'; + import { Box } from '@chakra-ui/layout'; -import { Outlet } from 'react-router-dom'; -import Loader from '@/components/Loader'; -import { useState, useEffect } from 'react'; import { useQuery } from '@tanstack/react-query'; + import { getWorkspaces } from '@/services/settings'; import { useStore } from '@/stores'; -import ServerError from '../ServerError'; -import useCustomToast from '@/hooks/useCustomToast'; -import { CustomToastStatus } from '@/components/Toast'; const MainLayout = (): JSX.Element => { const [isLoading, setIsLoading] = useState(true); const { contentContainerId } = useUiConfig(); + const location = useLocation(); + useEffect(() => { + const title = getTitle(location.pathname); + document.title = title; + }, [location.pathname]); const setActiveWorkspaceId = useStore((state) => state.setActiveWorkspaceId); const activeWorkspaceId = useStore((state) => state.workspaceId); From a5a1e0cdb02013af59cbb1b47a9d11ef50525e8a Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 13 Dec 2024 09:33:44 -0500 Subject: [PATCH 04/28] chore(CE): add resource link builder (#527) Co-authored-by: TivonB-AI2 <124182151+TivonB-AI2@users.noreply.github.com> --- .../concerns/resource_link_builder.rb | 62 ++++++++ .../concerns/resource_link_builder_spec.rb | 135 ++++++++++++++++++ 2 files changed, 197 insertions(+) create mode 100644 server/app/controllers/concerns/resource_link_builder.rb create mode 100644 server/spec/controllers/concerns/resource_link_builder_spec.rb diff --git a/server/app/controllers/concerns/resource_link_builder.rb b/server/app/controllers/concerns/resource_link_builder.rb new file mode 100644 index 00000000..de8aa2dc --- /dev/null +++ b/server/app/controllers/concerns/resource_link_builder.rb @@ -0,0 +1,62 @@ +# frozen_string_literal: true + +module ResourceLinkBuilder + extend ActiveSupport::Concern + def build_link!(resource_type: nil, resource: nil, resource_id: nil) + resource_type ||= controller_name.singularize.capitalize + case resource_type + when "Catalog", "Connector" + connectors_link(resource, resource_id) + when "Model" + models_link(resource, resource_id) + when "Schedule_sync", "Sync" + syncs_link(resource_id) + when "Data_app", "Custom_visual_component" + data_apps_link(resource_id) + when "Profile", "User" + members_link + else + reports_link(resource_id) + end + end + + private + + def connectors_link(resource, resource_id) + case resource.connector_type + when "source" + if resource.connector_category == "AI Model" + "/setup/sources/AIML%20Sources/#{resource_id}" + else + "/setup/sources/Data%20Sources/#{resource_id}" + end + else + "/setup/destinations/#{resource_id}" + end + end + + def models_link(resource, resource_id) + case resource.query_type + when "ai_ml" + "/define/models/ai/#{resource_id}" + else + "/define/models/#{resource_id}" + end + end + + def syncs_link(resource_id) + "/activate/syncs/#{resource_id}" + end + + def data_apps_link(resource_id) + "/data-apps/list/#{resource_id}" + end + + def members_link + "/settings/members" + end + + def reports_link(resource_id) + "/reports/#{resource_id}" + end +end diff --git a/server/spec/controllers/concerns/resource_link_builder_spec.rb b/server/spec/controllers/concerns/resource_link_builder_spec.rb new file mode 100644 index 00000000..629bb048 --- /dev/null +++ b/server/spec/controllers/concerns/resource_link_builder_spec.rb @@ -0,0 +1,135 @@ +# frozen_string_literal: true + +require "rails_helper" + +RSpec.describe ResourceLinkBuilder, type: :controller do + include ResourceLinkBuilder + let(:workspace) { create(:workspace) } + let(:user) { workspace.workspace_users.first.user } + let(:connector) { create(:connector, workspace:, connector_type: "source") } + let(:destination) { create(:connector, workspace:) } + let(:model) { create(:model, workspace:, connector:) } + let(:sync) do + build(:sync, workspace:, source: connector, destination:, model:, cursor_field: "timestamp", + current_cursor_field: "2022-01-01") + end + let!(:data_app) { create(:data_app, workspace:) } + let!(:visual_component) { create(:visual_component, data_app:, workspace:) } + let(:session) { create(:data_app_session, data_app:, session_id: "session_1") } + let!(:feedback) { create(:feedback, visual_component:) } + + before do + create(:catalog, connector:) + create(:catalog, connector: destination) + end + + describe "#build_link!" do + context "with valid params for connectors_link" do + it "creates a connectors link for AI Model" do + connector.id = 123 + connector.connector_category = "AI Model" + result = build_link!( + resource_type: "Catalog", + resource: connector, + resource_id: connector.id + ) + expect(result).to_not be_nil + expect(result).to eq("/setup/sources/AIML%20Sources/#{connector.id}") + end + end + + context "with valid params for connectors_link" do + it "creates a connectors link for Data Source" do + connector.id = 123 + result = build_link!( + resource_type: "Connector", + resource: connector, + resource_id: connector.id + ) + expect(result).to_not be_nil + expect(result).to eq("/setup/sources/Data%20Sources/#{connector.id}") + end + end + + context "with valid params for connectors_link" do + it "creates a connectors link for Destinations" do + connector.id = 123 + result = build_link!( + resource_type: "Connector", + resource: destination, + resource_id: destination.id + ) + expect(result).to_not be_nil + expect(result).to eq("/setup/destinations/#{destination.id}") + end + end + + context "with valid params for models_link" do + it "creates a model link for ai_ml query type models" do + model.query_type = "ai_ml" + result = build_link!( + resource_type: "Model", + resource: model, + resource_id: model.id + ) + expect(result).to_not be_nil + expect(result).to eq("/define/models/ai/#{model.id}") + end + end + + context "with valid params for models_link" do + it "creates a model link for raw_sql query type models" do + result = build_link!( + resource_type: "Model", + resource: model, + resource_id: model.id + ) + expect(result).to_not be_nil + expect(result).to eq("/define/models/#{model.id}") + end + end + + context "with valid params for syncs_link" do + it "creates a sync link for sync" do + result = build_link!( + resource_type: "Sync", + resource_id: sync.id + ) + expect(result).to_not be_nil + expect(result).to eq("/activate/syncs/#{sync.id}") + end + end + + context "with valid params for data_apps_link" do + it "creates a data app link for data app" do + result = build_link!( + resource_type: "Data_app", + resource_id: data_app.id + ) + expect(result).to_not be_nil + expect(result).to eq("/data-apps/list/#{data_app.id}") + end + end + + context "with valid params for members_link" do + it "creates a member link for user" do + result = build_link!( + resource_type: "User" + ) + expect(result).to_not be_nil + expect(result).to eq("/settings/members") + end + end + + context "with valid params for reports_link" do + it "creates a report link for feedback" do + result = build_link!( + resource_type: "Feedback", + resource_id: feedback.id + ) + expect(result).to_not be_nil + expect(result).to eq("/reports/#{feedback.id}") + end + end + end +end From 94b590b7382bc06242f269f2282c2627f8cbdb21 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 13 Dec 2024 20:07:43 +0530 Subject: [PATCH 05/28] feat(CE): login invited user fix (#695) (#526) Co-authored-by: Basil V Bose --- server/app/mailers/devise_mailer.rb | 1 + .../app/views/devise/mailer/invitation_instructions.html.erb | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/server/app/mailers/devise_mailer.rb b/server/app/mailers/devise_mailer.rb index 4f9f415f..80870b80 100644 --- a/server/app/mailers/devise_mailer.rb +++ b/server/app/mailers/devise_mailer.rb @@ -6,6 +6,7 @@ class DeviseMailer < Devise::Mailer def invitation_instructions(record, token, opts = {}) @workspace = opts[:workspace] @role = opts[:role] + @is_verified = opts[:is_verified] || false @token = token super end diff --git a/server/app/views/devise/mailer/invitation_instructions.html.erb b/server/app/views/devise/mailer/invitation_instructions.html.erb index 06a8630a..16b7ebf3 100644 --- a/server/app/views/devise/mailer/invitation_instructions.html.erb +++ b/server/app/views/devise/mailer/invitation_instructions.html.erb @@ -60,7 +60,7 @@ <%= "#{@resource.invited_by.name} has invited you to use AI Squared with them, in a workspace called #{@workspace.name}." %>

<% query_params = { invited: true, invited_user:@resource.email, invitation_token: @token, workspace_id: @workspace.id, workspace_name: @workspace.name, invited_by: @resource.invited_by.name } %> - <% custom_url = "#{ENV['UI_HOST']}/sign-up?#{query_params.to_query}" %> + <% custom_url = @is_verified ? "#{ENV['UI_HOST']}/sign-in" : "#{ENV['UI_HOST']}/sign-up?#{query_params.to_query}" %> - <% if @resource.invitation_due_at %> + <% if !@is_verified && @resource.invitation_due_at %>

<%= t("devise.mailer.invitation_instructions.accept_until", due_date: l(@resource.invitation_due_at, format: :'devise.mailer.invitation_instructions.accept_until_format')) %>

<% end %> From 3cd8cd65d1a6c87da28ba7f48ec463007807e0f7 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 16 Dec 2024 11:33:26 +0530 Subject: [PATCH 06/28] chore(CE): Add resource link to catalogs controller (#530) Co-authored-by: TivonB-AI2 <124182151+TivonB-AI2@users.noreply.github.com> --- server/app/controllers/api/v1/catalogs_controller.rb | 4 +++- server/spec/requests/api/v1/catalogs_controller_spec.rb | 4 ++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/server/app/controllers/api/v1/catalogs_controller.rb b/server/app/controllers/api/v1/catalogs_controller.rb index 195fe15a..cc7c73ec 100644 --- a/server/app/controllers/api/v1/catalogs_controller.rb +++ b/server/app/controllers/api/v1/catalogs_controller.rb @@ -5,6 +5,7 @@ module V1 class CatalogsController < ApplicationController include Catalogs include AuditLogger + include ResourceLinkBuilder before_action :set_connector, only: %i[create update] before_action :set_catalog, only: %i[update] @@ -70,7 +71,8 @@ def set_catalog def create_audit_log resource_id = params[:id] || params[:connector_id] - audit!(resource_id:, resource: @audit_resource, payload: @payload) + resource_link = build_link!(resource: @connector, resource_id: params[:connector_id]) + audit!(resource_id:, resource: @audit_resource, payload: @payload, resource_link:) end def catalog_params diff --git a/server/spec/requests/api/v1/catalogs_controller_spec.rb b/server/spec/requests/api/v1/catalogs_controller_spec.rb index 96f638db..2bd3693b 100644 --- a/server/spec/requests/api/v1/catalogs_controller_spec.rb +++ b/server/spec/requests/api/v1/catalogs_controller_spec.rb @@ -96,6 +96,7 @@ expect(audit_log.resource_id).to eq(connector.id) expect(audit_log.resource).to eq(connector.name) expect(audit_log.workspace_id).to eq(workspace.id) + expect(audit_log.resource_link).to eq("/setup/destinations/#{connector.id}") expect(audit_log.created_at).not_to be_nil expect(audit_log.updated_at).not_to be_nil end @@ -125,6 +126,7 @@ expect(audit_log.resource_id).to eq(connector.id) expect(audit_log.resource).to eq(connector.name) expect(audit_log.workspace_id).to eq(workspace.id) + expect(audit_log.resource_link).to eq("/setup/destinations/#{connector.id}") expect(audit_log.created_at).not_to be_nil expect(audit_log.updated_at).not_to be_nil end @@ -176,6 +178,7 @@ expect(audit_log.resource_id).to eq(existing_catalog.id) expect(audit_log.resource).to eq(connector.name) expect(audit_log.workspace_id).to eq(workspace.id) + expect(audit_log.resource_link).to eq("/setup/destinations/#{connector.id}") expect(audit_log.created_at).not_to be_nil expect(audit_log.updated_at).not_to be_nil end @@ -206,6 +209,7 @@ expect(audit_log.resource_id).to eq(existing_catalog.id) expect(audit_log.resource).to eq(connector.name) expect(audit_log.workspace_id).to eq(workspace.id) + expect(audit_log.resource_link).to eq("/setup/destinations/#{connector.id}") expect(audit_log.created_at).not_to be_nil expect(audit_log.updated_at).not_to be_nil end From 5e4fa604c1dc72760cc30e7419747823e17d0538 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 16 Dec 2024 09:33:10 -0500 Subject: [PATCH 07/28] chore(CE): Added exception handling for resource builder (#529) Co-authored-by: TivonB-AI2 --- server/app/controllers/concerns/resource_link_builder.rb | 8 ++++++++ .../controllers/concerns/resource_link_builder_spec.rb | 9 +++++++++ 2 files changed, 17 insertions(+) diff --git a/server/app/controllers/concerns/resource_link_builder.rb b/server/app/controllers/concerns/resource_link_builder.rb index de8aa2dc..db918f16 100644 --- a/server/app/controllers/concerns/resource_link_builder.rb +++ b/server/app/controllers/concerns/resource_link_builder.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true module ResourceLinkBuilder + # rubocop:disable Metrics/CyclomaticComplexity extend ActiveSupport::Concern def build_link!(resource_type: nil, resource: nil, resource_id: nil) resource_type ||= controller_name.singularize.capitalize @@ -18,6 +19,12 @@ def build_link!(resource_type: nil, resource: nil, resource_id: nil) else reports_link(resource_id) end + rescue StandardError => e + Rails.logger.error({ + error_message: e.message, + stack_trace: Rails.backtrace_cleaner.clean(e.backtrace) + }.to_s) + nil end private @@ -59,4 +66,5 @@ def members_link def reports_link(resource_id) "/reports/#{resource_id}" end + # rubocop:enable Metrics/CyclomaticComplexity end diff --git a/server/spec/controllers/concerns/resource_link_builder_spec.rb b/server/spec/controllers/concerns/resource_link_builder_spec.rb index 629bb048..dbb47195 100644 --- a/server/spec/controllers/concerns/resource_link_builder_spec.rb +++ b/server/spec/controllers/concerns/resource_link_builder_spec.rb @@ -131,5 +131,14 @@ expect(result).to eq("/reports/#{feedback.id}") end end + + context "when error arises" do + it "return nil" do + result = build_link!( + resource_type: "Catalog" + ) + expect(result).to be_nil + end + end end end From bc6b5b774d650163ef708132cc8f9b8bb93f59cb Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 16 Dec 2024 09:33:49 -0500 Subject: [PATCH 08/28] chore(CE): Add resource link to connectors controller (#533) Co-authored-by: TivonB-AI2 --- server/app/controllers/api/v1/connectors_controller.rb | 4 +++- server/spec/requests/api/v1/connectors_controller_spec.rb | 7 +++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/server/app/controllers/api/v1/connectors_controller.rb b/server/app/controllers/api/v1/connectors_controller.rb index 97838a89..0b210b9a 100644 --- a/server/app/controllers/api/v1/connectors_controller.rb +++ b/server/app/controllers/api/v1/connectors_controller.rb @@ -6,6 +6,7 @@ module V1 class ConnectorsController < ApplicationController include Connectors include AuditLogger + include ResourceLinkBuilder before_action :set_connector, only: %i[show update destroy discover query_source] # TODO: Enable this once we have query validation implemented for all the connectors # before_action :validate_query, only: %i[query_source] @@ -161,7 +162,8 @@ def validate_query def create_audit_log resource_id = @resource_id || params[:id] - audit!(action: @action, resource_id:, resource: @audit_resource, payload: @payload) + resource_link = @action == "delete" ? nil : build_link!(resource: @connector, resource_id:) + audit!(action: @action, resource_id:, resource: @audit_resource, payload: @payload, resource_link:) end def connector_params diff --git a/server/spec/requests/api/v1/connectors_controller_spec.rb b/server/spec/requests/api/v1/connectors_controller_spec.rb index c4129d3d..53591b2d 100644 --- a/server/spec/requests/api/v1/connectors_controller_spec.rb +++ b/server/spec/requests/api/v1/connectors_controller_spec.rb @@ -226,6 +226,7 @@ expect(audit_log.resource_id).to eq(response_hash["data"]["id"].to_i) expect(audit_log.resource).to eq(request_body.dig(:connector, :name)) expect(audit_log.workspace_id).to eq(workspace.id) + expect(audit_log.resource_link).to eq("/setup/sources/Data%20Sources/#{response_hash['data']['id'].to_i}") expect(audit_log.created_at).not_to be_nil expect(audit_log.updated_at).not_to be_nil end @@ -251,6 +252,7 @@ expect(audit_log.resource_id).to eq(response_hash["data"]["id"].to_i) expect(audit_log.resource).to eq(request_body.dig(:connector, :name)) expect(audit_log.workspace_id).to eq(workspace.id) + expect(audit_log.resource_link).to eq("/setup/sources/Data%20Sources/#{response_hash['data']['id'].to_i}") expect(audit_log.created_at).not_to be_nil expect(audit_log.updated_at).not_to be_nil end @@ -317,6 +319,7 @@ expect(audit_log.resource_id).to eq(connectors.second.id) expect(audit_log.resource).to eq(request_body.dig(:connector, :name)) expect(audit_log.workspace_id).to eq(workspace.id) + expect(audit_log.resource_link).to eq("/setup/sources/Data%20Sources/#{connectors.second.id}") expect(audit_log.created_at).not_to be_nil expect(audit_log.updated_at).not_to be_nil end @@ -340,6 +343,7 @@ expect(audit_log.resource_id).to eq(connectors.second.id) expect(audit_log.resource).to eq(request_body.dig(:connector, :name)) expect(audit_log.workspace_id).to eq(workspace.id) + expect(audit_log.resource_link).to eq("/setup/sources/Data%20Sources/#{connectors.second.id}") expect(audit_log.created_at).not_to be_nil expect(audit_log.updated_at).not_to be_nil end @@ -543,6 +547,7 @@ expect(audit_log.resource_id).to eq(connector.id) expect(audit_log.resource).to eq(connector.name) expect(audit_log.workspace_id).to eq(workspace.id) + expect(audit_log.resource_link).to eq("/setup/sources/Data%20Sources/#{connector.id}") expect(audit_log.created_at).not_to be_nil expect(audit_log.updated_at).not_to be_nil end @@ -565,6 +570,7 @@ expect(audit_log.resource_id).to eq(connector.id) expect(audit_log.resource).to eq(connector.name) expect(audit_log.workspace_id).to eq(workspace.id) + expect(audit_log.resource_link).to eq("/setup/sources/Data%20Sources/#{connector.id}") expect(audit_log.created_at).not_to be_nil expect(audit_log.updated_at).not_to be_nil end @@ -621,6 +627,7 @@ expect(audit_log.resource_id).to eq(connector.id) expect(audit_log.resource).to eq(connector.name) expect(audit_log.workspace_id).to eq(workspace.id) + expect(audit_log.resource_link).to eq("/setup/sources/Data%20Sources/#{connector.id}") expect(audit_log.created_at).not_to be_nil expect(audit_log.updated_at).not_to be_nil end From d672005d1f5b522dea1f6985483e728152623d8d Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 16 Dec 2024 09:34:34 -0500 Subject: [PATCH 09/28] chore(CE): Add resource link to Schedule Syncs Controller (#534) Co-authored-by: TivonB-AI2 --- server/app/controllers/api/v1/schedule_syncs_controller.rb | 4 +++- server/spec/requests/api/v1/schedule_syncs_controller_spec.rb | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/server/app/controllers/api/v1/schedule_syncs_controller.rb b/server/app/controllers/api/v1/schedule_syncs_controller.rb index 8e4bfeef..00e5c475 100644 --- a/server/app/controllers/api/v1/schedule_syncs_controller.rb +++ b/server/app/controllers/api/v1/schedule_syncs_controller.rb @@ -5,6 +5,7 @@ module V1 class ScheduleSyncsController < ApplicationController include Syncs include AuditLogger + include ResourceLinkBuilder before_action :set_sync before_action :validate_sync_status before_action :validate_sync_schedule_type @@ -54,7 +55,8 @@ def validate_sync_status def create_audit_log resource_id = @resource_id || params[:id] - audit!(action: @action, resource_id:, resource: @audit_resource, payload: @payload) + resource_link = @action == "delete" ? nil : build_link!(resource_id:) + audit!(action: @action, resource_id:, resource: @audit_resource, payload: @payload, resource_link:) end def validate_sync_schedule_type diff --git a/server/spec/requests/api/v1/schedule_syncs_controller_spec.rb b/server/spec/requests/api/v1/schedule_syncs_controller_spec.rb index ab1b2e12..c298771a 100644 --- a/server/spec/requests/api/v1/schedule_syncs_controller_spec.rb +++ b/server/spec/requests/api/v1/schedule_syncs_controller_spec.rb @@ -67,6 +67,7 @@ expect(audit_log.resource_id).to eq(request_body[:schedule_sync][:sync_id]) expect(audit_log.resource).to eq(request_body.dig(:sync, :name)) expect(audit_log.workspace_id).to eq(workspace.id) + expect(audit_log.resource_link).to eq("/activate/syncs/#{request_body[:schedule_sync][:sync_id]}") expect(audit_log.created_at).not_to be_nil expect(audit_log.updated_at).not_to be_nil end From 0935457ebd7844476255da7a50f7325a082e2af7 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 16 Dec 2024 09:35:17 -0500 Subject: [PATCH 10/28] chore(CE): Add resource link to Syncs Controller (#535) Co-authored-by: TivonB-AI2 --- server/app/controllers/api/v1/syncs_controller.rb | 4 +++- server/spec/requests/api/v1/syncs_controller_spec.rb | 10 ++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/server/app/controllers/api/v1/syncs_controller.rb b/server/app/controllers/api/v1/syncs_controller.rb index 290bb03f..97a396dd 100644 --- a/server/app/controllers/api/v1/syncs_controller.rb +++ b/server/app/controllers/api/v1/syncs_controller.rb @@ -6,6 +6,7 @@ module V1 class SyncsController < ApplicationController include Syncs include AuditLogger + include ResourceLinkBuilder before_action :set_sync, only: %i[show update enable destroy] before_action :modify_sync_params, only: %i[create update] @@ -129,7 +130,8 @@ def modify_sync_params def create_audit_log resource_id = @resource_id || params[:id] - audit!(action: @action, resource_id:, resource: @audit_resource, payload: @payload) + resource_link = @action == "delete" ? nil : build_link!(resource_id:) + audit!(action: @action, resource_id:, resource: @audit_resource, payload: @payload, resource_link:) end def sync_params diff --git a/server/spec/requests/api/v1/syncs_controller_spec.rb b/server/spec/requests/api/v1/syncs_controller_spec.rb index fa07b0af..49ea560b 100644 --- a/server/spec/requests/api/v1/syncs_controller_spec.rb +++ b/server/spec/requests/api/v1/syncs_controller_spec.rb @@ -225,6 +225,7 @@ expect(audit_log.resource_id).to eq(response_hash["data"]["id"].to_i) expect(audit_log.resource).to eq(request_body.dig(:sync, :name)) expect(audit_log.workspace_id).to eq(workspace.id) + expect(audit_log.resource_link).to eq("/activate/syncs/#{response_hash['data']['id'].to_i}") expect(audit_log.created_at).not_to be_nil expect(audit_log.updated_at).not_to be_nil end @@ -254,6 +255,7 @@ expect(audit_log.resource_id).to eq(response_hash["data"]["id"].to_i) expect(audit_log.resource).to eq(request_body.dig(:sync, :name)) expect(audit_log.workspace_id).to eq(workspace.id) + expect(audit_log.resource_link).to eq("/activate/syncs/#{response_hash['data']['id'].to_i}") expect(audit_log.created_at).not_to be_nil expect(audit_log.updated_at).not_to be_nil end @@ -284,6 +286,7 @@ expect(audit_log.resource_id).to eq(response_hash["data"]["id"].to_i) expect(audit_log.resource).to eq(request_body.dig(:sync, :name)) expect(audit_log.workspace_id).to eq(workspace.id) + expect(audit_log.resource_link).to eq("/activate/syncs/#{response_hash['data']['id'].to_i}") expect(audit_log.created_at).not_to be_nil expect(audit_log.updated_at).not_to be_nil end @@ -345,6 +348,7 @@ expect(audit_log.resource_id).to eq(response_hash["data"]["id"].to_i) expect(audit_log.resource).to eq(request_body.dig(:sync, :name)) expect(audit_log.workspace_id).to eq(workspace.id) + expect(audit_log.resource_link).to eq("/activate/syncs/#{response_hash['data']['id'].to_i}") expect(audit_log.created_at).not_to be_nil expect(audit_log.updated_at).not_to be_nil end @@ -371,6 +375,7 @@ expect(audit_log.resource_id).to eq(response_hash["data"]["id"].to_i) expect(audit_log.resource).to eq(request_body.dig(:sync, :name)) expect(audit_log.workspace_id).to eq(workspace.id) + expect(audit_log.resource_link).to eq("/activate/syncs/#{response_hash['data']['id'].to_i}") expect(audit_log.created_at).not_to be_nil expect(audit_log.updated_at).not_to be_nil end @@ -427,6 +432,7 @@ expect(audit_log.resource_id).to eq(syncs.first.id) expect(audit_log.resource).to eq(request_body.dig(:sync, :name)) expect(audit_log.workspace_id).to eq(workspace.id) + expect(audit_log.resource_link).to eq("/activate/syncs/#{syncs.first.id}") expect(audit_log.created_at).not_to be_nil expect(audit_log.updated_at).not_to be_nil end @@ -454,6 +460,7 @@ expect(audit_log.resource_id).to eq(syncs.first.id) expect(audit_log.resource).to eq(request_body.dig(:sync, :name)) expect(audit_log.workspace_id).to eq(workspace.id) + expect(audit_log.resource_link).to eq("/activate/syncs/#{syncs.first.id}") expect(audit_log.created_at).not_to be_nil expect(audit_log.updated_at).not_to be_nil end @@ -509,6 +516,7 @@ expect(audit_log.resource_id).to eq(syncs.first.id) expect(audit_log.resource).to eq(request_body.dig(:sync, :name)) expect(audit_log.workspace_id).to eq(workspace.id) + expect(audit_log.resource_link).to eq("/activate/syncs/#{syncs.first.id}") expect(audit_log.created_at).not_to be_nil expect(audit_log.updated_at).not_to be_nil end @@ -535,6 +543,7 @@ expect(audit_log.resource_id).to eq(syncs.first.id) expect(audit_log.resource).to eq(request_body.dig(:sync, :name)) expect(audit_log.workspace_id).to eq(workspace.id) + expect(audit_log.resource_link).to eq("/activate/syncs/#{syncs.first.id}") expect(audit_log.created_at).not_to be_nil expect(audit_log.updated_at).not_to be_nil end @@ -581,6 +590,7 @@ expect(audit_log.resource_id).to eq(syncs.first.id) expect(audit_log.resource).to eq(request_body.dig(:sync, :name)) expect(audit_log.workspace_id).to eq(workspace.id) + expect(audit_log.resource_link).to eq("/activate/syncs/#{syncs.first.id}") expect(audit_log.created_at).not_to be_nil expect(audit_log.updated_at).not_to be_nil end From de4ed5e49cfdb8705af5b4033ff8aed51ee7ce21 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 16 Dec 2024 09:35:44 -0500 Subject: [PATCH 11/28] chore(CE): Add resource link to Models Controller (#536) Co-authored-by: TivonB-AI2 --- server/app/controllers/api/v1/models_controller.rb | 4 +++- server/spec/requests/api/v1/models_controller_spec.rb | 8 ++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/server/app/controllers/api/v1/models_controller.rb b/server/app/controllers/api/v1/models_controller.rb index ac4937d5..df2a4fe3 100644 --- a/server/app/controllers/api/v1/models_controller.rb +++ b/server/app/controllers/api/v1/models_controller.rb @@ -5,6 +5,7 @@ module V1 class ModelsController < ApplicationController include Models include AuditLogger + include ResourceLinkBuilder attr_reader :connector, :model before_action :set_connector, only: %i[create] @@ -115,7 +116,8 @@ def validate_query def create_audit_log resource_id = @resource_id || params[:id] - audit!(action: @action, resource_id:, resource: @audit_resource, payload: @payload) + resource_link = @action == "delete" ? nil : build_link!(resource: @model, resource_id:) + audit!(action: @action, resource_id:, resource: @audit_resource, payload: @payload, resource_link:) end def model_params diff --git a/server/spec/requests/api/v1/models_controller_spec.rb b/server/spec/requests/api/v1/models_controller_spec.rb index ecc16e71..01264d1b 100644 --- a/server/spec/requests/api/v1/models_controller_spec.rb +++ b/server/spec/requests/api/v1/models_controller_spec.rb @@ -228,6 +228,7 @@ expect(audit_log.resource_id).to eq(response_hash["data"]["id"].to_i) expect(audit_log.resource).to eq(request_body.dig(:model, :name)) expect(audit_log.workspace_id).to eq(workspace.id) + expect(audit_log.resource_link).to eq("/define/models/#{response_hash['data']['id'].to_i}") expect(audit_log.created_at).not_to be_nil expect(audit_log.updated_at).not_to be_nil end @@ -272,6 +273,7 @@ expect(audit_log.resource_id).to eq(response_hash["data"]["id"].to_i) expect(audit_log.resource).to eq(request_body.dig(:model, :name)) expect(audit_log.workspace_id).to eq(workspace.id) + expect(audit_log.resource_link).to eq("/define/models/#{response_hash['data']['id'].to_i}") expect(audit_log.created_at).not_to be_nil expect(audit_log.updated_at).not_to be_nil end @@ -298,6 +300,7 @@ expect(audit_log.resource_id).to eq(response_hash["data"]["id"].to_i) expect(audit_log.resource).to eq(request_body.dig(:model, :name)) expect(audit_log.workspace_id).to eq(workspace.id) + expect(audit_log.resource_link).to eq("/define/models/#{response_hash['data']['id'].to_i}") expect(audit_log.created_at).not_to be_nil expect(audit_log.updated_at).not_to be_nil end @@ -334,6 +337,7 @@ expect(audit_log.resource_id).to eq(response_hash["data"]["id"].to_i) expect(audit_log.resource).to eq(request_body.dig(:model, :name)) expect(audit_log.workspace_id).to eq(workspace.id) + expect(audit_log.resource_link).to eq("/define/models/ai/#{response_hash['data']['id'].to_i}") expect(audit_log.created_at).not_to be_nil expect(audit_log.updated_at).not_to be_nil end @@ -424,6 +428,7 @@ expect(audit_log.resource_id).to eq(response_hash["data"]["id"].to_i) expect(audit_log.resource).to eq(request_body.dig(:model, :name)) expect(audit_log.workspace_id).to eq(workspace.id) + expect(audit_log.resource_link).to eq("/define/models/#{response_hash['data']['id'].to_i}") expect(audit_log.created_at).not_to be_nil expect(audit_log.updated_at).not_to be_nil end @@ -486,6 +491,7 @@ expect(audit_log.resource_id).to eq(models.second.id) expect(audit_log.resource).to eq(request_body.dig(:model, :name)) expect(audit_log.workspace_id).to eq(workspace.id) + expect(audit_log.resource_link).to eq("/define/models/#{models.second.id}") expect(audit_log.created_at).not_to be_nil expect(audit_log.updated_at).not_to be_nil end @@ -537,6 +543,7 @@ expect(audit_log.resource_id).to eq(models.second.id) expect(audit_log.resource).to eq(request_body.dig(:model, :name)) expect(audit_log.workspace_id).to eq(workspace.id) + expect(audit_log.resource_link).to eq("/define/models/#{models.second.id}") expect(audit_log.created_at).not_to be_nil expect(audit_log.updated_at).not_to be_nil end @@ -573,6 +580,7 @@ expect(audit_log.resource_id).to eq(models.second.id) expect(audit_log.resource).to eq(request_body.dig(:model, :name)) expect(audit_log.workspace_id).to eq(workspace.id) + expect(audit_log.resource_link).to eq("/define/models/ai/#{models.second.id}") expect(audit_log.created_at).not_to be_nil expect(audit_log.updated_at).not_to be_nil end From 6f6a63d7e52c1d11cdb5afc357c7d1bc25b5ed3f Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 16 Dec 2024 21:23:48 +0530 Subject: [PATCH 12/28] feat(CE): query_type filter for models (#528) Co-authored-by: Basil V Bose --- .../controllers/api/v1/models_controller.rb | 6 ++--- .../requests/api/v1/models_controller_spec.rb | 27 ++++++++++++++----- 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/server/app/controllers/api/v1/models_controller.rb b/server/app/controllers/api/v1/models_controller.rb index df2a4fe3..823c1dd4 100644 --- a/server/app/controllers/api/v1/models_controller.rb +++ b/server/app/controllers/api/v1/models_controller.rb @@ -17,9 +17,9 @@ class ModelsController < ApplicationController after_action :create_audit_log, only: %i[create update destroy] def index - filter = params[:query_type] || "all" - @models = current_workspace - .models.send(filter).page(params[:page] || 1) + query_type = request.query_parameters["query_type"].try(:split, ",") + workspace_models = current_workspace.models.page(params[:page] || 1) + @models = query_type ? workspace_models.where(query_type:) : workspace_models authorize @models render json: @models, status: :ok end diff --git a/server/spec/requests/api/v1/models_controller_spec.rb b/server/spec/requests/api/v1/models_controller_spec.rb index 01264d1b..9fcf5834 100644 --- a/server/spec/requests/api/v1/models_controller_spec.rb +++ b/server/spec/requests/api/v1/models_controller_spec.rb @@ -25,6 +25,9 @@ end let!(:raw_sql_model) { create(:model, query_type: :raw_sql, connector:, workspace:) } + let!(:dynamic_sql_model) do + create(:model, query_type: :dynamic_sql, connector:, configuration: { harvesters: [], json_schema: {} }, workspace:) + end let!(:dbt_model) { create(:model, query_type: :dbt, connector:, workspace:) } let!(:soql_model) { create(:model, query_type: :soql, connector:, workspace:) } let!(:ai_ml_model) do @@ -55,7 +58,7 @@ get "/api/v1/models", headers: auth_headers(user, workspace_id) expect(response).to have_http_status(:ok) response_hash = JSON.parse(response.body).with_indifferent_access - expect(response_hash[:data].count).to eql(7) + expect(response_hash[:data].count).to eql(8) expect(response_hash.dig(:data, 0, :type)).to eq("models") expect(response_hash.dig(:links, :first)).to include("http://www.example.com/api/v1/models?page=1") end @@ -65,7 +68,7 @@ get "/api/v1/models", headers: auth_headers(user, workspace_id) expect(response).to have_http_status(:ok) response_hash = JSON.parse(response.body).with_indifferent_access - expect(response_hash[:data].count).to eql(7) + expect(response_hash[:data].count).to eql(8) expect(response_hash.dig(:data, 0, :type)).to eq("models") expect(response_hash.dig(:links, :first)).to include("http://www.example.com/api/v1/models?page=1") end @@ -75,7 +78,7 @@ get "/api/v1/models", headers: auth_headers(user, workspace_id) expect(response).to have_http_status(:ok) response_hash = JSON.parse(response.body).with_indifferent_access - expect(response_hash[:data].count).to eql(7) + expect(response_hash[:data].count).to eql(8) expect(response_hash.dig(:data, 0, :type)).to eq("models") expect(response_hash.dig(:links, :first)).to include("http://www.example.com/api/v1/models?page=1") end @@ -83,22 +86,32 @@ it "filters models based on the query_type parameter" do get "/api/v1/models?query_type=data", headers: auth_headers(user, workspace_id) expect(response).to have_http_status(:ok) - expect(JSON.parse(response.body)["data"].map { |m| m["id"] }).not_to include(ai_ml_model.id) + response_ids = JSON.parse(response.body)["data"].map { |m| m["id"].to_i } + expect(response_ids).to eq([]) + end + + it "filters models based on the query_type parameter" do + get "/api/v1/models?query_type=dynamic_sql,ai_ml", headers: auth_headers(user, workspace_id) + expect(response).to have_http_status(:ok) + response_hash = JSON.parse(response.body).with_indifferent_access + expect(response_hash[:data].count).to eql(3) + response_ids = JSON.parse(response.body)["data"].map { |m| m["id"].to_i } + expect(response_ids).to match_array([ai_ml_source_model.id, dynamic_sql_model.id, ai_ml_model.id]) end it "filters models based on a different query_type" do get "/api/v1/models?query_type=ai_ml", headers: auth_headers(user, workspace_id) expect(response).to have_http_status(:ok) expect(JSON.parse(response.body)["data"].map do |m| - m["id"] - end).not_to include(raw_sql_model.id, dbt_model.id, soql_model.id) + m["id"].to_i + end).to match_array([ai_ml_model.id, ai_ml_source_model.id]) end it "returns all models" do get "/api/v1/models", headers: auth_headers(user, workspace_id) expect(response).to have_http_status(:ok) model_ids = JSON.parse(response.body)["data"].map { |m| m["id"] } - expect(model_ids.count).to eql(7) + expect(model_ids.count).to eql(8) end end end From 607f6eb5ec961361ba8ba9c16358f770cda87127 Mon Sep 17 00:00:00 2001 From: Sumit Dhanania Date: Mon, 16 Dec 2024 21:32:43 +0530 Subject: [PATCH 13/28] feat(CE): update model query types --- ui/src/components/ModelTable/ModelTable.tsx | 4 ++-- ui/src/services/models.ts | 6 +++--- ui/src/views/Models/ModelsList/ModelsList.tsx | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/ui/src/components/ModelTable/ModelTable.tsx b/ui/src/components/ModelTable/ModelTable.tsx index bc3920d1..c730a8c3 100644 --- a/ui/src/components/ModelTable/ModelTable.tsx +++ b/ui/src/components/ModelTable/ModelTable.tsx @@ -1,5 +1,5 @@ import GenerateTable from '@/components/Table/Table'; -import { getAllModels, GetAllModelsResponse } from '@/services/models'; +import { AllDataModels, getAllModels, GetAllModelsResponse } from '@/services/models'; import { addIconDataToArray, ConvertToTableData } from '@/utils'; import NoModels from '@/views/Models/NoModels'; import Loader from '@/components/Loader'; @@ -16,7 +16,7 @@ const ModelTable = ({ handleOnRowClick }: ModelTableProps): JSX.Element => { const { data } = useQueryWrapper, Error>( ['models', activeWorkspaceId], - () => getAllModels({ type: 'data' }), + () => getAllModels({ type: AllDataModels }), { refetchOnMount: true, refetchOnWindowFocus: false, diff --git a/ui/src/services/models.ts b/ui/src/services/models.ts index 33c6702c..6bd7dd36 100644 --- a/ui/src/services/models.ts +++ b/ui/src/services/models.ts @@ -47,10 +47,10 @@ export type GetAllModelsResponse = { attributes: ModelAttributes; }; -export type ModelQueryType = 'data' | 'ai_ml' | 'raw_sql' | 'dbt' | 'soql' | 'table_selector'; +export const AllDataModels = 'raw_sql,dbt,soql,table_selector'; export type GetAllModelsProps = { - type: ModelQueryType; + type: string; }; export const getModelPreview = async (query: string, connector_id: string): Promise => { @@ -59,7 +59,7 @@ export const getModelPreview = async (query: string, connector_id: string): Prom }; export const getAllModels = async ({ - type = 'data', + type = AllDataModels, }: GetAllModelsProps): Promise> => multiwovenFetch>({ method: 'get', diff --git a/ui/src/views/Models/ModelsList/ModelsList.tsx b/ui/src/views/Models/ModelsList/ModelsList.tsx index c214ddb3..95588c34 100644 --- a/ui/src/views/Models/ModelsList/ModelsList.tsx +++ b/ui/src/views/Models/ModelsList/ModelsList.tsx @@ -3,7 +3,7 @@ import TopBar from '@/components/TopBar'; import { Box } from '@chakra-ui/react'; import { FiPlus } from 'react-icons/fi'; import { useQuery } from '@tanstack/react-query'; -import { getAllModels, GetAllModelsResponse } from '@/services/models'; +import { AllDataModels, getAllModels, GetAllModelsResponse } from '@/services/models'; import Loader from '@/components/Loader'; import NoModels from '@/views/Models/NoModels'; import { useStore } from '@/stores'; @@ -22,7 +22,7 @@ const ModelsList = (): JSX.Element | null => { const { data, isLoading } = useQuery({ queryKey: ['models', activeWorkspaceId, 'data'], - queryFn: () => getAllModels({ type: 'data' }), + queryFn: () => getAllModels({ type: AllDataModels }), refetchOnMount: true, refetchOnWindowFocus: false, enabled: activeWorkspaceId > 0, From 7b3ed2c0aa468011e94776c75878383ff9dd84b9 Mon Sep 17 00:00:00 2001 From: "Rafael E. O'Neill" <106079170+RafaelOAiSquared@users.noreply.github.com> Date: Tue, 17 Dec 2024 01:01:30 -0400 Subject: [PATCH 14/28] Multiwoven release v0.38.0 (#540) Co-authored-by: github-actions --- release-notes.md | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/release-notes.md b/release-notes.md index 5242297e..93e7c7c2 100644 --- a/release-notes.md +++ b/release-notes.md @@ -2,23 +2,27 @@ All notable changes to this project will be documented in this file. -## [0.37.0] - 2024-12-06 +## [0.38.0] - 2024-12-16 -### 🐛 Bug Fixes +### 🚀 Features -- *(CE)* Audit logs serializer fix (#510) +- *(CE)* Added title change based on route +- *(CE)* Login invited user fix (#695) (#526) +- *(CE)* Query_type filter for models (#528) +- *(CE)* Update model query types ### ⚙️ Miscellaneous Tasks -- *(CE)* Remove audit logs from users controller (#509) -- *(CE)* Update audit log for schedule syncs controller (#513) -- *(CE)* Update audit logs for catalogs controller (#514) -- *(CE)* Update audit logs for connectors controller (#515) -- *(CE)* Update audit logs for models controller (#516) -- *(CE)* Update audit logs for syncs controller (#517) -- *(CE)* Add multiple_choice feedback type (#512) -- *(CE)* Add catalog and schedule sync resources (#511) -- *(CE)* Update HTTP model header (#519) -- *(CE)* Update Server Gem 0.15.8 (#518) +- *(CE)* Add create log message to HTTP model (#686) (#521) +- *(CE)* Update mailchimp catalog (#520) +- *(CE)* Update server gem 0.15.10 (#525) +- *(CE)* Add resource link to audit logs (#523) +- *(CE)* Add resource link builder (#527) +- *(CE)* Add resource link to catalogs controller (#530) +- *(CE)* Added exception handling for resource builder (#529) +- *(CE)* Add resource link to connectors controller (#533) +- *(CE)* Add resource link to Schedule Syncs Controller (#534) +- *(CE)* Add resource link to Syncs Controller (#535) +- *(CE)* Add resource link to Models Controller (#536) From 39932bee7bef836eb58b8bad081069f14257d991 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 17 Dec 2024 20:14:10 +0530 Subject: [PATCH 15/28] chore(CE): Add chatbot component_type to vc (#539) Co-authored-by: afthab vp --- server/app/models/visual_component.rb | 2 +- server/spec/models/visual_component_spec.rb | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/server/app/models/visual_component.rb b/server/app/models/visual_component.rb index 0c675e36..7e93c556 100644 --- a/server/app/models/visual_component.rb +++ b/server/app/models/visual_component.rb @@ -6,7 +6,7 @@ class VisualComponent < ApplicationRecord validates :model_id, presence: true validates :data_app_id, presence: true - enum component_type: { doughnut: 0, bar: 1, data_table: 2, visual_text: 3, custom: 4 } + enum component_type: { doughnut: 0, bar: 1, data_table: 2, visual_text: 3, custom: 4, chat_bot: 5 } belongs_to :workspace belongs_to :data_app diff --git a/server/spec/models/visual_component_spec.rb b/server/spec/models/visual_component_spec.rb index 57892eba..97d16e3e 100644 --- a/server/spec/models/visual_component_spec.rb +++ b/server/spec/models/visual_component_spec.rb @@ -9,7 +9,8 @@ it { should validate_presence_of(:data_app_id) } it { - should define_enum_for(:component_type).with_values(doughnut: 0, bar: 1, data_table: 2, visual_text: 3, custom: 4) + should define_enum_for(:component_type) + .with_values(doughnut: 0, bar: 1, data_table: 2, visual_text: 3, custom: 4, chat_bot: 5) } it { should belong_to(:workspace) } From fb78dd0c7d3e6a9885a44e83370bde3b22db6e5d Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 17 Dec 2024 21:49:29 -0500 Subject: [PATCH 16/28] chore(CE): Add resource link to audit logs serializer (#541) Co-authored-by: TivonB-AI2 <124182151+TivonB-AI2@users.noreply.github.com> --- server/app/serializers/audit_logs_serializer.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/app/serializers/audit_logs_serializer.rb b/server/app/serializers/audit_logs_serializer.rb index ba97db58..95dc9a6b 100644 --- a/server/app/serializers/audit_logs_serializer.rb +++ b/server/app/serializers/audit_logs_serializer.rb @@ -2,8 +2,8 @@ # app/serializers/audit_logs_serializer.rb class AuditLogsSerializer < ActiveModel::Serializer - attributes :id, :user_id, :user_name, :action, :resource_type, :resource_id, :resource, :workspace_id, - :metadata, :created_at, :updated_at + attributes :id, :user_id, :user_name, :action, :resource_type, :resource_id, :resource, :resource_link, + :workspace_id, :metadata, :created_at, :updated_at def user_name object.user&.name From 798e4f461303ae04928bd814345ac32850b8a3e3 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 19 Dec 2024 19:43:29 +0530 Subject: [PATCH 17/28] chore(CE): list api accept per page Co-authored-by: afthab vp --- server/app/controllers/api/v1/connectors_controller.rb | 2 +- server/app/controllers/api/v1/models_controller.rb | 2 +- server/app/controllers/api/v1/sync_records_controller.rb | 2 +- server/app/controllers/api/v1/sync_runs_controller.rb | 2 +- server/app/controllers/api/v1/syncs_controller.rb | 2 +- server/spec/requests/api/v1/connectors_controller_spec.rb | 5 +++-- server/spec/requests/api/v1/models_controller_spec.rb | 4 ++-- .../spec/requests/api/v1/sync_records_controller_spec.rb | 6 ++++-- server/spec/requests/api/v1/sync_runs_controller_spec.rb | 7 ++++++- server/spec/requests/api/v1/syncs_controller_spec.rb | 6 +++--- 10 files changed, 23 insertions(+), 15 deletions(-) diff --git a/server/app/controllers/api/v1/connectors_controller.rb b/server/app/controllers/api/v1/connectors_controller.rb index 0b210b9a..b596bec6 100644 --- a/server/app/controllers/api/v1/connectors_controller.rb +++ b/server/app/controllers/api/v1/connectors_controller.rb @@ -20,7 +20,7 @@ def index authorize @connectors @connectors = @connectors.send(params[:type].downcase) if params[:type] @connectors = @connectors.send(params[:category].downcase) if params[:category] - @connectors = @connectors.page(params[:page] || 1) + @connectors = @connectors.page(params[:page] || 1).per(params[:per_page]) render json: @connectors, status: :ok end diff --git a/server/app/controllers/api/v1/models_controller.rb b/server/app/controllers/api/v1/models_controller.rb index 823c1dd4..9efe1233 100644 --- a/server/app/controllers/api/v1/models_controller.rb +++ b/server/app/controllers/api/v1/models_controller.rb @@ -18,7 +18,7 @@ class ModelsController < ApplicationController def index query_type = request.query_parameters["query_type"].try(:split, ",") - workspace_models = current_workspace.models.page(params[:page] || 1) + workspace_models = current_workspace.models.page(params[:page] || 1).per(params[:per_page]) @models = query_type ? workspace_models.where(query_type:) : workspace_models authorize @models render json: @models, status: :ok diff --git a/server/app/controllers/api/v1/sync_records_controller.rb b/server/app/controllers/api/v1/sync_records_controller.rb index be7121f2..cf49b019 100644 --- a/server/app/controllers/api/v1/sync_records_controller.rb +++ b/server/app/controllers/api/v1/sync_records_controller.rb @@ -11,7 +11,7 @@ def index sync_records = @sync_run.sync_records.order(created_at: :asc) authorize sync_records sync_records = sync_records.where(status: params[:status]) if params[:status].present? - sync_records = sync_records.page(params[:page] || 1) + sync_records = sync_records.page(params[:page] || 1).per(params[:per_page]) render json: sync_records, status: :ok end diff --git a/server/app/controllers/api/v1/sync_runs_controller.rb b/server/app/controllers/api/v1/sync_runs_controller.rb index b35a3374..fe444510 100644 --- a/server/app/controllers/api/v1/sync_runs_controller.rb +++ b/server/app/controllers/api/v1/sync_runs_controller.rb @@ -11,7 +11,7 @@ def index sync_runs = @sync.sync_runs.order(started_at: :desc) authorize sync_runs sync_runs = sync_runs.where(status: params[:status]) if params[:status].present? - sync_runs = sync_runs.page(params[:page] || 1) + sync_runs = sync_runs.page(params[:page] || 1).per(params[:per_page]) render json: sync_runs, status: :ok end diff --git a/server/app/controllers/api/v1/syncs_controller.rb b/server/app/controllers/api/v1/syncs_controller.rb index 97a396dd..5319e34f 100644 --- a/server/app/controllers/api/v1/syncs_controller.rb +++ b/server/app/controllers/api/v1/syncs_controller.rb @@ -17,7 +17,7 @@ class SyncsController < ApplicationController def index @syncs = current_workspace - .syncs.all.page(params[:page] || 1) + .syncs.all.page(params[:page] || 1).per(params[:per_page]) authorize @syncs render json: @syncs, status: :ok end diff --git a/server/spec/requests/api/v1/connectors_controller_spec.rb b/server/spec/requests/api/v1/connectors_controller_spec.rb index 53591b2d..6f44d227 100644 --- a/server/spec/requests/api/v1/connectors_controller_spec.rb +++ b/server/spec/requests/api/v1/connectors_controller_spec.rb @@ -33,12 +33,13 @@ context "when it is an authenticated user" do it "returns success and all connectors admin role" do workspace.workspace_users.first.update(role: viewer_role) - get "/api/v1/connectors", headers: auth_headers(user, workspace_id) + get "/api/v1/connectors?page=1&per_page=20", headers: auth_headers(user, workspace_id) expect(response).to have_http_status(:ok) response_hash = JSON.parse(response.body).with_indifferent_access expect(response_hash[:data].count).to eql(connectors.count + 3) expect(response_hash.dig(:data, 0, :type)).to eq("connectors") - expect(response_hash.dig(:links, :first)).to include("http://www.example.com/api/v1/connectors?page=1") + expect(response_hash.dig(:links, :first)) + .to include("http://www.example.com/api/v1/connectors?page=1&per_page=20") end it "returns success and all connectors member role" do diff --git a/server/spec/requests/api/v1/models_controller_spec.rb b/server/spec/requests/api/v1/models_controller_spec.rb index 9fcf5834..80a378e4 100644 --- a/server/spec/requests/api/v1/models_controller_spec.rb +++ b/server/spec/requests/api/v1/models_controller_spec.rb @@ -55,12 +55,12 @@ context "when it is an authenticated user" do it "returns success and all model " do - get "/api/v1/models", headers: auth_headers(user, workspace_id) + get "/api/v1/models?page=1&per_page=20", headers: auth_headers(user, workspace_id) expect(response).to have_http_status(:ok) response_hash = JSON.parse(response.body).with_indifferent_access expect(response_hash[:data].count).to eql(8) expect(response_hash.dig(:data, 0, :type)).to eq("models") - expect(response_hash.dig(:links, :first)).to include("http://www.example.com/api/v1/models?page=1") + expect(response_hash.dig(:links, :first)).to include("http://www.example.com/api/v1/models?page=1&per_page=20") end it "returns success and all mode for viewer role" do diff --git a/server/spec/requests/api/v1/sync_records_controller_spec.rb b/server/spec/requests/api/v1/sync_records_controller_spec.rb index 6e2626f0..c15ceb25 100644 --- a/server/spec/requests/api/v1/sync_records_controller_spec.rb +++ b/server/spec/requests/api/v1/sync_records_controller_spec.rb @@ -40,7 +40,8 @@ context "when it is an authenticated user" do it "returns success and fetch sync " do - get "/api/v1/syncs/#{sync.id}/sync_runs/#{sync_run.id}/sync_records", headers: auth_headers(user, workspace_id) + get "/api/v1/syncs/#{sync.id}/sync_runs/#{sync_run.id}/sync_records?page=1&per_page=20", + headers: auth_headers(user, workspace_id) expect(response).to have_http_status(:ok) response_hash = JSON.parse(response.body).with_indifferent_access expect(response_hash[:data].size).to eq(2) @@ -61,7 +62,8 @@ expect(row.dig(:attributes, :logs)).to eq(sync_record.logs) expect { JSON.parse(row.dig(:attributes, :logs).to_json) }.not_to raise_error end - expect(response_hash.dig(:links, :first)).to include("http://www.example.com/api/v1/syncs/#{sync.id}/sync_runs/#{sync_run.id}/sync_records?page=1") + expect(response_hash.dig(:links, :first)) + .to include("http://www.example.com/api/v1/syncs/#{sync.id}/sync_runs/#{sync_run.id}/sync_records?page=1&per_page=20") end end diff --git a/server/spec/requests/api/v1/sync_runs_controller_spec.rb b/server/spec/requests/api/v1/sync_runs_controller_spec.rb index 18454f4a..a3e2f08a 100644 --- a/server/spec/requests/api/v1/sync_runs_controller_spec.rb +++ b/server/spec/requests/api/v1/sync_runs_controller_spec.rb @@ -38,7 +38,12 @@ context "when it is an authenticated user" do it "returns success and fetch sync " do +<<<<<<< HEAD get "/api/v1/syncs/#{sync.id}/sync_runs", headers: auth_headers(user, workspace_id) +======= + get "/api/v1/syncs/#{sync.id}/sync_runs?page=1&per_page=20", headers: auth_headers(user, workspace_id) + response_hash = JSON.parse(response.body).with_indifferent_access +>>>>>>> ac183819 (chore(CE): list api accept per page (#732)) expect(response).to have_http_status(:ok) response_hash = JSON.parse(response.body).with_indifferent_access expect(response_hash[:data].size).to eq(2) @@ -51,7 +56,7 @@ expect(row.dig(:attributes, :successful_rows)).to eq(sync_run.successful_rows) expect(row.dig(:attributes, :failed_rows)).to eq(sync_run.failed_rows) expect(row.dig(:attributes, :status)).to eq(sync_run.status) - expect(response_hash.dig(:links, :first)).to include("http://www.example.com/api/v1/syncs/#{sync.id}/sync_runs?page=1") + expect(response_hash.dig(:links, :first)).to include("http://www.example.com/api/v1/syncs/#{sync.id}/sync_runs?page=1&per_page=20") end end diff --git a/server/spec/requests/api/v1/syncs_controller_spec.rb b/server/spec/requests/api/v1/syncs_controller_spec.rb index 49ea560b..e2c90578 100644 --- a/server/spec/requests/api/v1/syncs_controller_spec.rb +++ b/server/spec/requests/api/v1/syncs_controller_spec.rb @@ -45,7 +45,7 @@ context "when it is an authenticated user" do it "returns success and get all syncs" do - get "/api/v1/syncs", headers: auth_headers(user, workspace_id) + get "/api/v1/syncs?page=1&per_page=20", headers: auth_headers(user, workspace_id) expect(response).to have_http_status(:ok) response_hash = JSON.parse(response.body).with_indifferent_access expect(response_hash[:data].count).to eql(syncs.count) @@ -55,7 +55,7 @@ expect(response_hash[:data][0][:attributes][:model].keys).to include("id", "name", "description", "query", "query_type", "primary_key", "created_at", "updated_at", "connector") - expect(response_hash.dig(:links, :first)).to include("http://www.example.com/api/v1/syncs?page=1") + expect(response_hash.dig(:links, :first)).to include("http://www.example.com/api/v1/syncs?page=1&per_page=20") end it "returns success and get all syncs for member role" do @@ -71,7 +71,7 @@ expect(response_hash[:data][0][:attributes][:model].keys).to include("id", "name", "description", "query", "query_type", "primary_key", "created_at", "updated_at", "connector") - expect(response_hash.dig(:links, :first)).to include("http://www.example.com/api/v1/syncs?page=1") + expect(response_hash.dig(:links, :first)).to include("http://www.example.com/api/v1/syncs?page=1&per_page=10") end it "returns success and get all syncs for viewer role" do From 4103196ad85e94f67797df5cf3e4306177fdfc5d Mon Sep 17 00:00:00 2001 From: "Rafael E. O'Neill" <106079170+RafaelOAiSquared@users.noreply.github.com> Date: Thu, 19 Dec 2024 14:19:55 -0400 Subject: [PATCH 18/28] chore(CE): Update Oracle Instant Client Installation #545 Co-authored-by: TivonB-AI2 <124182151+TivonB-AI2@users.noreply.github.com> --- .github/workflows/integrations-ci.yml | 14 ++++++++++++++ .github/workflows/integrations-main.yml | 15 +++++++++++++++ .github/workflows/server-ci.yml | 13 ++++++++++++- 3 files changed, 41 insertions(+), 1 deletion(-) diff --git a/.github/workflows/integrations-ci.yml b/.github/workflows/integrations-ci.yml index 535776ad..f7c9aa9a 100644 --- a/.github/workflows/integrations-ci.yml +++ b/.github/workflows/integrations-ci.yml @@ -33,11 +33,25 @@ jobs: - name: Download and Install Oracle Instant Client run: | +<<<<<<< HEAD sudo apt-get install -y libaio1 alien wget http://yum.oracle.com/repo/OracleLinux/OL7/oracle/instantclient/x86_64/getPackage/oracle-instantclient19.6-basic-19.6.0.0.0-1.x86_64.rpm wget http://yum.oracle.com/repo/OracleLinux/OL7/oracle/instantclient/x86_64/getPackage/oracle-instantclient19.6-devel-19.6.0.0.0-1.x86_64.rpm sudo alien -i --scripts oracle-instantclient*.rpm rm -f oracle-instantclient*.rpm +======= + wget http://ftp.debian.org/debian/pool/main/liba/libaio/libaio1_0.3.113-4_amd64.deb + sudo dpkg -i libaio1_0.3.113-4_amd64.deb + sudo apt-get install -f + sudo apt-get install -y alien unixodbc-dev + wget http://yum.oracle.com/repo/OracleLinux/OL7/oracle/instantclient/x86_64/getPackage/oracle-instantclient19.6-basic-19.6.0.0.0-1.x86_64.rpm + wget http://yum.oracle.com/repo/OracleLinux/OL7/oracle/instantclient/x86_64/getPackage/oracle-instantclient19.6-devel-19.6.0.0.0-1.x86_64.rpm + sudo alien -i --scripts oracle-instantclient*.rpm + rm -f oracle-instantclient*.rpm + echo "export LD_LIBRARY_PATH=/usr/lib/oracle/19.6/client64/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV + echo "export C_INCLUDE_PATH=/usr/include/oracle/19.6/client64:$C_INCLUDE_PATH" >> $GITHUB_ENV + echo "export CPLUS_INCLUDE_PATH=/usr/include/oracle/19.6/client64:$CPLUS_INCLUDE_PATH" >> $GITHUB_ENV +>>>>>>> 9879f3ea (chore(CE): Update Oracle Instant Client Installation (#723)) - name: Install FreeTDS for TinyTDS gem run: | diff --git a/.github/workflows/integrations-main.yml b/.github/workflows/integrations-main.yml index 8e9b3a16..23d09ccc 100644 --- a/.github/workflows/integrations-main.yml +++ b/.github/workflows/integrations-main.yml @@ -41,11 +41,26 @@ jobs: - name: Download and Install Oracle Instant Client run: | +<<<<<<< HEAD sudo apt-get install -y libaio1 alien wget http://yum.oracle.com/repo/OracleLinux/OL7/oracle/instantclient/x86_64/getPackage/oracle-instantclient19.6-basic-19.6.0.0.0-1.x86_64.rpm wget http://yum.oracle.com/repo/OracleLinux/OL7/oracle/instantclient/x86_64/getPackage/oracle-instantclient19.6-devel-19.6.0.0.0-1.x86_64.rpm sudo alien -i --scripts oracle-instantclient*.rpm rm -f oracle-instantclient*.rpm +======= + wget http://ftp.debian.org/debian/pool/main/liba/libaio/libaio1_0.3.113-4_amd64.deb + sudo dpkg -i libaio1_0.3.113-4_amd64.deb + sudo apt-get install -f + sudo apt-get install -y alien unixodbc-dev + wget http://yum.oracle.com/repo/OracleLinux/OL7/oracle/instantclient/x86_64/getPackage/oracle-instantclient19.6-basic-19.6.0.0.0-1.x86_64.rpm + wget http://yum.oracle.com/repo/OracleLinux/OL7/oracle/instantclient/x86_64/getPackage/oracle-instantclient19.6-devel-19.6.0.0.0-1.x86_64.rpm + sudo alien -i --scripts oracle-instantclient*.rpm + rm -f oracle-instantclient*.rpm + echo "export LD_LIBRARY_PATH=/usr/lib/oracle/19.6/client64/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV + echo "export C_INCLUDE_PATH=/usr/include/oracle/19.6/client64:$C_INCLUDE_PATH" >> $GITHUB_ENV + echo "export CPLUS_INCLUDE_PATH=/usr/include/oracle/19.6/client64:$CPLUS_INCLUDE_PATH" >> $GITHUB_ENV + +>>>>>>> 9879f3ea (chore(CE): Update Oracle Instant Client Installation (#723)) - name: Install FreeTDS for TinyTDS gem run: | diff --git a/.github/workflows/server-ci.yml b/.github/workflows/server-ci.yml index 3a79bc4c..cc576dfb 100644 --- a/.github/workflows/server-ci.yml +++ b/.github/workflows/server-ci.yml @@ -52,12 +52,23 @@ jobs: - name: Download and Install Oracle Instant Client run: | - sudo apt-get install -y libaio1 alien + wget http://ftp.debian.org/debian/pool/main/liba/libaio/libaio1_0.3.113-4_amd64.deb + sudo dpkg -i libaio1_0.3.113-4_amd64.deb + sudo apt-get install -f + sudo apt-get install -y alien unixodbc-dev wget http://yum.oracle.com/repo/OracleLinux/OL7/oracle/instantclient/x86_64/getPackage/oracle-instantclient19.6-basic-19.6.0.0.0-1.x86_64.rpm wget http://yum.oracle.com/repo/OracleLinux/OL7/oracle/instantclient/x86_64/getPackage/oracle-instantclient19.6-devel-19.6.0.0.0-1.x86_64.rpm sudo alien -i --scripts oracle-instantclient*.rpm rm -f oracle-instantclient*.rpm +<<<<<<< HEAD +======= + echo "export LD_LIBRARY_PATH=/usr/lib/oracle/19.6/client64/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV + echo "export C_INCLUDE_PATH=/usr/include/oracle/19.6/client64:$C_INCLUDE_PATH" >> $GITHUB_ENV + echo "export CPLUS_INCLUDE_PATH=/usr/include/oracle/19.6/client64:$CPLUS_INCLUDE_PATH" >> $GITHUB_ENV + + +>>>>>>> 9879f3ea (chore(CE): Update Oracle Instant Client Installation (#723)) - name: Install FreeTDS for TinyTDS gem run: | sudo apt-get install libc6-dev From 800b6fba482e5e63e764195e5f8df9368f2bdea1 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 24 Dec 2024 11:25:19 +0530 Subject: [PATCH 19/28] chore(CE): configure database from env (#549) Co-authored-by: datafloyd --- server/config/database.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/config/database.yml b/server/config/database.yml index 0de161a4..77737fa9 100644 --- a/server/config/database.yml +++ b/server/config/database.yml @@ -8,18 +8,18 @@ development: host: <%= ENV['DB_HOST'] %> username: <%= ENV['DB_USERNAME'] %> password: <%= ENV['DB_PASSWORD'] %> - database: multiwoven_server_development + database: <%= ENV.fetch("DB_NAME") { "multiwoven_server_development" } %> test: <<: *default host: <%= ENV['DB_HOST'] %> username: <%= ENV['DB_USERNAME'] %> password: <%= ENV['DB_PASSWORD'] %> - database: multiwoven_server_test + database: <%= ENV.fetch("DB_NAME") { "multiwoven_server_test" } %> production: <<: *default host: <%= ENV['DB_HOST'] %> username: <%= ENV['DB_USERNAME'] %> password: <%= ENV['DB_PASSWORD'] %> - database: multiwoven_server_production + database: <%= ENV.fetch("DB_NAME") { "multiwoven_server_production" } %> From 9dd96706ba2de216596f999e0d7cd37f693c4b0b Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 26 Dec 2024 13:48:07 +0530 Subject: [PATCH 20/28] feat(CE): stream support in http model (#532) Co-authored-by: afthab vp --- integrations/Gemfile.lock | 2 +- integrations/lib/multiwoven/integrations.rb | 2 + .../integrations/core/http_client.rb | 30 +---- .../integrations/core/http_helper.rb | 36 ++++++ .../integrations/core/source_connector.rb | 22 ++++ .../core/streaming_http_client.rb | 21 ++++ .../lib/multiwoven/integrations/rollout.rb | 2 +- .../integrations/source/http_model/client.rb | 108 ++++++++++-------- .../source/http_model/config/spec.json | 15 ++- .../core/streaming_http_client_spec.rb | 54 +++++++++ .../source/http_model/client_spec.rb | 70 +++++++++++- 11 files changed, 280 insertions(+), 82 deletions(-) create mode 100644 integrations/lib/multiwoven/integrations/core/http_helper.rb create mode 100644 integrations/lib/multiwoven/integrations/core/streaming_http_client.rb create mode 100644 integrations/spec/multiwoven/integrations/core/streaming_http_client_spec.rb diff --git a/integrations/Gemfile.lock b/integrations/Gemfile.lock index 31c38bc6..f801d94b 100644 --- a/integrations/Gemfile.lock +++ b/integrations/Gemfile.lock @@ -7,7 +7,7 @@ GIT PATH remote: . specs: - multiwoven-integrations (0.15.10) + multiwoven-integrations (0.15.11) MailchimpMarketing activesupport async-websocket diff --git a/integrations/lib/multiwoven/integrations.rb b/integrations/lib/multiwoven/integrations.rb index f8164b95..6c3ac56f 100644 --- a/integrations/lib/multiwoven/integrations.rb +++ b/integrations/lib/multiwoven/integrations.rb @@ -52,7 +52,9 @@ require_relative "integrations/core/base_connector" require_relative "integrations/core/source_connector" require_relative "integrations/core/destination_connector" +require_relative "integrations/core/http_helper" require_relative "integrations/core/http_client" +require_relative "integrations/core/streaming_http_client" require_relative "integrations/core/query_builder" # Source diff --git a/integrations/lib/multiwoven/integrations/core/http_client.rb b/integrations/lib/multiwoven/integrations/core/http_client.rb index 0e36b49a..fe0d2f1b 100644 --- a/integrations/lib/multiwoven/integrations/core/http_client.rb +++ b/integrations/lib/multiwoven/integrations/core/http_client.rb @@ -3,40 +3,14 @@ module Multiwoven module Integrations::Core class HttpClient + extend HttpHelper class << self def request(url, method, payload: nil, headers: {}, config: {}) uri = URI(url) - http = Net::HTTP.new(uri.host, uri.port) - http.use_ssl = (uri.scheme == "https") - - # Set timeout if provided - if config[:timeout] - timeout_value = config[:timeout].to_f - http.open_timeout = timeout_value - http.read_timeout = timeout_value - end - + http = configure_http(uri, config) request = build_request(method, uri, payload, headers) http.request(request) end - - private - - def build_request(method, uri, payload, headers) - request_class = case method.upcase - when Constants::HTTP_GET then Net::HTTP::Get - when Constants::HTTP_POST then Net::HTTP::Post - when Constants::HTTP_PUT then Net::HTTP::Put - when Constants::HTTP_PATCH then Net::HTTP::Patch - when Constants::HTTP_DELETE then Net::HTTP::Delete - else raise ArgumentError, "Unsupported HTTP method: #{method}" - end - - request = request_class.new(uri) - headers.each { |key, value| request[key] = value } - request.body = payload.to_json if payload && %w[POST PUT PATCH].include?(method.upcase) - request - end end end end diff --git a/integrations/lib/multiwoven/integrations/core/http_helper.rb b/integrations/lib/multiwoven/integrations/core/http_helper.rb new file mode 100644 index 00000000..f4a8df46 --- /dev/null +++ b/integrations/lib/multiwoven/integrations/core/http_helper.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +module Multiwoven + module Integrations::Core + module HttpHelper + def build_request(method, uri, payload, headers) + request_class = case method.upcase + when Constants::HTTP_GET then Net::HTTP::Get + when Constants::HTTP_POST then Net::HTTP::Post + when Constants::HTTP_PUT then Net::HTTP::Put + when Constants::HTTP_PATCH then Net::HTTP::Patch + when Constants::HTTP_DELETE then Net::HTTP::Delete + else raise ArgumentError, "Unsupported HTTP method: #{method}" + end + + request = request_class.new(uri) + headers.each { |key, value| request[key] = value } + request.body = payload.to_json if payload && %w[POST PUT PATCH].include?(method.upcase) + request + end + + def configure_http(uri, config) + http = Net::HTTP.new(uri.host, uri.port) + http.use_ssl = (uri.scheme == "https") + + if config[:timeout] + timeout_value = config[:timeout].to_f + http.open_timeout = timeout_value + http.read_timeout = timeout_value + end + + http + end + end + end +end diff --git a/integrations/lib/multiwoven/integrations/core/source_connector.rb b/integrations/lib/multiwoven/integrations/core/source_connector.rb index 76ac2264..ebe9e6bd 100644 --- a/integrations/lib/multiwoven/integrations/core/source_connector.rb +++ b/integrations/lib/multiwoven/integrations/core/source_connector.rb @@ -39,6 +39,28 @@ def batched_query(sql_query, limit, offset) # Appending the LIMIT and OFFSET clauses to the SQL query "#{sql_query} LIMIT #{limit} OFFSET #{offset}" end + + def send_request(options = {}) + Multiwoven::Integrations::Core::HttpClient.request( + options[:url], + options[:http_method], + payload: options[:payload], + headers: options[:headers], + config: options[:config] + ) + end + + def send_streaming_request(options = {}) + Multiwoven::Integrations::Core::StreamingHttpClient.request( + options[:url], + options[:http_method], + payload: options[:payload], + headers: options[:headers], + config: options[:config] + ) do |chunk| + yield chunk if block_given? # Pass each chunk for processing (streaming response) + end + end end end end diff --git a/integrations/lib/multiwoven/integrations/core/streaming_http_client.rb b/integrations/lib/multiwoven/integrations/core/streaming_http_client.rb new file mode 100644 index 00000000..ad8aa306 --- /dev/null +++ b/integrations/lib/multiwoven/integrations/core/streaming_http_client.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Multiwoven + module Integrations::Core + class StreamingHttpClient + extend HttpHelper + class << self + def request(url, method, payload: nil, headers: {}, config: {}) + uri = URI(url) + http = configure_http(uri, config) + request = build_request(method, uri, payload, headers) + http.request(request) do |response| + response.read_body do |chunk| + yield chunk if block_given? # Pass each response chunk + end + end + end + end + end + end +end diff --git a/integrations/lib/multiwoven/integrations/rollout.rb b/integrations/lib/multiwoven/integrations/rollout.rb index 961bc54b..f7dbce86 100644 --- a/integrations/lib/multiwoven/integrations/rollout.rb +++ b/integrations/lib/multiwoven/integrations/rollout.rb @@ -2,7 +2,7 @@ module Multiwoven module Integrations - VERSION = "0.15.10" + VERSION = "0.15.11" ENABLED_SOURCES = %w[ Snowflake diff --git a/integrations/lib/multiwoven/integrations/source/http_model/client.rb b/integrations/lib/multiwoven/integrations/source/http_model/client.rb index e5c8c045..8fe2883d 100644 --- a/integrations/lib/multiwoven/integrations/source/http_model/client.rb +++ b/integrations/lib/multiwoven/integrations/source/http_model/client.rb @@ -5,24 +5,17 @@ module HttpModel include Multiwoven::Integrations::Core class Client < SourceConnector def check_connection(connection_config) - connection_config = connection_config.with_indifferent_access - url_host = connection_config[:url_host] - http_method = connection_config[:http_method] - headers = connection_config[:headers] - payload = JSON.parse(connection_config[:request_format]) - config = connection_config[:config] - config[:timeout] ||= 30 - response = send_request(url_host, http_method, payload, headers, config) - if success?(response) - success_status - else - failure_status(nil) - end + connection_config = prepare_config(connection_config) + response = send_request( + url: connection_config[:url_host], + http_method: connection_config[:http_method], + payload: JSON.parse(connection_config[:request_format]), + headers: connection_config[:headers], + config: connection_config[:config] + ) + success?(response) ? success_status : failure_status(nil) rescue StandardError => e - handle_exception(e, { - context: "HTTP MODEL:CHECK_CONNECTION:EXCEPTION", - type: "error" - }) + handle_exception(e, { context: "HTTP MODEL:CHECK_CONNECTION:EXCEPTION", type: "error" }) failure_status(e) end @@ -31,40 +24,66 @@ def discover(_connection_config = nil) catalog = build_catalog(catalog_json) catalog.to_multiwoven_message rescue StandardError => e - handle_exception(e, { - context: "HTTP MODEL:DISCOVER:EXCEPTION", - type: "error" - }) + handle_exception(e, { context: "HTTP MODEL:DISCOVER:EXCEPTION", type: "error" }) end def read(sync_config) - connection_config = sync_config.source.connection_specification - connection_config = connection_config.with_indifferent_access + connection_config = prepare_config(sync_config.source.connection_specification) + stream = connection_config[:is_stream] ||= false # The server checks the ConnectorQueryType. # If it's "ai_ml," the server calculates the payload and passes it as a query in the sync config model protocol. # This query is then sent to the AI/ML model. - payload = JSON.parse(sync_config.model.query) - run_model(connection_config, payload) + payload = parse_json(sync_config.model.query) + + if stream + run_model_stream(connection_config, payload) { |message| yield message if block_given? } + else + run_model(connection_config, payload) + end rescue StandardError => e - handle_exception(e, { - context: "HTTP MODEL:READ:EXCEPTION", - type: "error" - }) + handle_exception(e, { context: "HTTP MODEL:READ:EXCEPTION", type: "error" }) end private + def prepare_config(config) + config.with_indifferent_access.tap do |conf| + conf[:config][:timeout] ||= 30 + end + end + + def parse_json(json_string) + JSON.parse(json_string) + rescue JSON::ParserError => e + handle_exception(e, { context: "HTTP MODEL:PARSE_JSON:EXCEPTION", type: "error" }) + {} + end + def run_model(connection_config, payload) - connection_config = connection_config.with_indifferent_access - url_host = connection_config[:url_host] - headers = connection_config[:headers] - config = connection_config[:config] - http_method = connection_config[:http_method] - config[:timeout] ||= 30 - response = send_request(url_host, http_method, payload, headers, config) + response = send_request( + url: connection_config[:url_host], + http_method: connection_config[:http_method], + payload: payload, + headers: connection_config[:headers], + config: connection_config[:config] + ) process_response(response) rescue StandardError => e - handle_exception(e, context: "HTTP MODEL:RUN_MODEL:EXCEPTION", type: "error") + handle_exception(e, { context: "HTTP MODEL:RUN_MODEL:EXCEPTION", type: "error" }) + end + + def run_model_stream(connection_config, payload) + send_streaming_request( + url: connection_config[:url_host], + http_method: connection_config[:http_method], + payload: payload, + headers: connection_config[:headers], + config: connection_config[:config] + ) do |chunk| + process_streaming_response(chunk) { |message| yield message if block_given? } + end + rescue StandardError => e + handle_exception(e, { context: "HTTP MODEL:RUN_STREAM_MODEL:EXCEPTION", type: "error" }) end def process_response(response) @@ -74,16 +93,15 @@ def process_response(response) else create_log_message("HTTP MODEL:RUN_MODEL", "error", "request failed: #{response.body}") end + rescue StandardError => e + handle_exception(e, { context: "HTTP MODEL:PROCESS_RESPONSE:EXCEPTION", type: "error" }) end - def send_request(url, http_method, payload, headers, config) - Multiwoven::Integrations::Core::HttpClient.request( - url, - http_method, - payload: payload, - headers: headers, - config: config - ) + def process_streaming_response(chunk) + data = JSON.parse(chunk) + yield [RecordMessage.new(data: data, emitted_at: Time.now.to_i).to_multiwoven_message] if block_given? + rescue StandardError => e + handle_exception(e, { context: "HTTP MODEL:PROCESS_STREAMING_RESPONSE:EXCEPTION", type: "error" }) end end end diff --git a/integrations/lib/multiwoven/integrations/source/http_model/config/spec.json b/integrations/lib/multiwoven/integrations/source/http_model/config/spec.json index 2b6989bb..9e83dc8b 100644 --- a/integrations/lib/multiwoven/integrations/source/http_model/config/spec.json +++ b/integrations/lib/multiwoven/integrations/source/http_model/config/spec.json @@ -19,10 +19,17 @@ "title": "URL", "order": 1 }, + "is_stream": { + "type": "boolean", + "title": "Streaming Enabled", + "description": "Enables data streaming for such as chat, when supported by the model. When true, messages and model data are processed in chunks for immediate delivery, enhancing responsiveness. Default is false, processing only after the entire response is received.", + "default": false, + "order": 2 + }, "headers": { "title": "HTTP Headers", "description": "Custom headers to include in the HTTP request. Useful for authentication, content type specifications, and other request metadata.", - "order": 2, + "order": 3, "additionalProperties": { "type": "string" }, @@ -42,21 +49,21 @@ "order": 0 } }, - "order": 3 + "order": 4 }, "request_format": { "title": "Request Format", "description": "Sample Request Format", "type": "string", "x-request-format": true, - "order": 4 + "order": 5 }, "response_format": { "title": "Response Format", "description": "Sample Response Format", "type": "string", "x-response-format": true, - "order": 5 + "order": 6 } } } diff --git a/integrations/spec/multiwoven/integrations/core/streaming_http_client_spec.rb b/integrations/spec/multiwoven/integrations/core/streaming_http_client_spec.rb new file mode 100644 index 00000000..c01c8c6e --- /dev/null +++ b/integrations/spec/multiwoven/integrations/core/streaming_http_client_spec.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +module Multiwoven + module Integrations::Core + RSpec.describe StreamingHttpClient do + describe ".request" do + let(:url) { "https://example.com/api/stream" } + let(:method) { "GET" } + let(:headers) { { "Authorization" => "Bearer token" } } + let(:config) { { timeout: 5 } } + let(:mock_response) { double("Net::HTTPResponse", code: "200") } + + it "makes a streaming HTTP request" do + http = double("Net::HTTP") + allow(Net::HTTP).to receive(:new).and_return(http) + allow(http).to receive(:use_ssl=) + allow(http).to receive(:open_timeout=) + allow(http).to receive(:read_timeout=) + allow(http).to receive(:request) do |&block| + block.call(mock_response) + end + + allow(mock_response).to receive(:read_body) do |&block| + block.call("chunk1") + block.call("chunk2") + block.call("chunk3") + end + chunks = [] + described_class.request(url, method, headers: headers, config: config) do |chunk| + chunks << chunk + end + expect(chunks).to eq(%w[chunk1 chunk2 chunk3]) + end + + it "handles errors gracefully" do + http = double("Net::HTTP") + allow(Net::HTTP).to receive(:new).and_return(http) + allow(http).to receive(:use_ssl=) + allow(http).to receive(:open_timeout=) + allow(http).to receive(:read_timeout=) + allow(http).to receive(:request) do |&block| + block.call(mock_response) + end + + allow(mock_response).to receive(:read_body).and_raise(StandardError, "Network error") + + expect do + described_class.request(url, method, headers: headers, config: config) { |_| } + end.to raise_error(StandardError, "Network error") + end + end + end + end +end diff --git a/integrations/spec/multiwoven/integrations/source/http_model/client_spec.rb b/integrations/spec/multiwoven/integrations/source/http_model/client_spec.rb index c69e0ae2..cce8315f 100644 --- a/integrations/spec/multiwoven/integrations/source/http_model/client_spec.rb +++ b/integrations/spec/multiwoven/integrations/source/http_model/client_spec.rb @@ -63,7 +63,10 @@ end let(:sync_config) { Multiwoven::Integrations::Protocol::SyncConfig.from_json(sync_config_json.to_json) } - + let(:sync_config_stream) do + sync_config_json[:source][:connection_specification][:is_stream] = true + Multiwoven::Integrations::Protocol::SyncConfig.from_json(sync_config_json.to_json) + end before do allow(Multiwoven::Integrations::Core::HttpClient).to receive(:request) end @@ -144,7 +147,7 @@ context "when the read is successful" do let(:response_body) { { "message" => "Hello! how can I help" }.to_json } before do - response = Net::HTTPSuccess.new("1.1", "200", "Unauthorized") + response = Net::HTTPSuccess.new("1.1", "200", "success") response.content_type = "application/json" url = sync_config_json[:source][:connection_specification][:url_host] http_method = sync_config_json[:source][:connection_specification][:http_method] @@ -168,7 +171,7 @@ end end - context "when the write operation fails" do + context "when the read operation fails" do let(:response_body) { { "message" => "failed" }.to_json } before do response = Net::HTTPSuccess.new("1.1", "401", "Unauthorized") @@ -198,4 +201,65 @@ end end end + + describe "#read with is_stream = true" do + context "when the read is successful" do + before do + payload = sync_config_json[:model][:query] + streaming_chunk_first = { "message" => "streaming data 1" }.to_json + streaming_chunk_second = { "message" => "streaming data 2" }.to_json + + allow(Multiwoven::Integrations::Core::StreamingHttpClient).to receive(:request) + .with(sync_config_json[:source][:connection_specification][:url_host], + sync_config_json[:source][:connection_specification][:http_method], + payload: JSON.parse(payload), + headers: sync_config_json[:source][:connection_specification][:headers], + config: sync_config_json[:source][:connection_specification][:config]) + .and_yield(streaming_chunk_first) + .and_yield(streaming_chunk_second) + + response = Net::HTTPSuccess.new("1.1", "200", "success") + response.content_type = "application/json" + end + + it "streams data and processes chunks" do + results = [] + client.read(sync_config_stream) { |message| results << message } + expect(results.first).to be_an(Array) + expect(results.first.first.record).to be_a(Multiwoven::Integrations::Protocol::RecordMessage) + expect(results.first.first.record.data["message"]).to eq("streaming data 1") + + expect(results.last).to be_an(Array) + expect(results.last.first.record).to be_a(Multiwoven::Integrations::Protocol::RecordMessage) + expect(results.last.first.record.data["message"]).to eq("streaming data 2") + end + end + + context "when streaming fails on a chunk" do + let(:streaming_chunk_first) { { "message" => "streaming data chunk 1" }.to_json } + + before do + url = sync_config_json[:source][:connection_specification][:url_host] + http_method = sync_config_json[:stream][:request_method] + headers = sync_config_json[:source][:connection_specification][:headers] + config = sync_config_json[:source][:connection_specification][:config] + allow(Multiwoven::Integrations::Core::StreamingHttpClient).to receive(:request) + .with(url, + http_method, + payload: JSON.parse(payload.to_json), + headers: headers, + config: config) + .and_yield(streaming_chunk_first) + .and_raise(StandardError, "Streaming error on chunk 2") + end + + it "handles streaming errors gracefully" do + results = [] + client.read(sync_config_stream) { |message| results << message } + expect(results.last).to be_an(Array) + expect(results.last.first.record).to be_a(Multiwoven::Integrations::Protocol::RecordMessage) + expect(results.last.first.record.data["message"]).to eq("streaming data chunk 1") + end + end + end end From 9bb9389510421e6fd40ab7db4a34455b19457641 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 26 Dec 2024 13:50:50 +0530 Subject: [PATCH 21/28] chore(CE): list api accept per page (#543) Co-authored-by: afthab vp --- .../spec/requests/api/v1/models_controller_spec.rb | 12 ++++++++++++ .../requests/api/v1/sync_runs_controller_spec.rb | 8 +++----- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/server/spec/requests/api/v1/models_controller_spec.rb b/server/spec/requests/api/v1/models_controller_spec.rb index 80a378e4..447c9ecf 100644 --- a/server/spec/requests/api/v1/models_controller_spec.rb +++ b/server/spec/requests/api/v1/models_controller_spec.rb @@ -63,6 +63,18 @@ expect(response_hash.dig(:links, :first)).to include("http://www.example.com/api/v1/models?page=1&per_page=20") end + it "returns success and no models when the data is empty" do + workspace.models.destroy_all + get "/api/v1/models", headers: auth_headers(user, workspace_id) + expect(response).to have_http_status(:ok) + response_hash = JSON.parse(response.body).with_indifferent_access + expect(response_hash[:data].count).to eql(0) + expect(response_hash.dig(:links, :first)).to include("http://www.example.com/api/v1/models?page=1") + expect(response_hash.dig(:links, :last)).to include("http://www.example.com/api/v1/models?page=1") + expect(response_hash.dig(:links, :next)).to be_nil + expect(response_hash.dig(:links, :prev)).to be_nil + end + it "returns success and all mode for viewer role" do workspace.workspace_users.first.update(role: viewer_role) get "/api/v1/models", headers: auth_headers(user, workspace_id) diff --git a/server/spec/requests/api/v1/sync_runs_controller_spec.rb b/server/spec/requests/api/v1/sync_runs_controller_spec.rb index a3e2f08a..278b277e 100644 --- a/server/spec/requests/api/v1/sync_runs_controller_spec.rb +++ b/server/spec/requests/api/v1/sync_runs_controller_spec.rb @@ -38,15 +38,13 @@ context "when it is an authenticated user" do it "returns success and fetch sync " do -<<<<<<< HEAD - get "/api/v1/syncs/#{sync.id}/sync_runs", headers: auth_headers(user, workspace_id) -======= get "/api/v1/syncs/#{sync.id}/sync_runs?page=1&per_page=20", headers: auth_headers(user, workspace_id) response_hash = JSON.parse(response.body).with_indifferent_access ->>>>>>> ac183819 (chore(CE): list api accept per page (#732)) expect(response).to have_http_status(:ok) - response_hash = JSON.parse(response.body).with_indifferent_access expect(response_hash[:data].size).to eq(2) + first_row_date = DateTime.parse(response_hash[:data].first.dig(:attributes, :updated_at)) + second_row_date = DateTime.parse(response_hash[:data].last.dig(:attributes, :updated_at)) + expect(first_row_date).to be > second_row_date response_hash[:data].each_with_index do |row, _index| sync_run = sync_runs.find { |sr| sr.id == row[:id].to_i } From c0ce381379ab14370827a16d3e013996144a8a43 Mon Sep 17 00:00:00 2001 From: "Rafael E. O'Neill" <106079170+RafaelOAiSquared@users.noreply.github.com> Date: Thu, 26 Dec 2024 04:25:54 -0400 Subject: [PATCH 22/28] fix(CE): pagination fix for empty data (#546) Co-authored-by: afthab vp --- server/config/initializers/custom_pagination_links.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/config/initializers/custom_pagination_links.rb b/server/config/initializers/custom_pagination_links.rb index a57ca3ce..e39f93cc 100644 --- a/server/config/initializers/custom_pagination_links.rb +++ b/server/config/initializers/custom_pagination_links.rb @@ -23,8 +23,8 @@ def pages_from {}.tap do |pages| pages[:first] = FIRST_PAGE pages[:prev] = first_page? ? nil : collection.current_page - FIRST_PAGE - pages[:next] = last_page? ? nil : collection.current_page + FIRST_PAGE - pages[:last] = collection.total_pages + pages[:next] = (!last_page? && collection.total_pages > 1) ? collection.current_page + FIRST_PAGE : nil + pages[:last] = [collection.total_pages, FIRST_PAGE].max end end From ed80bd86cefa8a86702d1950a9e270af7131ba2f Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 26 Dec 2024 14:50:31 +0530 Subject: [PATCH 23/28] feat(CE): Open AI ai ml source connector (#547) Co-authored-by: afthab vp --- integrations/Gemfile.lock | 2 +- integrations/lib/multiwoven/integrations.rb | 1 + .../multiwoven/integrations/core/constants.rb | 2 + .../lib/multiwoven/integrations/rollout.rb | 3 +- .../source/http_model/config/meta.json | 2 +- .../integrations/source/open_ai/client.rb | 117 ++++++++ .../source/open_ai/config/catalog.json | 6 + .../source/open_ai/config/meta.json | 15 + .../source/open_ai/config/spec.json | 54 ++++ .../integrations/source/open_ai/icon.svg | 1 + .../source/open_ai/client_spec.rb | 260 ++++++++++++++++++ 11 files changed, 460 insertions(+), 3 deletions(-) create mode 100644 integrations/lib/multiwoven/integrations/source/open_ai/client.rb create mode 100644 integrations/lib/multiwoven/integrations/source/open_ai/config/catalog.json create mode 100644 integrations/lib/multiwoven/integrations/source/open_ai/config/meta.json create mode 100644 integrations/lib/multiwoven/integrations/source/open_ai/config/spec.json create mode 100644 integrations/lib/multiwoven/integrations/source/open_ai/icon.svg create mode 100644 integrations/spec/multiwoven/integrations/source/open_ai/client_spec.rb diff --git a/integrations/Gemfile.lock b/integrations/Gemfile.lock index f801d94b..58b9073a 100644 --- a/integrations/Gemfile.lock +++ b/integrations/Gemfile.lock @@ -7,7 +7,7 @@ GIT PATH remote: . specs: - multiwoven-integrations (0.15.11) + multiwoven-integrations (0.16.0) MailchimpMarketing activesupport async-websocket diff --git a/integrations/lib/multiwoven/integrations.rb b/integrations/lib/multiwoven/integrations.rb index 6c3ac56f..1218e832 100644 --- a/integrations/lib/multiwoven/integrations.rb +++ b/integrations/lib/multiwoven/integrations.rb @@ -73,6 +73,7 @@ require_relative "integrations/source/aws_sagemaker_model/client" require_relative "integrations/source/google_vertex_model/client" require_relative "integrations/source/http_model/client" +require_relative "integrations/source/open_ai/client" # Destination require_relative "integrations/destination/klaviyo/client" diff --git a/integrations/lib/multiwoven/integrations/core/constants.rb b/integrations/lib/multiwoven/integrations/core/constants.rb index fcabb17f..7a02845c 100644 --- a/integrations/lib/multiwoven/integrations/core/constants.rb +++ b/integrations/lib/multiwoven/integrations/core/constants.rb @@ -63,6 +63,8 @@ module Constants # google sheets GOOGLE_SHEETS_SCOPE = "https://www.googleapis.com/auth/drive" GOOGLE_SPREADSHEET_ID_REGEX = %r{/d/([-\w]{20,})/}.freeze + + OPEN_AI_URL = "https://api.openai.com/v1/chat/completions" end end end diff --git a/integrations/lib/multiwoven/integrations/rollout.rb b/integrations/lib/multiwoven/integrations/rollout.rb index f7dbce86..add3445d 100644 --- a/integrations/lib/multiwoven/integrations/rollout.rb +++ b/integrations/lib/multiwoven/integrations/rollout.rb @@ -2,7 +2,7 @@ module Multiwoven module Integrations - VERSION = "0.15.11" + VERSION = "0.16.0" ENABLED_SOURCES = %w[ Snowflake @@ -20,6 +20,7 @@ module Integrations AwsSagemakerModel VertexModel HttpModel + OpenAI ].freeze ENABLED_DESTINATIONS = %w[ diff --git a/integrations/lib/multiwoven/integrations/source/http_model/config/meta.json b/integrations/lib/multiwoven/integrations/source/http_model/config/meta.json index f1e85784..373ea430 100644 --- a/integrations/lib/multiwoven/integrations/source/http_model/config/meta.json +++ b/integrations/lib/multiwoven/integrations/source/http_model/config/meta.json @@ -4,7 +4,7 @@ "title": "HTTP Model Endpoint", "connector_type": "source", "category": "AI Model", - "documentation_url": "https://docs.mutliwoven.com", + "documentation_url": "https://docs.mutltiwoven.com", "github_issue_label": "source-http-model", "icon": "icon.svg", "license": "MIT", diff --git a/integrations/lib/multiwoven/integrations/source/open_ai/client.rb b/integrations/lib/multiwoven/integrations/source/open_ai/client.rb new file mode 100644 index 00000000..cce398de --- /dev/null +++ b/integrations/lib/multiwoven/integrations/source/open_ai/client.rb @@ -0,0 +1,117 @@ +# frozen_string_literal: true + +module Multiwoven::Integrations::Source + module OpenAI + include Multiwoven::Integrations::Core + class Client < SourceConnector + def check_connection(connection_config) + connection_config = prepare_config(connection_config) + response = send_request( + url: OPEN_AI_URL, + http_method: HTTP_POST, + payload: JSON.parse(connection_config[:request_format]), + headers: auth_headers(connection_config[:api_key]), + config: connection_config[:config] + ) + success?(response) ? success_status : failure_status(nil) + rescue StandardError => e + handle_exception(e, { context: "OPEN AI:CHECK_CONNECTION:EXCEPTION", type: "error" }) + failure_status(e) + end + + def discover(_connection_config = nil) + catalog_json = read_json(CATALOG_SPEC_PATH) + catalog = build_catalog(catalog_json) + catalog.to_multiwoven_message + rescue StandardError => e + handle_exception(e, { context: "OPEN AI:DISCOVER:EXCEPTION", type: "error" }) + end + + def read(sync_config) + connection_config = prepare_config(sync_config.source.connection_specification) + stream = connection_config[:is_stream] ||= false + # The server checks the ConnectorQueryType. + # If it's "ai_ml," the server calculates the payload and passes it as a query in the sync config model protocol. + # This query is then sent to the AI/ML model. + payload = parse_json(sync_config.model.query) + + if stream + run_model_stream(connection_config, payload) { |message| yield message if block_given? } + else + run_model(connection_config, payload) + end + rescue StandardError => e + handle_exception(e, { context: "OPEN AI:READ:EXCEPTION", type: "error" }) + end + + private + + def prepare_config(config) + config.with_indifferent_access.tap do |conf| + conf[:config][:timeout] ||= 30 + end + end + + def parse_json(json_string) + JSON.parse(json_string) + rescue JSON::ParserError => e + handle_exception(e, { context: "OPEN AI:PARSE_JSON:EXCEPTION", type: "error" }) + {} + end + + def run_model(connection_config, payload) + response = send_request( + url: OPEN_AI_URL, + http_method: HTTP_POST, + payload: payload, + headers: auth_headers(connection_config[:api_key]), + config: connection_config[:config] + ) + process_response(response) + rescue StandardError => e + handle_exception(e, { context: "OPEN AI:RUN_MODEL:EXCEPTION", type: "error" }) + end + + def run_model_stream(connection_config, payload) + send_streaming_request( + url: OPEN_AI_URL, + http_method: HTTP_POST, + payload: payload, + headers: auth_headers(connection_config[:api_key]), + config: connection_config[:config] + ) do |chunk| + process_streaming_response(chunk) { |message| yield message if block_given? } + end + rescue StandardError => e + handle_exception(e, { context: "OPEN AI:RUN_STREAM_MODEL:EXCEPTION", type: "error" }) + end + + def process_response(response) + if success?(response) + data = JSON.parse(response.body) + [RecordMessage.new(data: data, emitted_at: Time.now.to_i).to_multiwoven_message] + else + create_log_message("OPEN AI:RUN_MODEL", "error", "request failed: #{response.body}") + end + rescue StandardError => e + handle_exception(e, { context: "OPEN AI:PROCESS_RESPONSE:EXCEPTION", type: "error" }) + end + + def extract_data_entries(chunk) + chunk.split(/^data: /).map(&:strip).reject(&:empty?) + end + + def process_streaming_response(chunk) + data_entries = extract_data_entries(chunk) + data_entries.each do |entry| + next if entry == "[DONE]" + + data = parse_json(entry) + yield [RecordMessage.new(data: data, emitted_at: Time.now.to_i).to_multiwoven_message] if block_given? + rescue StandardError => e + handle_exception(e, { context: "OPEN AI:PROCESS_STREAMING_RESPONSE:EXCEPTION", type: "error", entry: entry }) + end + end + end + end +end diff --git a/integrations/lib/multiwoven/integrations/source/open_ai/config/catalog.json b/integrations/lib/multiwoven/integrations/source/open_ai/config/catalog.json new file mode 100644 index 00000000..dacb788b --- /dev/null +++ b/integrations/lib/multiwoven/integrations/source/open_ai/config/catalog.json @@ -0,0 +1,6 @@ +{ + "request_rate_limit": 600, + "request_rate_limit_unit": "minute", + "request_rate_concurrency": 10, + "streams": [] +} diff --git a/integrations/lib/multiwoven/integrations/source/open_ai/config/meta.json b/integrations/lib/multiwoven/integrations/source/open_ai/config/meta.json new file mode 100644 index 00000000..8cb0b153 --- /dev/null +++ b/integrations/lib/multiwoven/integrations/source/open_ai/config/meta.json @@ -0,0 +1,15 @@ +{ + "data": { + "name": "OpenAI", + "title": "OpenAI Model Endpoint", + "connector_type": "source", + "category": "AI Model", + "documentation_url": "https://docs.mutltiwoven.com", + "github_issue_label": "source-open-ai-model", + "icon": "icon.svg", + "license": "MIT", + "release_stage": "alpha", + "support_level": "community", + "tags": ["language:ruby", "multiwoven"] + } +} diff --git a/integrations/lib/multiwoven/integrations/source/open_ai/config/spec.json b/integrations/lib/multiwoven/integrations/source/open_ai/config/spec.json new file mode 100644 index 00000000..b123a89c --- /dev/null +++ b/integrations/lib/multiwoven/integrations/source/open_ai/config/spec.json @@ -0,0 +1,54 @@ +{ + "documentation_url": "https://docs.multiwoven.com/integrations/source/open-ai-endpoint", + "stream_type": "user_defined", + "connector_query_type": "ai_ml", + "connection_specification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Open AI Endpoint", + "type": "object", + "required": ["api_key", "request_format", "response_format"], + "properties": { + "api_key": { + "type": "string", + "multiwoven_secret": true, + "title": "API Key", + "order": 0 + }, + "is_stream": { + "type": "boolean", + "title": "Streaming Enabled", + "description": "Enables data streaming for such as chat, when supported by the model. When true, messages and model data are processed in chunks for immediate delivery, enhancing responsiveness. Default is false, processing only after the entire response is received.", + "default": false, + "order": 1 + }, + "config": { + "title": "", + "type": "object", + "properties": { + "timeout": { + "type": "string", + "default": "30", + "title": "HTTP Timeout", + "description": "The maximum time, in seconds, to wait for a response from the server before the request is canceled.", + "order": 0 + } + }, + "order": 2 + }, + "request_format": { + "title": "Request Format", + "description": "Sample Request Format", + "type": "string", + "x-request-format": true, + "order": 3 + }, + "response_format": { + "title": "Response Format", + "description": "Sample Response Format", + "type": "string", + "x-response-format": true, + "order": 4 + } + } + } +} diff --git a/integrations/lib/multiwoven/integrations/source/open_ai/icon.svg b/integrations/lib/multiwoven/integrations/source/open_ai/icon.svg new file mode 100644 index 00000000..859d7af3 --- /dev/null +++ b/integrations/lib/multiwoven/integrations/source/open_ai/icon.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/integrations/spec/multiwoven/integrations/source/open_ai/client_spec.rb b/integrations/spec/multiwoven/integrations/source/open_ai/client_spec.rb new file mode 100644 index 00000000..0ccf4a85 --- /dev/null +++ b/integrations/spec/multiwoven/integrations/source/open_ai/client_spec.rb @@ -0,0 +1,260 @@ +# frozen_string_literal: true + +RSpec.describe Multiwoven::Integrations::Source::OpenAI::Client do + include WebMock::API + + before(:each) do + WebMock.disable_net_connect!(allow_localhost: true) + end + + let(:client) { described_class.new } + let(:mock_http_session) { double("Net::Http::Session") } + let(:api_key) { "test_api_key" } + let(:payload) do + { + queries: "Hello there" + } + end + + let(:sync_config_json) do + { + source: { + name: "DestinationConnectorName", + type: "destination", + connection_specification: { + api_key: api_key, + config: { + timeout: 25 + }, + request_format: payload.to_json + } + }, + destination: { + name: "Http", + type: "destination", + connection_specification: { + example_destination_key: "example_destination_value" + } + }, + model: { + name: "ExampleModel", + query: payload.to_json, + query_type: "ai_ml", + primary_key: "id" + }, + stream: { + name: "example_stream", + json_schema: { "field1": "type1" }, + request_method: "POST", + request_rate_limit: 4, + rate_limit_unit_seconds: 1 + }, + sync_mode: "full_refresh", + cursor_field: "timestamp", + destination_sync_mode: "upsert", + sync_id: "1" + } + end + + let(:sync_config) { Multiwoven::Integrations::Protocol::SyncConfig.from_json(sync_config_json.to_json) } + let(:sync_config_stream) do + sync_config_json[:source][:connection_specification][:is_stream] = true + Multiwoven::Integrations::Protocol::SyncConfig.from_json(sync_config_json.to_json) + end + before do + allow(Multiwoven::Integrations::Core::HttpClient).to receive(:request) + end + let(:headers) do + { + "Accept" => "application/json", + "Authorization" => "Bearer #{api_key}", + "Content-Type" => "application/json" + } + end + let(:endpoint) { "https://api.openai.com/v1/chat/completions" } + + describe "#check_connection" do + context "when the connection is successful" do + let(:response_body) { { "message" => "success" }.to_json } + before do + response = Net::HTTPSuccess.new("1.1", "200", "Unauthorized") + response.content_type = "application/json" + config = sync_config_json[:source][:connection_specification][:config] + allow(response).to receive(:body).and_return(response_body) + allow(Multiwoven::Integrations::Core::HttpClient).to receive(:request) + .with(endpoint, + "POST", + payload: JSON.parse(payload.to_json), + headers: headers, + config: config) + .and_return(response) + end + + it "returns a successful connection status" do + response = client.check_connection(sync_config_json[:source][:connection_specification]) + expect(response).to be_a(Multiwoven::Integrations::Protocol::MultiwovenMessage) + expect(response.connection_status.status).to eq("succeeded") + end + end + + context "when the connection fails" do + let(:response_body) { { "message" => "failed" }.to_json } + before do + response = Net::HTTPSuccess.new("1.1", "401", "Unauthorized") + response.content_type = "application/json" + allow(response).to receive(:body).and_return(response_body) + allow(Multiwoven::Integrations::Core::HttpClient).to receive(:request) + .with(endpoint, + "POST", + headers: headers) + .and_return(response) + end + + it "returns a failed connection status with an error message" do + response = client.check_connection(sync_config_json[:source][:connection_specification]) + + expect(response).to be_a(Multiwoven::Integrations::Protocol::MultiwovenMessage) + expect(response.connection_status.status).to eq("failed") + end + end + end + + describe "#discover" do + it "successfully returns the catalog message" do + message = client.discover(nil) + catalog = message.catalog + expect(catalog).to be_a(Multiwoven::Integrations::Protocol::Catalog) + expect(catalog.request_rate_limit).to eql(600) + expect(catalog.request_rate_limit_unit).to eql("minute") + expect(catalog.request_rate_concurrency).to eql(10) + end + + it "handles exceptions during discovery" do + allow(client).to receive(:read_json).and_raise(StandardError.new("test error")) + expect(client).to receive(:handle_exception).with( + an_instance_of(StandardError), + hash_including(context: "OPEN AI:DISCOVER:EXCEPTION", type: "error") + ) + client.discover + end + end + + describe "#read" do + context "when the read is successful" do + let(:response_body) { { "message" => "Hello! how can I help" }.to_json } + before do + response = Net::HTTPSuccess.new("1.1", "200", "success") + response.content_type = "application/json" + config = sync_config_json[:source][:connection_specification][:config] + allow(response).to receive(:body).and_return(response_body) + allow(Multiwoven::Integrations::Core::HttpClient).to receive(:request) + .with(endpoint, + "POST", + payload: JSON.parse(payload.to_json), + headers: headers, + config: config) + .and_return(response) + end + + it "successfully reads records" do + records = client.read(sync_config) + expect(records).to be_an(Array) + expect(records.first.record).to be_a(Multiwoven::Integrations::Protocol::RecordMessage) + expect(records.first.record.data).to eq(JSON.parse(response_body)) + end + end + + context "when the read operation fails" do + let(:response_body) { { "message" => "failed" }.to_json } + before do + response = Net::HTTPSuccess.new("1.1", "401", "Unauthorized") + response.content_type = "application/json" + config = sync_config_json[:source][:connection_specification][:config] + allow(response).to receive(:body).and_return(response_body) + allow(Multiwoven::Integrations::Core::HttpClient).to receive(:request) + .with(endpoint, + "POST", + headers: headers, + config: config) + .and_return(response) + end + + it "handles exceptions during reading" do + error_instance = StandardError.new("test error") + allow(client).to receive(:run_model).and_raise(error_instance) + expect(client).to receive(:handle_exception).with( + error_instance, + hash_including(context: "OPEN AI:READ:EXCEPTION", type: "error") + ) + + client.read(sync_config) + end + end + end + + describe "#read with is_stream = true" do + context "when the read is successful" do + before do + payload = sync_config_json[:model][:query] + streaming_chunk_first = <<~DATA + data: {"choices":[{"delta":{"content":"How I "}}]} + + data: {"choices":[{"delta":{"content":"can help "}}]} + DATA + streaming_chunk_second = "data: {\"choices\":[{\"delta\":{\"content\":\"you\"}}]}\n\n" + + allow(Multiwoven::Integrations::Core::StreamingHttpClient).to receive(:request) + .with(endpoint, + "POST", + payload: JSON.parse(payload), + headers: headers, + config: sync_config_json[:source][:connection_specification][:config]) + .and_yield(streaming_chunk_first) + .and_yield(streaming_chunk_second) + + response = Net::HTTPSuccess.new("1.1", "200", "success") + response.content_type = "application/json" + end + + it "streams data and processes chunks" do + results = [] + client.read(sync_config_stream) { |message| results << message } + expect(results.first).to be_an(Array) + expect(results.first.first.record).to be_a(Multiwoven::Integrations::Protocol::RecordMessage) + expect(results.first.first.record.data.dig("choices", 0, "delta", "content")).to eq("How I ") + + expect(results[1]).to be_an(Array) + expect(results[1].first.record).to be_a(Multiwoven::Integrations::Protocol::RecordMessage) + expect(results[1].first.record.data.dig("choices", 0, "delta", "content")).to eq("can help ") + + expect(results[2]).to be_an(Array) + expect(results[2].first.record).to be_a(Multiwoven::Integrations::Protocol::RecordMessage) + expect(results[2].first.record.data.dig("choices", 0, "delta", "content")).to eq("you") + end + end + + context "when streaming fails on a chunk" do + let(:streaming_chunk_first) { { "message" => "streaming data chunk 1" }.to_json } + + before do + config = sync_config_json[:source][:connection_specification][:config] + allow(Multiwoven::Integrations::Core::StreamingHttpClient).to receive(:request) + .with(endpoint, + "POST", + payload: JSON.parse(payload.to_json), + headers: headers, + config: config) + .and_yield(streaming_chunk_first) + .and_raise(StandardError, "Streaming error on chunk 2") + end + + it "handles streaming errors gracefully" do + results = [] + client.read(sync_config_stream) { |message| results << message } + expect(results.last).to be_an(Array) + expect(results.last.first.record).to be_a(Multiwoven::Integrations::Protocol::RecordMessage) + expect(results.last.first.record.data["message"]).to eq("streaming data chunk 1") + end + end + end +end From f30b2c3ab193426b2e3dc5d361ff00d55ce2ee79 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 27 Dec 2024 15:12:39 +0530 Subject: [PATCH 24/28] chore(CE): Update HTTP model spec (#551) Co-authored-by: TivonB-AI2 <124182151+TivonB-AI2@users.noreply.github.com> --- integrations/Gemfile.lock | 2 +- integrations/lib/multiwoven/integrations/rollout.rb | 2 +- .../multiwoven/integrations/source/http_model/config/spec.json | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/integrations/Gemfile.lock b/integrations/Gemfile.lock index 58b9073a..cd85a166 100644 --- a/integrations/Gemfile.lock +++ b/integrations/Gemfile.lock @@ -7,7 +7,7 @@ GIT PATH remote: . specs: - multiwoven-integrations (0.16.0) + multiwoven-integrations (0.16.1) MailchimpMarketing activesupport async-websocket diff --git a/integrations/lib/multiwoven/integrations/rollout.rb b/integrations/lib/multiwoven/integrations/rollout.rb index add3445d..6f6409fb 100644 --- a/integrations/lib/multiwoven/integrations/rollout.rb +++ b/integrations/lib/multiwoven/integrations/rollout.rb @@ -2,7 +2,7 @@ module Multiwoven module Integrations - VERSION = "0.16.0" + VERSION = "0.16.1" ENABLED_SOURCES = %w[ Snowflake diff --git a/integrations/lib/multiwoven/integrations/source/http_model/config/spec.json b/integrations/lib/multiwoven/integrations/source/http_model/config/spec.json index 9e83dc8b..194808b4 100644 --- a/integrations/lib/multiwoven/integrations/source/http_model/config/spec.json +++ b/integrations/lib/multiwoven/integrations/source/http_model/config/spec.json @@ -6,7 +6,7 @@ "$schema": "http://json-schema.org/draft-07/schema#", "title": "HTTP Model Endpoint", "type": "object", - "required": ["url_host"], + "required": ["url_host", "http_method"], "properties": { "http_method": { "type": "string", From fcb9004bd7daba94f6d9afdaed163191a4dab564 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 27 Dec 2024 15:55:05 +0530 Subject: [PATCH 25/28] chore(CE): Update server gem 0.16.1 (#550) Co-authored-by: TivonB-AI2 <124182151+TivonB-AI2@users.noreply.github.com> --- server/Gemfile | 2 +- server/Gemfile.lock | 40 +++++++++++++++++++++------------------- 2 files changed, 22 insertions(+), 20 deletions(-) diff --git a/server/Gemfile b/server/Gemfile index 107eb5cc..4b39537e 100644 --- a/server/Gemfile +++ b/server/Gemfile @@ -13,7 +13,7 @@ gem "interactor", "~> 3.0" gem "ruby-odbc", git: "https://github.com/Multiwoven/ruby-odbc.git" -gem "multiwoven-integrations", "~> 0.15.10" +gem "multiwoven-integrations", "~> 0.16.1" gem "temporal-ruby", github: "coinbase/temporal-ruby" diff --git a/server/Gemfile.lock b/server/Gemfile.lock index fb29b8c8..7cd7f621 100644 --- a/server/Gemfile.lock +++ b/server/Gemfile.lock @@ -1677,7 +1677,7 @@ GEM crass (1.0.6) css_parser (1.17.1) addressable - csv (3.3.0) + csv (3.3.1) data_migrate (9.4.0) activerecord (>= 6.1) railties (>= 6.1) @@ -1759,18 +1759,19 @@ GEM railties (>= 5.0.0) faker (3.2.3) i18n (>= 1.8.11, < 2) - faraday (2.11.0) - faraday-net_http (>= 2.0, < 3.4) + faraday (2.12.2) + faraday-net_http (>= 2.0, < 3.5) + json logger faraday-follow_redirects (0.3.0) faraday (>= 1, < 3) faraday-mashify (0.1.1) faraday (~> 2.0) hashie - faraday-multipart (1.0.4) - multipart-post (~> 2) - faraday-net_http (3.3.0) - net-http + faraday-multipart (1.1.0) + multipart-post (~> 2.0) + faraday-net_http (3.4.0) + net-http (>= 0.5.0) faraday-retry (2.2.1) faraday (~> 2.0) ffi (1.17.0-aarch64-linux-gnu) @@ -1810,14 +1811,14 @@ GEM mutex_m representable (~> 3.0) retriable (>= 2.0, < 4.a) - google-apis-sheets_v4 (0.38.0) + google-apis-sheets_v4 (0.39.0) google-apis-core (>= 0.15.0, < 2.a) google-cloud-ai_platform-v1 (0.60.0) gapic-common (>= 0.21.1, < 2.a) google-cloud-errors (~> 1.0) google-cloud-location (>= 0.7, < 2.a) google-iam-v1 (>= 0.7, < 2.a) - google-cloud-bigquery (1.51.0) + google-cloud-bigquery (1.51.1) bigdecimal (~> 3.0) concurrent-ruby (~> 1.0) google-apis-bigquery_v2 (~> 0.71) @@ -1849,7 +1850,7 @@ GEM grpc (~> 1.41) googleapis-common-protos-types (1.11.0) google-protobuf (~> 3.18) - googleauth (1.12.0) + googleauth (1.12.2) faraday (>= 1.0, < 3.a) google-cloud-env (~> 2.2) google-logging-utils (~> 0.1) @@ -1889,7 +1890,7 @@ GEM interactor (3.1.2) io-console (0.7.1) io-endpoint (0.14.0) - io-event (1.7.4) + io-event (1.7.5) io-stream (0.6.1) irb (1.11.1) rdoc @@ -1924,7 +1925,7 @@ GEM kaminari-core (1.2.2) language_server-protocol (3.17.0.3) liquid (5.4.0) - logger (1.6.2) + logger (1.6.4) loofah (2.22.0) crass (~> 1.0.2) nokogiri (>= 1.12.0) @@ -1940,7 +1941,7 @@ GEM msgpack (1.7.2) multi_json (1.15.0) multipart-post (2.4.1) - multiwoven-integrations (0.15.10) + multiwoven-integrations (0.16.1) MailchimpMarketing activesupport async-websocket @@ -2096,8 +2097,8 @@ GEM responders (3.1.1) actionpack (>= 5.2) railties (>= 5.2) - restforce (7.5.0) - faraday (>= 1.1.0, < 2.12.0) + restforce (8.0.0) + faraday (>= 1.1.0, < 3.0.0) faraday-follow_redirects (<= 0.3.0, < 1.0.0) faraday-multipart (>= 1.0.0, < 2.0.0) faraday-net_http (< 4.0.0) @@ -2171,15 +2172,16 @@ GEM faraday-multipart gli hashie - sorbet-runtime (0.5.11690) + sorbet-runtime (0.5.11707) stringio (3.1.0) - stripe (13.2.0) + stripe (13.3.0) strong_migrations (1.8.0) activerecord (>= 5.2) thor (1.3.0) timecop (0.9.8) timeout (0.4.1) - tiny_tds (2.1.7) + tiny_tds (3.0.0) + bigdecimal (~> 3) traces (0.14.1) trailblazer-option (0.1.2) typhoeus (1.4.1) @@ -2247,7 +2249,7 @@ DEPENDENCIES jwt kaminari liquid - multiwoven-integrations (~> 0.15.10) + multiwoven-integrations (~> 0.16.1) mysql2 newrelic_rpm parallel From 8c395c7597948317fb4c1b884690298786029027 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 27 Dec 2024 15:55:49 +0530 Subject: [PATCH 26/28] chore(CE): add payload limit to databricks model (#554) Co-authored-by: TivonB-AI2 <124182151+TivonB-AI2@users.noreply.github.com> --- integrations/Gemfile.lock | 2 +- .../lib/multiwoven/integrations/rollout.rb | 2 +- .../source/databrics_model/client.rb | 8 +++++-- .../source/databricks_model/client_spec.rb | 21 +++++++++++++++++++ 4 files changed, 29 insertions(+), 4 deletions(-) diff --git a/integrations/Gemfile.lock b/integrations/Gemfile.lock index cd85a166..37a81655 100644 --- a/integrations/Gemfile.lock +++ b/integrations/Gemfile.lock @@ -7,7 +7,7 @@ GIT PATH remote: . specs: - multiwoven-integrations (0.16.1) + multiwoven-integrations (0.16.2) MailchimpMarketing activesupport async-websocket diff --git a/integrations/lib/multiwoven/integrations/rollout.rb b/integrations/lib/multiwoven/integrations/rollout.rb index 6f6409fb..d833d1d1 100644 --- a/integrations/lib/multiwoven/integrations/rollout.rb +++ b/integrations/lib/multiwoven/integrations/rollout.rb @@ -2,7 +2,7 @@ module Multiwoven module Integrations - VERSION = "0.16.1" + VERSION = "0.16.2" ENABLED_SOURCES = %w[ Snowflake diff --git a/integrations/lib/multiwoven/integrations/source/databrics_model/client.rb b/integrations/lib/multiwoven/integrations/source/databrics_model/client.rb index 9132d001..33420e8b 100644 --- a/integrations/lib/multiwoven/integrations/source/databrics_model/client.rb +++ b/integrations/lib/multiwoven/integrations/source/databrics_model/client.rb @@ -62,8 +62,12 @@ def run_model(connection_config, payload) def process_response(response) if success?(response) - data = JSON.parse(response.body) - [RecordMessage.new(data: data, emitted_at: Time.now.to_i).to_multiwoven_message] + begin + data = JSON.parse(response.body) + [RecordMessage.new(data: data, emitted_at: Time.now.to_i).to_multiwoven_message] + rescue JSON::ParserError + create_log_message("DATABRICKS MODEL:RUN_MODEL", "error", "parsing failed: please send a valid payload") + end else create_log_message("DATABRICKS MODEL:RUN_MODEL", "error", "request failed: #{response.body}") end diff --git a/integrations/spec/multiwoven/integrations/source/databricks_model/client_spec.rb b/integrations/spec/multiwoven/integrations/source/databricks_model/client_spec.rb index de785f16..f17b6e9b 100644 --- a/integrations/spec/multiwoven/integrations/source/databricks_model/client_spec.rb +++ b/integrations/spec/multiwoven/integrations/source/databricks_model/client_spec.rb @@ -177,6 +177,27 @@ expect(records.first.record.data).to eq(JSON.parse(response_body)) end end + + context "when the payload is invalid in read" do + let(:response_body) { "{\"key\": invalid_json}" }.to_json + before do + response = Net::HTTPSuccess.new("1.1", "200", "Unauthorized") + response.content_type = "application/json" + allow(response).to receive(:body).and_return(response_body) + allow(Multiwoven::Integrations::Core::HttpClient).to receive(:request) + .with("https://test-host.databricks.com/serving-endpoints/test/invocations", + "POST", + payload: JSON.parse(payload.to_json), + headers: headers) + .and_return(response) + end + it "handles exceptions during reading" do + records = client.read(sync_config) + expect(records.log).to be_a(Multiwoven::Integrations::Protocol::LogMessage) + expect(records.log.message).to eq("parsing failed: please send a valid payload") + end + end + context "when the read is failed" do it "handles exceptions during reading" do error_instance = StandardError.new("test error") From 5c82069e6afbfd39b768d438bc3e1e7001295e87 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 27 Dec 2024 16:02:23 +0530 Subject: [PATCH 27/28] feat(CE): Sync alerts - added models (#741) (#555) Co-authored-by: Basil V Bose --- server/app/models/alert.rb | 10 ++++++++ server/app/models/alert_channel.rb | 8 ++++++ server/app/models/workspace.rb | 1 + .../migrate/20241219163555_create_alerts.rb | 13 ++++++++++ .../20241219163606_create_alert_channels.rb | 10 ++++++++ server/db/schema.rb | 25 ++++++++++++++++++- server/spec/factories/alert_channels.rb | 17 +++++++++++++ server/spec/factories/alerts.rb | 12 +++++++++ server/spec/models/alert_channel_spec.rb | 19 ++++++++++++++ server/spec/models/alert_spec.rb | 15 +++++++++++ 10 files changed, 129 insertions(+), 1 deletion(-) create mode 100644 server/app/models/alert.rb create mode 100644 server/app/models/alert_channel.rb create mode 100644 server/db/migrate/20241219163555_create_alerts.rb create mode 100644 server/db/migrate/20241219163606_create_alert_channels.rb create mode 100644 server/spec/factories/alert_channels.rb create mode 100644 server/spec/factories/alerts.rb create mode 100644 server/spec/models/alert_channel_spec.rb create mode 100644 server/spec/models/alert_spec.rb diff --git a/server/app/models/alert.rb b/server/app/models/alert.rb new file mode 100644 index 00000000..f480239f --- /dev/null +++ b/server/app/models/alert.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +class Alert < ApplicationRecord + belongs_to :workspace + has_many :alert_channels, dependent: :destroy + accepts_nested_attributes_for :alert_channels + + validates :workspace_id, presence: true + validates :row_failure_threshold_percent, numericality: { only_integer: true, allow_nil: true } +end diff --git a/server/app/models/alert_channel.rb b/server/app/models/alert_channel.rb new file mode 100644 index 00000000..15742ab3 --- /dev/null +++ b/server/app/models/alert_channel.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +class AlertChannel < ApplicationRecord + belongs_to :alert + + validates :platform, presence: true, inclusion: { in: %w[email slack] } + enum :platform, %i[email slack] +end diff --git a/server/app/models/workspace.rb b/server/app/models/workspace.rb index eb347afe..84c7c665 100644 --- a/server/app/models/workspace.rb +++ b/server/app/models/workspace.rb @@ -24,6 +24,7 @@ class Workspace < ApplicationRecord has_many :data_app_sessions, dependent: :nullify has_many :audit_logs, dependent: :nullify has_many :custom_visual_component_files, dependent: :nullify + has_many :alerts, dependent: :nullify belongs_to :organization STATUS_ACTIVE = "active" diff --git a/server/db/migrate/20241219163555_create_alerts.rb b/server/db/migrate/20241219163555_create_alerts.rb new file mode 100644 index 00000000..0221687f --- /dev/null +++ b/server/db/migrate/20241219163555_create_alerts.rb @@ -0,0 +1,13 @@ +class CreateAlerts < ActiveRecord::Migration[7.1] + def change + create_table :alerts do |t| + t.string :name + t.references :workspace, null: false, foreign_key: true + t.boolean :alert_sync_success, default: false + t.boolean :alert_sync_failure, default: false + t.boolean :alert_row_failure, default: false + t.integer :row_failure_threshold_percent + t.timestamps + end + end +end diff --git a/server/db/migrate/20241219163606_create_alert_channels.rb b/server/db/migrate/20241219163606_create_alert_channels.rb new file mode 100644 index 00000000..abd8a0d5 --- /dev/null +++ b/server/db/migrate/20241219163606_create_alert_channels.rb @@ -0,0 +1,10 @@ +class CreateAlertChannels < ActiveRecord::Migration[7.1] + def change + create_table :alert_channels do |t| + t.references :alert, null: false, foreign_key: true + t.integer :platform, null: false + t.jsonb :configuration + t.timestamps + end + end +end diff --git a/server/db/schema.rb b/server/db/schema.rb index 3c7e33ba..dc4444fa 100644 --- a/server/db/schema.rb +++ b/server/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.1].define(version: 2024_12_06_211928) do +ActiveRecord::Schema[7.1].define(version: 2024_12_19_163606) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -42,6 +42,27 @@ t.index ["blob_id", "variation_digest"], name: "index_active_storage_variant_records_uniqueness", unique: true end + create_table "alert_channels", force: :cascade do |t| + t.bigint "alert_id", null: false + t.integer "platform", null: false + t.jsonb "configuration" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index ["alert_id"], name: "index_alert_channels_on_alert_id" + end + + create_table "alerts", force: :cascade do |t| + t.string "name" + t.bigint "workspace_id", null: false + t.boolean "alert_sync_success", default: false + t.boolean "alert_sync_failure", default: false + t.boolean "alert_row_failure", default: false + t.integer "row_failure_threshold_percent" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index ["workspace_id"], name: "index_alerts_on_workspace_id" + end + create_table "audit_logs", force: :cascade do |t| t.integer "user_id" t.string "action", null: false @@ -303,6 +324,8 @@ add_foreign_key "active_storage_attachments", "active_storage_blobs", column: "blob_id" add_foreign_key "active_storage_variant_records", "active_storage_blobs", column: "blob_id" + add_foreign_key "alert_channels", "alerts" + add_foreign_key "alerts", "workspaces" add_foreign_key "workspace_users", "roles" add_foreign_key "workspace_users", "users" add_foreign_key "workspace_users", "workspaces", on_delete: :nullify diff --git a/server/spec/factories/alert_channels.rb b/server/spec/factories/alert_channels.rb new file mode 100644 index 00000000..8bff7acd --- /dev/null +++ b/server/spec/factories/alert_channels.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +FactoryBot.define do + factory :alert_channel do + association :alert + + trait :email do + platform { "email" } + configuration { { extra_email_recipients: ["user1@example.com", "user2@example.com"] } } + end + + trait :slack do + platform { "slack" } + configuration { { slack_email_alias: ["slackemailtemplate@slack.com"] } } + end + end +end diff --git a/server/spec/factories/alerts.rb b/server/spec/factories/alerts.rb new file mode 100644 index 00000000..2d5dd3d2 --- /dev/null +++ b/server/spec/factories/alerts.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +FactoryBot.define do + factory :alert do + association :workspace + name { "Test alert" } + alert_sync_success { false } + alert_sync_failure { false } + alert_row_failure { false } + row_failure_threshold_percent { 50 } + end +end diff --git a/server/spec/models/alert_channel_spec.rb b/server/spec/models/alert_channel_spec.rb new file mode 100644 index 00000000..dd301455 --- /dev/null +++ b/server/spec/models/alert_channel_spec.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +require "rails_helper" + +RSpec.describe AlertChannel, type: :model do + describe "associations" do + it { should belong_to(:alert) } + end + + describe "validations" do + it { should validate_presence_of(:platform) } + end + + describe "platform" do + it "defines platform enum with specified values" do + expect(AlertChannel.platforms).to eq({ "email" => 0, "slack" => 1 }) + end + end +end diff --git a/server/spec/models/alert_spec.rb b/server/spec/models/alert_spec.rb new file mode 100644 index 00000000..d866f4f6 --- /dev/null +++ b/server/spec/models/alert_spec.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +require "rails_helper" + +RSpec.describe Alert, type: :model do + describe "associations" do + it { should belong_to(:workspace) } + it { should have_many(:alert_channels).dependent(:destroy) } + end + + describe "validations" do + it { should validate_presence_of(:workspace_id) } + it { should validate_numericality_of(:row_failure_threshold_percent).only_integer.allow_nil } + end +end From 90f8e1b2e750c3816aec51398de072940d9665f8 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 27 Dec 2024 16:04:10 +0530 Subject: [PATCH 28/28] chore(CE): Update server gem 0.16.2 (#553) Co-authored-by: TivonB-AI2 <124182151+TivonB-AI2@users.noreply.github.com> --- server/Gemfile | 2 +- server/Gemfile.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/server/Gemfile b/server/Gemfile index 4b39537e..afa26721 100644 --- a/server/Gemfile +++ b/server/Gemfile @@ -13,7 +13,7 @@ gem "interactor", "~> 3.0" gem "ruby-odbc", git: "https://github.com/Multiwoven/ruby-odbc.git" -gem "multiwoven-integrations", "~> 0.16.1" +gem "multiwoven-integrations", "~> 0.16.2" gem "temporal-ruby", github: "coinbase/temporal-ruby" diff --git a/server/Gemfile.lock b/server/Gemfile.lock index 7cd7f621..e88970ec 100644 --- a/server/Gemfile.lock +++ b/server/Gemfile.lock @@ -1677,7 +1677,7 @@ GEM crass (1.0.6) css_parser (1.17.1) addressable - csv (3.3.1) + csv (3.3.2) data_migrate (9.4.0) activerecord (>= 6.1) railties (>= 6.1) @@ -1941,7 +1941,7 @@ GEM msgpack (1.7.2) multi_json (1.15.0) multipart-post (2.4.1) - multiwoven-integrations (0.16.1) + multiwoven-integrations (0.16.2) MailchimpMarketing activesupport async-websocket @@ -2172,7 +2172,7 @@ GEM faraday-multipart gli hashie - sorbet-runtime (0.5.11707) + sorbet-runtime (0.5.11708) stringio (3.1.0) stripe (13.3.0) strong_migrations (1.8.0) @@ -2249,7 +2249,7 @@ DEPENDENCIES jwt kaminari liquid - multiwoven-integrations (~> 0.16.1) + multiwoven-integrations (~> 0.16.2) mysql2 newrelic_rpm parallel