chore(CE): add request response log for Google Sheets

Multiwoven · Aug 12, 2024 · d5e7501 · d5e7501
1 parent 8222794
commit d5e7501
Show file tree

Hide file tree

Showing 4 changed files with 22 additions and 16 deletions.
diff --git a/integrations/Gemfile.lock b/integrations/Gemfile.lock
@@ -7,7 +7,7 @@ GIT
 PATH
   remote: .
   specs:
-    multiwoven-integrations (0.7.1)
+    multiwoven-integrations (0.7.3)
       activesupport
       async-websocket
       aws-sdk-athena

diff --git a/integrations/lib/multiwoven/integrations/destination/google_sheets/client.rb b/integrations/lib/multiwoven/integrations/destination/google_sheets/client.rb
@@ -153,13 +153,15 @@ def extract_spreadsheet_id(link)
 
           # Batch has a limit of sending 2MB data. So creating a chunk of records to meet that limit
           def process_record_chunks(records, sync_config)
+            log_message_array = []
             write_success = 0
             write_failure = 0
 
             records.each_slice(MAX_CHUNK_SIZE) do |chunk|
               values = prepare_chunk_values(chunk, sync_config.stream)
-              update_sheet_values(values, sync_config.stream.name)
+              request, response = *update_sheet_values(values, sync_config.stream.name)
               write_success += values.size
+              log_message_array << log_request_response("info", request, response)
             rescue StandardError => e
               handle_exception(e, {
                                  context: "GOOGLE_SHEETS:RECORD:WRITE:EXCEPTION",
@@ -168,9 +170,9 @@ def process_record_chunks(records, sync_config)
                                  sync_run_id: sync_config.sync_run_id
                                })
               write_failure += chunk.size
+              log_message_array << log_request_response("error", request, e.message)
             end
-
-            tracking_message(write_success, write_failure)
+            tracking_message(write_success, write_failure, log_message_array)
           end
 
           # We need to format the data to adhere to google sheets API format. This converts the sync mapped data to 2D array format expected by google sheets API
@@ -199,19 +201,14 @@ def update_sheet_values(values, stream_name)
             )
 
             # TODO: Remove & this is added for the test to pass we need
-            @client&.batch_update_values(@spreadsheet_id, batch_update_request)
+            response = @client&.batch_update_values(@spreadsheet_id, batch_update_request)
+            [batch_update_request, response]
           end
 
           def load_catalog
             read_json(CATALOG_SPEC_PATH)
           end
 
-          def tracking_message(success, failure)
-            Multiwoven::Integrations::Protocol::TrackingMessage.new(
-              success: success, failed: failure
-            ).to_multiwoven_message
-          end
-
           def delete_extra_sheets(sheet_ids)
             # Leave one sheet intact as a spreadsheet must have at least one sheet.
             # Delete all other sheets.

diff --git a/integrations/lib/multiwoven/integrations/rollout.rb b/integrations/lib/multiwoven/integrations/rollout.rb
@@ -2,7 +2,7 @@
 
 module Multiwoven
   module Integrations
-    VERSION = "0.7.1"
+    VERSION = "0.7.3"
 
     ENABLED_SOURCES = %w[
       Snowflake

diff --git a/integrations/spec/multiwoven/integrations/destination/google_sheets/client_spec.rb b/integrations/spec/multiwoven/integrations/destination/google_sheets/client_spec.rb
@@ -245,22 +245,31 @@
 
         expect(response.tracking.success).to eq(records.size)
         expect(response.tracking.failed).to eq(0)
+        log_message = response.tracking.logs.first
+        expect(log_message).to be_a(Multiwoven::Integrations::Protocol::LogMessage)
+        expect(log_message.level).to eql("info")
+
+        expect(log_message.message).to include("request")
+        expect(log_message.message).to include("response")
       end
     end
 
     context "when the write operation fails" do
       before do
-        batch_update_request = instance_double(Google::Apis::SheetsV4::BatchUpdateValuesRequest)
-        allow(google_sheets_service).to receive(:batch_update_values)
-          .with(@spreadsheet_id, batch_update_request)
-          .and_raise(Google::Apis::ClientError.new("Invalid request"))
+        allow(@client).to receive(:update_sheet_values).and_raise(StandardError.new("Failed to update_sheet_values"))
       end
 
       it "increments the failure count" do
         response = client.write(sync_config, records)
 
         expect(response.tracking.failed).to eq(records.size)
         expect(response.tracking.success).to eq(0)
+        log_message = response.tracking.logs.first
+        expect(log_message).to be_a(Multiwoven::Integrations::Protocol::LogMessage)
+        expect(log_message.level).to eql("error")
+
+        expect(log_message.message).to include("request")
+        expect(log_message.message).to include("response")
       end
     end
   end