Enhancements to prod to staging loader (#1026)

Co-authored-by: nfstern02 <[email protected]>

app/models/dashboard_exporter_importer.rb

@@ -45,6 +45,7 @@ def download_all_table_data
     0
   end
 
+  # rubocop:disable Lint/RescueException
   def upload_all_table_data
     CSV_TYPES_ALL_TABLES_CLASSES.each do |table_class|
       table_name = table_class.to_s
@@ -53,23 +54,46 @@ def upload_all_table_data
       next if table_name.eql?('InstitutionSchoolRating')
       # This table has CORS issues loading to the staging server
       next if table_name.eql?('CipCode') && @login_url.eql?(STAGE_URL)
-
-      upload_csv_file_for(table_name)
+      # Weam  has it's own routine for uploading
+      next if table_name.eql?('Weam')
+
+      begin
+        upload_csv_file_for(table_name)
+      rescue Exception => e
+        log_and_puts("       Error: #{e.message}...")
+        retry_upload_for(table_name)
+      end
     end
 
     0
   end
 
+  def retry_upload_for(table_name)
+    sleep(10)
+    begin
+      log_out_and_back_in(table_name)
+      upload_csv_file_for(table_name)
+    rescue Exception => e
+      log_and_puts("       Failed again, #{e.message}, skipping...")
+    end
+  end
+  # rubocop:enable Lint/RescueException
+
   def finalize
     @headless.destroy
-    @bsess.close
+    log_and_puts('*** All done! You can close this terminal window. ***')
   end
 
   private
 
   def set_logger
     logger_time = Time.now.getlocal.strftime('%Y%m%d_%H%M%S')
-    @eilogger = Logger.new(Rails.root.join('log', "export_import_#{logger_time}.log"))
+    log_file_name = Rails.root.join('log', "export_import_#{logger_time}.log")
+    @eilogger = Logger.new(log_file_name)
+    log_and_puts("***** Starting export_import_#{logger_time} *****")
+
+    # open a terminal and tail the log in it
+    `gnome-terminal --title="Tail log #{logger_time}" -- bash -c "tail -f #{log_file_name}; exec bash -i"`
   end
 
   # load_env is target enviroment for upload/download
@@ -94,17 +118,37 @@ def set_download_dir
     Rails.root.join('tmp')
   end
 
+  # rubocop:disable Lint/RescueException
+  # rubocop:disable Metrics/MethodLength
+  # rubocop:disable Rails/Exit
   def login_to_dashboard
     log_and_puts('*** Logging in to the Dashboard ***')
 
-    client = Selenium::WebDriver::Remote::Http::Default.new
-    client.read_timeout = TIMEOUT # seconds
-
-    @bsess = Watir::Browser.start(@login_url, http_client: client)
+    begin
+      client = Selenium::WebDriver::Remote::Http::Default.new
+      client.read_timeout = TIMEOUT # seconds
+      @bsess = Watir::Browser.start(@login_url, http_client: client)
+    rescue Exception => e
+      log_and_puts("       Error trying to initiate browser session #{e.message}...")
+      log_and_puts('       Trying again in 10 seconds...')
+      sleep(10)
+      @bsess = nil
+      begin
+        client = Selenium::WebDriver::Remote::Http::Default.new
+        client.read_timeout = TIMEOUT # seconds
+        @bsess = Watir::Browser.start(@login_url, http_client: client)
+      rescue Exception => e
+        log_and_puts("       Failed again #{e.message}...")
+        exit 1
+      end
+    end
     @bsess.text_field(id: 'user_email').set(@user)
     @bsess.text_field(id: 'user_password').set(@pass)
     @bsess.form(id: 'new_user').submit
   end
+  # rubocop:enable Rails/Exit
+  # rubocop:enable Metrics/MethodLength
+  # rubocop:enable Lint/RescueException
 
   def remove_existing_csv_file_for(table_name)
     log_and_puts("  Removing existing CSV file for #{table_name}")
@@ -131,76 +175,50 @@ def download_csv_file_for(table_name)
 
   def upload_csv_file_for(table_name)
     log_and_puts("     Uploading CSV file for #{table_name}")
-
-    if table_name.eql?('Weam')
-      %w[Weam1 Weam2 Weam3].each { |weam_file_name| remove_existing_csv_file_for(weam_file_name) }
-
-      split_weams_file
-
-      %w[Weam1 Weam2 Weam3].each do |weam_file_name|
-        multiple_file_upload = (weam_file_name.include?('Weam1') ? false : true)
-        upload_with_parameters('Weam', weam_file_name, multiple_file_upload)
-      end
-    else
-      upload_with_parameters(table_name, table_name, false)
-    end
-
+    upload_with_parameters(table_name, 0)
     log_out_and_back_in(table_name)
   end
 
-  def upload_with_parameters(table_name, file_name, multiple_file_upload = false)
+  def upload_with_parameters(table_name, retry_count = 0)
+    log_and_puts("         Uploading #{table_name}")
     button = @bsess.link(role: 'button', href: "#{@import_prefix}#{table_name}", visible_text: 'Upload')
     button.click
 
     @bsess.text_field(id: 'upload_skip_lines').set(0)
-    @bsess.file_field(id: 'upload_upload_file').set("#{@download_dir}/#{file_name}.csv")
+    @bsess.file_field(id: 'upload_upload_file').set("#{@download_dir}/#{table_name}.csv")
 
     @bsess
       .text_field(id: 'upload_comment')
       .set("Uploaded on #{Time.now.getlocal} from Production export")
 
-    @bsess.checkbox(id: 'upload_multiple_file_upload').check if multiple_file_upload
     @bsess.form(id: 'new_upload').submit
 
     if @bsess.link(text: 'View Dashboard').present?
+      log_and_puts("         Successfully uploaded #{table_name}")
       @bsess.link(text: 'View Dashboard').click
-    else
+    else # retry once
       log_and_puts('    Could not find the dashboard link - most likely it failed')
+      sleep(30)
       @bsess.goto(@dashboard_url)
-    end
-  end
+      return if retry_count.positive?
 
-  # Weam has approx 75k lines, split into 3 files so that we don't run out of memory in Staging
-  def split_weams_file
-    file1 = File.open("#{@download_dir}/Weam1.csv", 'w')
-    file2 = File.open("#{@download_dir}/Weam2.csv", 'w')
-    file3 = File.open("#{@download_dir}/Weam3.csv", 'w')
-
-    File.open("#{@download_dir}/Weam.csv").each_with_index do |row, index|
-      case index
-      when 0
-        file1.write(row)
-        file2.write(row)
-        file3.write(row)
-      when 1..24_999
-        file1.write(row)
-      when 25_000..49_999
-        file2.write(row)
-      else
-        file3.write(row)
-      end
+      log_out_and_back_in(table_name)
+      upload_with_parameters(table_name, 1)
     end
-
-    file1.close
-    file2.close
-    file3.close
   end
 
   def log_out_and_back_in(table_name)
     log_and_puts('*** Logging out')
-    @bsess.link(text: 'Log Out').click
+    @bsess.link(text: 'Log Out').click if @bsess.link(text: 'Log Out').present?
+    @bsess = nil # close the browser session to free up memory
     log_and_puts ''
-    login_to_dashboard unless table_name.eql?('Section1015') # last table in the array
+    sleep(5)
+
+    if table_name.eql?('Section1015') # last table in the array
+      log_and_puts('*** Finished uploading tables ***')
+    else
+      login_to_dashboard
+    end
   end
 
   def log_and_puts(msg)