Allow UpdateCachedAppealsAttributesJob to complete and send warnings

app/jobs/update_cached_appeals_attributes_job.rb

@@ -26,6 +26,8 @@ def perform
     datadog_report_runtime(metric_group_name: METRIC_GROUP_NAME)
   rescue StandardError => error
     log_error(@start_time, error)
+  else
+    log_warning unless warning_msgs.empty?
   end
 
   # rubocop:disable Metrics/MethodLength
@@ -102,16 +104,14 @@ def cache_legacy_appeal_postgres_data(legacy_appeals)
     legacy_appeals.in_groups_of(POSTGRES_BATCH_SIZE, false) do |batch_legacy_appeals|
       values_to_cache = batch_legacy_appeals.map do |appeal|
         regional_office = RegionalOffice::CITIES[appeal.closest_regional_office]
-        # bypass PowerOfAttorney model completely and always prefer BGS cache
-        bgs_poa = fetch_bgs_power_of_attorney_by_file_number(appeal.veteran_file_number)
         {
           vacols_id: appeal.vacols_id,
           appeal_id: appeal.id,
           appeal_type: LegacyAppeal.name,
           closest_regional_office_city: regional_office ? regional_office[:city] : COPY::UNKNOWN_REGIONAL_OFFICE,
           closest_regional_office_key: regional_office ? appeal.closest_regional_office : COPY::UNKNOWN_REGIONAL_OFFICE,
           docket_type: appeal.docket_name, # "legacy"
-          power_of_attorney_name: (bgs_poa&.representative_name || appeal.representative_name),
+          power_of_attorney_name: poa_representative_name_for(appeal),
           suggested_hearing_location: appeal.suggested_hearing_location&.formatted_location
         }
       end
@@ -130,6 +130,16 @@ def cache_legacy_appeal_postgres_data(legacy_appeals)
   end
   # rubocop:enable Metrics/MethodLength
 
+  # bypass PowerOfAttorney model completely and always prefer BGS cache
+  def poa_representative_name_for(appeal)
+    bgs_poa = fetch_bgs_power_of_attorney_by_file_number(appeal.veteran_file_number)
+    # both representative_name calls can result in BGS connection error
+    bgs_poa&.representative_name || appeal.representative_name
+  rescue Errno::ECONNRESET => error
+    warning_msgs << "#{appeal.class.name} #{appeal.id}: #{error}" if warning_msgs.count < 100
+    nil
+  end
+
   def fetch_bgs_power_of_attorney_by_file_number(file_number)
     return if file_number.blank?
 
@@ -217,6 +227,15 @@ def increment_appeal_count(count, appeal_type)
     end
   end
 
+  def warning_msgs
+    @warning_msgs ||= []
+  end
+
+  def log_warning
+    slack_msg = "[WARN] UpdateCachedAppealsAttributesJob first 100 warnings: \n#{warning_msgs.join("\n")}"
+    slack_service.send_notification(slack_msg)
+  end
+
   def log_error(start_time, err)
     duration = time_ago_in_words(start_time)
     msg = "UpdateCachedAppealsAttributesJob failed after running for #{duration}. Fatal error: #{err.message}"

spec/jobs/update_cached_appeals_attributes_job_spec.rb

@@ -149,5 +149,27 @@
       expect(CachedAppeal.find_by(vacols_id: legacy_appeal2.vacols_id).former_travel).to eq(false)
       expect(CachedAppeal.find_by(vacols_id: legacy_appeal3.vacols_id).former_travel).to eq(true)
     end
+
+    context "when BGS fails" do
+      before do
+        bgs = Fakes::BGSService.new
+        allow(Fakes::BGSService).to receive(:new).and_return(bgs)
+        allow(bgs).to receive(:fetch_person_by_ssn)
+          .and_raise(Errno::ECONNRESET, "mocked error for testing")
+      end
+
+      it "completes and sends warning to Slack" do
+        slack_msg = ""
+        allow_any_instance_of(SlackService).to receive(:send_notification) { |_, first_arg| slack_msg = first_arg }
+
+        job = described_class.new
+        job.perform_now
+        expect(job.warning_msgs.count).to eq 3
+
+        expect(slack_msg.lines.count).to eq 4
+        expected_msg = "\\[WARN\\] UpdateCachedAppealsAttributesJob .*"
+        expect(slack_msg).to match(/#{expected_msg}/)
+      end
+    end
   end
 end