[CST] - Update failed document upload logic

app/sidekiq/evss/document_upload.rb

@@ -80,6 +80,9 @@ def self.update_evidence_submission(msg)
     current_personalisation = JSON.parse(evidence_submission.template_metadata)['personalisation']
     evidence_submission.update(
       upload_status: BenefitsDocuments::Constants::UPLOAD_STATUS[:FAILED],
+      failed_date: DateTime.now.utc,
+      acknowledgement_date: (DateTime.current + 30.days).utc,
+      error_message: 'EVSS::DocumentUpload document upload failure',
       template_metadata: {
         personalisation: update_personalisation(current_personalisation, msg['failed_at'])
       }.to_json

app/sidekiq/lighthouse/evidence_submissions/document_upload.rb

@@ -58,6 +58,9 @@ def self.update_evidence_submission_for_failure(msg)
         current_personalisation = JSON.parse(evidence_submission.template_metadata)['personalisation']
         evidence_submission.update(
           upload_status: BenefitsDocuments::Constants::UPLOAD_STATUS[:FAILED],
+          failed_date: DateTime.now.utc,
+          acknowledgement_date: (DateTime.current + 30.days).utc,
+          error_message: 'Lighthouse::EvidenceSubmissions::DocumentUpload document upload failure',
           template_metadata: {
             personalisation: update_personalisation(current_personalisation, msg['failed_at'])
           }.to_json

app/sidekiq/lighthouse/evidence_submissions/failure_notification_email_job.rb

@@ -39,8 +39,7 @@ def notify_client
 
       def send_failed_evidence_submissions
         failed_uploads.each do |upload|
-          personalisation = create_personalisation_from_upload(upload)
-
+          personalisation = BenefitsDocuments::Utilities::Helpers.create_personalisation_from_upload(upload)
           # NOTE: The file_name in the personalisation that is passed in is obscured
           response = notify_client.send_email(
             recipient_identifier: { id_value: upload.user_account.icn, id_type: 'ICN' },
@@ -55,16 +54,6 @@ def send_failed_evidence_submissions
         nil
       end
 
-      # This will be used to send an upload failure email
-      # We created a new personalisation with the obfuscated_file_name so the filename is hidden in the email
-      def create_personalisation_from_upload(upload)
-        personalisation = JSON.parse(upload.template_metadata)['personalisation']
-        personalisation['file_name'] = personalisation['obfuscated_filename']
-        personalisation.delete('obfuscated_filename')
-
-        personalisation
-      end
-
       def record_email_send_success(upload, response)
         # Update evidence_submissions table record with the va_notify_id and va_notify_date
         upload.update(va_notify_id: response.id, va_notify_date: DateTime.now)

lib/lighthouse/benefits_documents/upload_status_updater.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require 'lighthouse/benefits_documents/constants'
+require 'lighthouse/benefits_documents/utilities/helpers'
 
 module BenefitsDocuments
   class UploadStatusUpdater
@@ -95,10 +96,20 @@ def process_failure
         upload_status: BenefitsDocuments::Constants::UPLOAD_STATUS[:FAILED],
         failed_date: DateTime.now.utc,
         acknowledgement_date: (DateTime.current + 30.days).utc,
-        error_message: @lighthouse_document_status_response['error']
+        error_message: @lighthouse_document_status_response['error'],
+        template_metadata: {
+          personalisation: update_personalisation
+        }.to_json
       )
     end
 
+    # Update personalisation here since an evidence submission record was previously created
+    def update_personalisation
+      personalisation = JSON.parse(@pending_evidence_submission.template_metadata)['personalisation'].clone
+      personalisation['date_failed'] = BenefitsDocuments::Utilities::Helpers.format_date_for_mailers(DateTime.current)
+      personalisation
+    end
+
     def process_upload
       @pending_evidence_submission.update!(
         upload_status: BenefitsDocuments::Constants::UPLOAD_STATUS[:SUCCESS],

lib/lighthouse/benefits_documents/utilities/helpers.rb

@@ -28,6 +28,17 @@ def self.format_date_for_mailers(date)
         # We display dates in mailers in the format "May 1, 2024 3:01 p.m. EDT"
         timestamp.strftime('%B %-d, %Y %-l:%M %P %Z').sub(/([ap])m/, '\1.m.')
       end
+
+      # Used to take the current personalisation from an evidence_submission records template_metadata
+      # and return a new personalisation with a file_name field that is obscured and doesnt have the field
+      # obfuscated_file_name. This will be used in the failed notification email template.
+      def self.create_personalisation_from_upload(upload)
+        personalisation = JSON.parse(upload.template_metadata)['personalisation']
+        personalisation['file_name'] = personalisation['obfuscated_file_name']
+        personalisation.delete('obfuscated_file_name')
+
+        personalisation
+      end
     end
   end
 end

spec/factories/lighthouse/benefits_documents/evidence_submission.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require 'lighthouse/benefits_documents/constants'
+require 'lighthouse/benefits_documents/utilities/helpers'
 
 FactoryBot.define do
   factory :bd_evidence_submission, class: 'EvidenceSubmission' do
@@ -37,25 +38,75 @@
         'document_type' => 'Birth Certificate',
         'file_name' => 'testfile.txt',
         'obfuscated_file_name' => 'tesXXile.txt',
-        'date_submitted' => DateTime.now.utc.to_s,
+        'date_submitted' => BenefitsDocuments::Utilities::Helpers.format_date_for_mailers(DateTime.now),
         'date_failed' => nil
       } }.to_json
     end
   end
 
-  factory :bd_evidence_submission_failed, class: 'EvidenceSubmission' do
+  # Document Upload Failures for Type 1 errors occur in app/sidekiq/lighthouse/evidence_submissions/document_upload.rb
+  # when a error happens before the document upload is sent to lighthouse
+  factory :bd_lh_evidence_submission_failed_type1_error, class: 'EvidenceSubmission' do
     association :user_account, factory: :user_account
     created_at { DateTime.now.utc }
+    job_class { 'Lighthouse::BenefitsDocuments::Service' }
     upload_status { BenefitsDocuments::Constants::UPLOAD_STATUS[:FAILED] }
-    job_class { 'Lighthouse::EvidenceSubmissions::DocumentUpload' }
+    failed_date { DateTime.now.utc }
+    acknowledgement_date { DateTime.now.utc + 30.days }
+    error_message { 'Lighthouse::EvidenceSubmissions::DocumentUpload document upload failure' }
+    template_metadata do
+      { 'personalisation' => {
+        'first_name' => 'test',
+        'document_type' => 'Birth Certificate',
+        'file_name' => 'test.txt',
+        'obfuscated_file_name' => 'tesXXile.txt',
+        'date_submitted' => BenefitsDocuments::Utilities::Helpers.format_date_for_mailers(DateTime.now),
+        'date_failed' => BenefitsDocuments::Utilities::Helpers.format_date_for_mailers(DateTime.now)
+      } }.to_json
+    end
+  end
+
+  # Document Upload Failures for Type 2 errors occur in lib/lighthouse/benefits_documents/upload_status_updater.rb
+  # when the polling job to grab the upload status from lighthouse occurs and there is a failure processing
+  # the upload on lighthouses side
+  factory :bd_lh_evidence_submission_failed_type2_error, class: 'EvidenceSubmission' do
+    association :user_account, factory: :user_account
+    created_at { DateTime.now.utc }
+    job_class { 'Lighthouse::BenefitsDocuments::Service' }
+    upload_status { BenefitsDocuments::Constants::UPLOAD_STATUS[:FAILED] }
+    failed_date { DateTime.now.utc }
+    acknowledgement_date { DateTime.now.utc + 30.days }
+    error_message { 'test - there was an error returned from lh api' }
+    template_metadata do
+      { 'personalisation' => {
+        'first_name' => 'test',
+        'document_type' => 'Birth Certificate',
+        'file_name' => 'test.txt',
+        'obfuscated_file_name' => 'tesXXile.txt',
+        'date_submitted' => BenefitsDocuments::Utilities::Helpers.format_date_for_mailers(DateTime.now),
+        'date_failed' => BenefitsDocuments::Utilities::Helpers.format_date_for_mailers(DateTime.now)
+      } }.to_json
+    end
+  end
+
+  # Document Upload Failures for Type 1 errors occur in app/sidekiq/evss/document_upload.rb
+  # when a error happens before or when we send the upload to evss
+  factory :bd_evss_evidence_submission_failed_type1_error, class: 'EvidenceSubmission' do
+    association :user_account, factory: :user_account
+    created_at { DateTime.now.utc }
+    job_class { 'EVSS::DocumentUpload' }
+    upload_status { BenefitsDocuments::Constants::UPLOAD_STATUS[:FAILED] }
+    failed_date { DateTime.now.utc }
+    acknowledgement_date { DateTime.now.utc + 30.days }
+    error_message { 'EVSS::DocumentUpload document upload failure' }
     template_metadata do
       { 'personalisation' => {
         'first_name' => 'test',
         'document_type' => 'Birth Certificate',
         'file_name' => 'test.txt',
         'obfuscated_file_name' => 'tesXXile.txt',
-        'date_submitted' => DateTime.now.utc.to_s,
-        'date_failed' => DateTime.now.utc.to_s
+        'date_submitted' => BenefitsDocuments::Utilities::Helpers.format_date_for_mailers(DateTime.now),
+        'date_failed' => BenefitsDocuments::Utilities::Helpers.format_date_for_mailers(DateTime.now)
       } }.to_json
     end
   end

spec/lib/lighthouse/benefits_documents/upload_status_updater_spec.rb

@@ -3,12 +3,14 @@
 require 'rails_helper'
 require 'lighthouse/benefits_documents/upload_status_updater'
 require 'lighthouse/benefits_documents/constants'
+require 'lighthouse/benefits_documents/utilities/helpers'
 
 RSpec.describe BenefitsDocuments::UploadStatusUpdater do
   let(:lighthouse_document_upload) { create(:bd_evidence_submission_pending, job_class: 'BenefitsDocuments::Service') }
   let(:lighthouse_document_upload_timeout) { create(:bd_evidence_submission_timeout) }
   let(:past_date_time) { DateTime.new(1985, 10, 26) }
   let(:current_date_time) { DateTime.now.utc }
+  let(:issue_instant) { Time.now.to_i }
 
   describe '#update_status' do
     shared_examples 'status updater' do |status, error_message = nil|
@@ -19,6 +21,19 @@
         }.compact
       end
       let(:status_updater) { described_class.new(document_status_response, lighthouse_document_upload) }
+      let(:date) do
+        BenefitsDocuments::Utilities::Helpers.format_date_for_mailers(issue_instant)
+      end
+      let(:updated_template_metadata) do
+        { 'personalisation' => {
+          'first_name' => 'test',
+          'document_type' => 'Birth Certificate',
+          'file_name' => 'testfile.txt',
+          'obfuscated_file_name' => 'tesXXile.txt',
+          'date_submitted' => date,
+          'date_failed' => date
+        } }.to_json
+      end
 
       it 'logs the document_status_response to the Rails logger' do
         Timecop.freeze(past_date_time) do
@@ -42,7 +57,7 @@
             end
           end
 
-          it 'updates status, failed_date, and acknowledgement_date' do
+          it 'updates status, failed_date, acknowledgement_date and template_metadata' do
             Timecop.freeze(current_date_time) do
               expect { status_updater.update_status }
                 .to change(lighthouse_document_upload, :acknowledgement_date)
@@ -54,6 +69,9 @@
                 .and change(lighthouse_document_upload, :upload_status)
                 .from(BenefitsDocuments::Constants::UPLOAD_STATUS[:PENDING])
                 .to(BenefitsDocuments::Constants::UPLOAD_STATUS[:FAILED])
+                .and change(lighthouse_document_upload, :template_metadata)
+                .from(lighthouse_document_upload.template_metadata)
+                .to(updated_template_metadata)
             end
           end
         end

spec/sidekiq/evss/document_upload_spec.rb

@@ -36,10 +36,10 @@
   end
   let(:job_class) { 'EVSS::DocumentUpload' }
   let(:job_id) { job }
-
   let(:client_stub) { instance_double(EVSS::DocumentsService) }
   let(:notify_client_stub) { instance_double(VaNotify::Service) }
   let(:issue_instant) { Time.now.to_i }
+  let(:current_date_time) { DateTime.now.utc }
   let(:msg) do
     {
       'jid' => job_id,
@@ -103,7 +103,9 @@
     end
 
     context 'when upload fails' do
-      let(:evidence_submission_failed) { create(:bd_evidence_submission_failed) }
+      let(:evidence_submission_failed) do
+        create(:bd_evss_evidence_submission_failed_type1_error)
+      end
       let!(:evidence_submission_pending) do
         create(:bd_evidence_submission_pending,
                tracked_item_id:,
@@ -133,7 +135,13 @@
         evidence_submission = EvidenceSubmission.find_by(job_id: job_id)
         current_personalisation = JSON.parse(evidence_submission.template_metadata)['personalisation']
         expect(evidence_submission.upload_status).to eql(BenefitsDocuments::Constants::UPLOAD_STATUS[:FAILED])
+        expect(evidence_submission.error_message).to eql('EVSS::DocumentUpload document upload failure')
         expect(current_personalisation['date_failed']).to eql(failed_date)
+
+        Timecop.freeze(current_date_time) do
+          expect(evidence_submission.failed_date).to be_within(1.second).of(current_date_time.utc)
+          expect(evidence_submission.acknowledgement_date).to be_within(1.second).of((current_date_time + 30.days).utc)
+        end
       end
 
       it 'fails to create a failed evidence submission record when args malformed' do

spec/sidekiq/lighthouse/evidence_submissions/document_upload_spec.rb

@@ -35,10 +35,10 @@
   end
   let(:user_account) { create(:user_account) }
   let(:job_id) { job }
-
   let(:client_stub) { instance_double(BenefitsDocuments::WorkerService) }
   let(:job_class) { 'Lighthouse::EvidenceSubmissions::DocumentUpload' }
   let(:issue_instant) { Time.now.to_i }
+  let(:current_date_time) { DateTime.now.utc }
   let(:msg) do
     {
       'jid' => job_id,
@@ -58,7 +58,9 @@
   end
 
   # Create Evidence Submission records from factory
-  let(:evidence_submission_failed) { create(:bd_evidence_submission_failed) }
+  let(:evidence_submission_failed) do
+    create(:bd_lh_evidence_submission_failed_type1_error)
+  end
   let(:evidence_submission_pending) do
     create(:bd_evidence_submission_pending,
            tracked_item_id: tracked_item_ids,
@@ -90,6 +92,9 @@ def mock_response(status:, body:)
           }
         )
       end
+      let(:failed_date) do
+        BenefitsDocuments::Utilities::Helpers.format_date_for_mailers(issue_instant)
+      end
 
       it 'retrieves the file, uploads to Lighthouse and returns a success response' do
         allow(LighthouseDocumentUploader).to receive(:new) { uploader_stub }
@@ -153,7 +158,14 @@ def mock_response(status:, body:)
         evidence_submission = EvidenceSubmission.find_by(job_id: job_id)
         current_personalisation = JSON.parse(evidence_submission.template_metadata)['personalisation']
         expect(evidence_submission.upload_status).to eql(BenefitsDocuments::Constants::UPLOAD_STATUS[:FAILED])
+        expect(evidence_submission.error_message)
+          .to eql('Lighthouse::EvidenceSubmissions::DocumentUpload document upload failure')
         expect(current_personalisation['date_failed']).to eql(failed_date)
+
+        Timecop.freeze(current_date_time) do
+          expect(evidence_submission.failed_date).to be_within(1.second).of(current_date_time.utc)
+          expect(evidence_submission.acknowledgement_date).to be_within(1.second).of((current_date_time + 30.days).utc)
+        end
       end
 
       it 'fails to create a failed evidence submission record when args malformed' do