12179 create Answers table and migrate existing response data to populate it

app/controllers/admin/questions_controller.rb

@@ -11,7 +11,7 @@ def index
     @question_sets = QuestionSet.find_for_division(selected_division_or_root)
     if @question_sets.any?
       @question_set = params.key?(:qset) ? QuestionSet.find(params[:qset]) : @question_sets.first
-      @questions = ActiveModel::Serializer::CollectionSerializer.new(top_level_questions(@question_set))
+      @questions = ActiveModel::Serializer::CollectionSerializer.new(top_level_questions(@question_set), include_inheritance: true)
       @selected_division_depth = selected_division.depth
     end
   end

app/controllers/admin/response_sets_controller.rb

@@ -18,6 +18,7 @@ def create
       handle_conflict
     else
       @response_set.save!
+      @response_set.save_answers(response_set_params)
       redirect_to display_path, notice: I18n.t(:notice_created)
     end
   end
@@ -45,6 +46,7 @@ def update
       redirect_to display_path
     else
       @response_set.update!(adjusted_params)
+      @response_set.save_answers(adjusted_params)
       redirect_to display_path, notice: I18n.t(:notice_updated)
     end
   rescue ActiveRecord::StaleObjectError

app/models/answer.rb

@@ -0,0 +1,181 @@
+class Answer < ApplicationRecord
+  belongs_to :response_set,  optional: false
+  belongs_to :question, optional: false
+  delegate :data_type, to: :question
+  validate :has_data
+  validate :question_is_not_group
+
+  # this method is temporary for spr 2022 overhaul
+  def compare_to_custom_data
+    custom_data_raw_data = response_set.custom_data[question.id.to_s]
+    custom_data_response = Response.new(loan: response_set.loan, question: question, response_set: response_set, data: custom_data_raw_data)
+    answer_response = Response.new(loan: response_set.loan, question: question, response_set: response_set, data: raw_value)
+    methods = [:loan, :question, :response_set, :text, :number, :boolean, :rating, :url, :start_cell, :end_cell, :owner, :breakeven, :business_canvas, :not_applicable]
+    methods.each do |m|
+      custom_data_value = custom_data_response.send(m)
+      answer_value = answer_response.send(m)
+      unless custom_data_value == answer_value
+        raise "ERROR for answer #{id}: for RS #{response_set.id} custom data value for #{m} is #{custom_data_value} but is #{answer_value} for answer #{id}"
+      end
+    end
+  end
+
+  def to_s
+    "RS: #{response_set.question_set.kind}, Q: #{question.label.to_s} | NA: #{not_applicable}; text: #{text_data}; numeric: #{numeric_data}; boolean: #{boolean_data}; doc: #{linked_document_data}; breakeven: #{breakeven_data}; canvas: #{business_canvas_data}"
+  end
+
+  def blank?
+    !not_applicable? &&
+    text_data.blank? &&
+    numeric_data.blank? &&
+    boolean_data.blank? &&
+    linked_document_data_blank? &&
+    business_canvas_blank? &&
+    breakeven_data_blank?
+  end
+
+  def linked_document_data_blank?
+    linked_document_data.blank? || json_answer_blank?(linked_document_data)
+  end
+
+  def business_canvas_blank?
+    business_canvas_data.blank? || json_answer_blank?(business_canvas_data)
+  end
+
+  def breakeven_data_blank?
+    breakeven_data.blank? || json_answer_blank?(breakeven_data)
+  end
+
+  def self.json_answer_blank?(answer_json)
+    answer_json.values.all?{|v| v.blank?}
+  end
+
+  # expects 'raw_value' type json e.g. the value of a "field_110" key in form submission
+  # or the value of a q_id key e.g. "5126" in custom_data
+  def self.contains_answer_data?(hash_data)
+    hash_data.each do |key, value|
+      if %w(text number rating url start_cell end_cell).include?(key)
+        return true unless value.blank?
+      elsif key == "not_applicable"
+        return true if value == "yes"
+      elsif key == "boolean"
+        return true unless value.nil?
+      elsif %w(business_canvas).include?(key)
+        return true unless self.json_answer_blank?(value)
+      elsif %w(breakeven).include?(key)
+        value.each do |subkey, subvalue|
+          if %w(products fixed_costs).include?(subkey)
+            subvalue.each {|i| return true unless self.json_answer_blank?(i)}
+          else
+            return true unless subvalue.empty?
+          end
+        end
+      end
+    end
+    return false
+  end
+
+  # this method is temporary for spr 2022 overhaul
+  # doesn't save blank answers
+  def self.save_from_form_field_params(question, fields, response_set)
+    unless question.group? || !self.contains_answer_data?(fields)
+      not_applicable = fields.key?("not_applicable") ? (fields["not_applicable"] == "yes") : "no"
+      text_data = fields.key?("text") ? fields["text"] : nil
+      numeric_data = if fields.key?("number")
+          fields["number"]
+        elsif fields.key?("rating")
+          fields["rating"]
+        else
+          nil
+        end
+      boolean_data = fields.key?("boolean") ? (fields["boolean"] == "yes") : nil
+      breakeven_data = fields.key?("breakeven") ? fields["breakeven"] : nil
+      business_canvas_data = fields.key?("business_canvas") ? fields["business_canvas"] : nil
+      linked_document_data = fields.key?("url") ? {"url": fields["url"] } : {"url": ""}
+      linked_document_data["start_cell"] = fields.key?("start_cell") ? fields["start_cell"] : ""
+      linked_document_data["end_cell"] = fields.key?("end_cell") ? fields["end_cell"] : ""
+      answer = Answer.find_or_create_by(response_set: response_set, question: question)
+      answer.update!({
+          not_applicable: not_applicable,
+          text_data: text_data,
+          numeric_data: numeric_data,
+          boolean_data: boolean_data,
+          breakeven_data: breakeven_data,
+          business_canvas_data: business_canvas_data,
+          linked_document_data: linked_document_data
+        })
+    end
+  end
+
+  def question_is_not_group
+    question.data_type != "group"
+  end
+
+  def has_data
+    errors.add("Answer contains no data") unless
+      not_applicable ||
+      text_data.present? ||
+      numeric_data.present? ||
+      !boolean_data.nil? ||
+      linked_document_data.present? ||
+      business_canvas_data.present? ||
+      breakeven_data.present?
+  end
+
+  # temp method for spr 2022 overhaul
+  def raw_value
+    json = {}
+    json["not_applicable"] = self.not_applicable ? "yes" : "no"
+    if self.text_data.present?
+      json["text"] = self.text_data
+    end
+    unless self.boolean_data.nil?
+      json["boolean"] =  self.boolean_data ? "yes" : "no"
+    end
+    if self.breakeven_data.present?
+      json["breakeven"] = self.breakeven_data
+    end
+    if self.business_canvas_data.present?
+      json["business_canvas"] = self.business_canvas_data
+    end
+    if self.numeric_data.present?
+      if self.question.data_type == "range"
+        json["rating"] = self.numeric_data
+      else
+        json["number"] = self.numeric_data
+      end
+    end
+    if self.linked_document_data.present?
+      json["url"] = self.linked_document_data["url"]
+      json["start_cell"] = self.linked_document_data["start_cell"]
+      json["end_cell"] = self.linked_document_data["end_cell"]
+    end
+    json
+  end
+
+  # temp method for spr 2022 overhaul
+  # return the value of json that would be in legacy custom_data field on response set for this answer's question
+  def custom_data_json
+    return {"#{self.question.json_key}": self.raw_value}
+  end
+
+  def answer_for_csv(allow_text_like_numeric: false)
+    return nil if not_applicable
+
+    case question.data_type
+    when "text"
+      text_data
+    when "number", "currency", "percentage", "range"
+      if allow_text_like_numeric || (true if Float(numeric_data) rescue false)
+       numeric_data.to_s
+      else
+        nil
+      end
+    when "boolean"
+      boolean_data.nil? ? nil : (boolean_data ? "yes" : "no")
+    # "breakeven" and "business_canvas" never exported to csv
+    else
+      raise "invalid question data type #{question.data_type}"
+    end
+  end
+end

app/models/data_export.rb

@@ -43,6 +43,7 @@ def process_data
   end
 
   def to_csv!
+    logger.debug("LOGGING FROM data_export.rb")
     raise ArgumentError, "No data found" if data.blank?
     raise TypeError, "Data should be a 2D Array" unless (data.is_a?(Array) && data.first.is_a?(Array))
 
@@ -76,7 +77,12 @@ def task
   protected
 
   def insert_in_row(header_symbol, row_array, value)
-    row_array[header_symbols_to_indices[header_symbol]] = value
+    index = header_symbols_to_indices[header_symbol]
+    if index.nil?
+      raise "Header symbol #{header_symbol} not found in #{header_symbols_to_indices}. Failed to add #{value}"
+    else
+      row_array[index] = value
+    end
   end
 
   private
@@ -89,7 +95,7 @@ def hash_to_row(hash)
 
   # Builds a hash of header symbols to their appropriate indices in the row arrays.
   def header_symbols_to_indices
-    @header_symbols_to_indices = header_symbols.each_with_index.to_h
+    @header_symbols_to_indices ||= header_symbols.each_with_index.to_h
   end
 
   def set_name

app/models/enhanced_loan_data_export.rb

@@ -11,39 +11,15 @@ def response_hash(loan)
     result = {}
     response_sets = ResponseSet.joins(:question_set).where(loan: loan).order("question_sets.kind")
     response_sets.each do |response_set|
-      response_set.custom_data.each do |q_id, response_data|
-        question = questions_by_id[q_id.to_i]
-        if question.present?
-          response = Response.new(loan: loan, question: question,
-                                  response_set: response_set, data: response_data)
-
-          # Note, this approach will exclude parts of compound data types, such as `range`,
-          # which can have both a `rating` and a `text` component.
-          # `url`, `start_cell`, and `end_cell` components from questions with `has_embeddable_media`=true
-          # are also not included, nor are `business_canvas`, and `breakeven`, which would be way too big
-          # to put in a CSV cell.
-          result[q_id.to_i] =
-            if response.not_applicable?
-              ""
-            elsif response.has_rating?
-              response.rating
-            elsif response.has_number?
-              include_numeric_answer_in_export?(response.number) ? response.number : ""
-            elsif response.has_boolean?
-              response.boolean
-            elsif response.has_text?
-              response.text
-            end
+      response_set.answers.each do |a|
+        if q_data_types.include?(a.data_type)
+          result[a.question_id] = a.answer_for_csv(allow_text_like_numeric: allow_text_like_numeric?)
         end
       end
     end
     result
   end
 
-  def include_numeric_answer_in_export?(str)
-    true #include all numeric answers, even if invalid text
-  end
-
   def q_data_types
     ["boolean", "text", "number", "percentage", "rating", "currency", "range"]
   end
@@ -74,9 +50,10 @@ def questions
   end
 
   def question_sets
-    # We want self to come first for deterministic behavior in specs. After that it doesn't really matter.
-    # self_and_descendants orders by depth so we are good.
-    division.self_and_descendants.flat_map { |d| QuestionSet.where(division: d).order(:kind).to_a }
+    # get all question sets for all response sets of loans in this division and descendants
+    loans = division.self_and_descendants.flat_map { |d| Loan.where(division: d) }
+    question_sets = loans.flat_map{|l| l.response_sets.map{|rs| rs.question_set}}.uniq
+    question_sets.sort_by(&:kind)
   end
 
   def questions_by_id
@@ -87,4 +64,8 @@ def questions_by_id
   def header_symbols
     @header_symbols ||= StandardLoanDataExport::HEADERS + questions.map(&:id)
   end
+
+  def allow_text_like_numeric?
+    true #include all numeric answers, even if invalid text
+  end
 end

app/models/filtered_question.rb

@@ -37,6 +37,7 @@ def parent
       end
   end
 
+
   def visible?
     selected_division.self_or_descendant_of?(question.division) ||
       include_descendant_divisions && selected_division.self_or_ancestor_of?(question.division)

app/models/numeric_answer_data_export.rb

@@ -3,7 +3,7 @@ def q_data_types
     ["number", "percentage", "rating", "currency", "range"]
   end
 
-  def include_numeric_answer_in_export?(str)
-    true if Float(str) rescue false
+  def allow_text_like_numeric?
+    false
   end
 end

app/models/option.rb

@@ -17,8 +17,6 @@ class Option < ApplicationRecord
 
   def ensure_value_assigned
     unless value
-      # puts "defaulting value to id: #{self.id}"
-      # make sure we don't have any recursive callbacks
       self.update_column(:value, self.id)
     end
   end

app/models/question.rb

@@ -52,6 +52,10 @@ def top_level?
     parent.root?
   end
 
+  def data_types
+    DATA_TYPES
+  end
+
   # This is a duck type method, and is overridden by LoanFilteredQuestion, which implements
   # the real logic about which questions are required contextually.
   # We need to implement this here because plan Questions need to be serialized for the manage