class Eco::API::Common::People::DefaultParsers::CSVParser < Eco::API::Common::Loaders::Parser attribute :csv def parser(data, deps) Eco::CSV.parse(data, headers: true, skip_blanks: true).tap do |table| check_headers(table) if deps[:check_headers] end.each_with_object([]) do |row, arr_hash| row_hash = row.headers.uniq.each_with_object({}) do |attr, hash| next if attr.to_s.strip.empty? hash[attr.strip] = parse_string(row[attr]) end arr_hash.push(row_hash) end end def serializer(array_hash, deps) arr_rows = [] unless array_hash.empty? header = array_hash.first.keys arr_rows = array_hash.map do |csv_row| CSV::Row.new(header, csv_row.values_at(*header)) end end CSV::Table.new(arr_rows).to_csv end private def parse_string(value) return nil if value.to_s.empty? return nil if null?(value) value end def null?(value) return true if !value str = value.strip.upcase ["NULL"].any? {|token| str == token} end def check_headers(table) headers = table.headers missing = missing_headers(headers) unknown = unknown_headers(headers) unless missing.empty? && unknown.empty? msg = "Detected possible HEADER ISSUES !!!\n" msg << "There might be Missing or Wrong HEADER names in the CSV file:\n" msg << " * UNKNOWN (or not used?): #{unknown}\n" unless unknown.empty? msg << " * MISSING DIRECT: #{missing[:direct]}\n" unless (missing[:direct] || []).empty? unless (data = missing[:indirect] || []).empty? msg << " * MISSING INDIRECT:\n" data.each do |ext, info| msg << " - '#{ext}' => " msg << (info[:attrs] || {}).map do |status, attrs| if status == :inactive "makes inactive: #{attrs}" elsif status == :active "there could be missing info in: #{attrs}" end end.compact.join("; ") + "\n" end end logger.warn(msg) sleep(2) end end def unknown_headers(headers) (headers - known_headers) - all_internal_attrs end def missing_headers(headers) hint = headers & all_internal_attrs hext = headers - hint int_head = hint + hext.map {|e| fields_mapper.to_internal(e)}.compact known_as_int = known_headers.select do |e| i = fields_mapper.to_internal(e) int_head.include?(i) end ext = headers.select do |e| i = fields_mapper.to_internal(e) int_head.include?(i) end ext_present = known_as_int | ext ext_miss = known_headers - ext_present #int_miss = ext_miss.map {|ext| fields_mapper.to_internal(ext)} ext_miss.each_with_object({}) do |ext, missing| next unless int = fields_mapper.to_internal(ext) if all_internal_attrs.include?(int) missing[:direct] ||= [] missing[:direct] << ext end related_attrs_requirements = required_attrs.values.select do |req| req.dependant?(int) && !int_head.include?(req.attr) end next if related_attrs_requirements.empty? missing[:indirect] ||= {} data = missing[:indirect][ext] = {} data[:int] = int data[:attrs] = {} related_attrs_requirements.each_with_object(data[:attrs]) do |req, attrs| status = req.active?(*int_head) ? :active : :inactive attrs[status] ||= [] attrs[status] << req.attr end end end def known_headers @known_headers ||= fields_mapper.list(:external).compact end def fields_mapper session.fields_mapper end def required_attrs @required_attrs ||= person_parser.required_attrs.each_with_object({}) do |ra, out| out[ra.attr] = ra end end def all_internal_attrs person_parser.all_attrs(include_defined_parsers: true) end def person_parser session.entry_factory.person_parser end end