require 'fuzzy_match/rule' require 'fuzzy_match/rule/grouping' require 'fuzzy_match/rule/identity' require 'fuzzy_match/result' require 'fuzzy_match/record' require 'fuzzy_match/similarity' require 'fuzzy_match/score' # See the README for more information. class FuzzyMatch class << self def engine @engine end def engine=(alt_engine) @engine = alt_engine end def score_class case engine when :pure_ruby Score::PureRuby when :amatch Score::Amatch else raise ::ArgumentError, "[fuzzy_match] #{engine.inspect} is not a recognized engine." end end end DEFAULT_ENGINE = :pure_ruby #TODO refactor at least all the :find_X things DEFAULT_OPTIONS = { :must_match_grouping => false, :must_match_at_least_one_word => false, :gather_last_result => false, :find_all => false, :find_all_with_score => false, :threshold => 0, :find_best => false, :find_with_score => false, } self.engine = DEFAULT_ENGINE attr_reader :haystack attr_reader :groupings attr_reader :identities attr_reader :stop_words attr_accessor :read attr_reader :default_options # haystack - a bunch of records that will compete to see who best matches the needle # # Rules (can only be specified at initialization or by using a setter) # * :identities - regexps # * :groupings - regexps # * :stop_words - regexps # * :read - how to interpret each record in the 'haystack', either a Proc or a symbol # # Options (can be specified at initialization or when calling #find) # * :must_match_grouping - don't return a match unless the needle fits into one of the groupings you specified # * :must_match_at_least_one_word - don't return a match unless the needle shares at least one word with the match # * :gather_last_result - enable last_result # * :threshold - set a score threshold below which not to return results (not generally recommended - please test the results of setting a threshold thoroughly - one set of results and their scores probably won't be enough to determine the appropriate number). Only checked against the Pair Distance score and ignored when one string or the other is of length 1. def initialize(haystack, options_and_rules = {}) o = options_and_rules.dup # rules @read = o.delete(:read) || o.delete(:haystack_reader) @groupings = (o.delete(:groupings) || o.delete(:blockings) || []).map { |regexp| Rule::Grouping.make(regexp) }.flatten @identities = (o.delete(:identities) || []).map { |regexp| Rule::Identity.new(regexp) } @stop_words = o.delete(:stop_words) || [] # options if deprecated = o.delete(:must_match_blocking) o[:must_match_grouping] = deprecated end @default_options = DEFAULT_OPTIONS.merge(o).freeze @haystack = haystack.map { |original| Record.new original, :stop_words => @stop_words, :read => @read } end def last_result @last_result or raise("You can't access the last result until you've run a find with :gather_last_result => true") end # Return everything in sorted order def find_all(needle, options = {}) options = options.merge(:find_all => true) find needle, options end # Return the top results with the same score def find_best(needle, options = {}) options = options.merge(:find_best => true) find needle, options end # Return everything in sorted order with score def find_all_with_score(needle, options = {}) options = options.merge(:find_all_with_score => true) find needle, options end # Return one with score def find_with_score(needle, options = {}) options = options.merge(:find_with_score => true) find needle, options end def find(needle, options = {}) options = default_options.merge options threshold = options[:threshold] gather_last_result = options[:gather_last_result] is_find_all_with_score = options[:find_all_with_score] is_find_with_score = options[:find_with_score] is_find_best = options[:find_best] is_find_all = options[:find_all] || is_find_all_with_score || is_find_best must_match_grouping = options[:must_match_grouping] must_match_at_least_one_word = options[:must_match_at_least_one_word] if gather_last_result @last_result = Result.new last_result.read = read last_result.haystack = haystack last_result.options = options end if gather_last_result last_result.identities = identities last_result.groupings = groupings last_result.stop_words = stop_words end needle = Record.new needle if gather_last_result last_result.needle = needle end if groupings.any? first_grouping = groupings.detect { |grouping| grouping.xmatch? needle } if gather_last_result if first_grouping last_result.timeline << "Grouping: #{first_grouping.inspect}" else last_result.timeline << "No grouping." end end end if must_match_grouping and not first_grouping if gather_last_result last_result.timeline << <<-EOS The needle didn't match any of the #{groupings.length} groupings, which was a requirement. \t#{groupings.map(&:inspect).join("\n\t")} EOS end if is_find_all return [] else return nil end end if groupings.any? and not first_grouping passed_grouping_requirement = haystack.reject do |straw| groupings.any? { |grouping| grouping.xmatch? straw } end else passed_grouping_requirement = haystack end if must_match_at_least_one_word passed_word_requirement = passed_grouping_requirement.select do |straw| (needle.words & straw.words).any? end if gather_last_result last_result.timeline << <<-EOS Since :must_match_at_least_one_word => true, the competition was reduced to records sharing at least one word with the needle. \tNeedle words: #{needle.words.map(&:inspect).join(', ')} \tPassed (first 3): #{passed_word_requirement[0,3].map(&:inspect).join(', ')} \tFailed (first 3): #{(passed_grouping_requirement-passed_word_requirement)[0,3].map(&:inspect).join(', ')} EOS end else passed_word_requirement = passed_grouping_requirement end if first_grouping joint = passed_word_requirement.select do |straw| first_grouping.xjoin? needle, straw end # binding.pry if gather_last_result last_result.timeline << <<-EOS Since there were groupings, the competition was reduced to #{joint.length} records in the same group as the needle. \t#{joint.map(&:inspect).join("\n\t")} EOS end else joint = passed_word_requirement.dup end if joint.none? if must_match_grouping if gather_last_result last_result.timeline << <<-EOS Since :must_match_at_least_one_word => true and none of the competition was in the same group as the needle, the search stopped. EOS end if is_find_all return [] else return nil end else joint = passed_word_requirement.dup end end if identities.any? possibly_identical = joint.select do |straw| identities.all? do |identity| answer = identity.identical? needle, straw answer.nil? or answer == true end end if gather_last_result last_result.timeline << <<-EOS Since there were identities, the competition was reduced to records that might be identical to the needle (in other words, are not certainly different) \tIdentities (first 10 of #{identities.length}): #{identities[0,9].map(&:inspect).join(', ')} \tPassed (first 10 of #{possibly_identical.length}): #{possibly_identical[0,9].map(&:inspect).join(', ')} \tFailed (first 10 of #{(joint-possibly_identical).length}): #{(joint-possibly_identical)[0,9].map(&:inspect).join(', ')} EOS end else possibly_identical = joint.dup end similarities = possibly_identical.map { |straw| needle.similarity straw }.sort.reverse if gather_last_result last_result.timeline << <<-EOS The competition was sorted in order of similarity to the needle. \t#{similarities[0,9].map { |s| "#{s.record2.similarity(needle).inspect}" }.join("\n\t")} EOS end if is_find_all_with_score memo = [] similarities.each do |similarity| if similarity.satisfy?(needle, threshold) bs = similarity.best_score memo << [similarity.record2.original, bs.dices_coefficient_similar, bs.levenshtein_similar] end end return memo end if is_find_best memo = [] best_bs = nil similarities.each do |similarity| if similarity.satisfy?(needle, threshold) bs = similarity.best_score best_bs ||= bs if bs >= best_bs memo << similarity.record2.original else break end end end return memo end if is_find_all memo = [] similarities.each do |similarity| if similarity.satisfy?(needle, threshold) memo << similarity.record2.original end end return memo end best_similarity = similarities.first winner = nil if best_similarity and best_similarity.satisfy?(needle, threshold) winner = best_similarity.record2.original if gather_last_result last_result.winner = winner last_result.score = best_similarity.best_score.dices_coefficient_similar last_result.timeline << <<-EOS A winner was determined because the Dice's Coefficient similarity (#{'%0.5f' % best_similarity.best_score.dices_coefficient_similar}) is greater than zero or because it shared a word with the needle. EOS end if is_find_with_score bs = best_similarity.best_score return [winner, bs.dices_coefficient_similar, bs.levenshtein_similar] else return winner end elsif gather_last_result best_similarity_record = if best_similarity and best_similarity.record2 best_similarity.record2.original end last_result.timeline << <<-EOS No winner assigned because the score of the best similarity (#{best_similarity_record.inspect}) was zero and it didn't match any words with the needle (#{needle.inspect}). EOS end nil # ugly end # Explain is like mysql's EXPLAIN command. You give it a needle and it tells you about how it was located (successfully or not) in the haystack. # # d = FuzzyMatch.new ['737', '747', '757' ] # d.explain 'boeing 737-100' def explain(needle, options = {}) find needle, options.merge(:gather_last_result => true) last_result.explain end end