').gsub(REGEXES[:replaceFontsRe], '<\1span>') @remove_unlikely_candidates = @options[:remove_unlikely_candidates] @weight_classes = @options[:weight_classes] @clean_conditionally = @options[:clean_conditionally] @best_candidate_has_image = true make_html handle_exclusions!(@options[:whitelist], @options[:blacklist]) end def prepare_candidates @html.css("script, style").each { |i| i.remove } remove_unlikely_candidates! if @remove_unlikely_candidates transform_misused_divs_into_paragraphs! @candidates = score_paragraphs(options[:min_text_length]) @best_candidate = select_best_candidate(@candidates) end def handle_exclusions!(whitelist, blacklist) return unless whitelist || blacklist if blacklist elems = @html.css(blacklist) if elems elems.each do |e| e.remove end end end if whitelist elems = @html.css(whitelist).to_s if body = @html.at_css('body') body.inner_html = elems end end @input = @html.to_s end def make_html(whitelist=nil, blacklist=nil) @html = Nokogiri::HTML(@input, nil, @options[:encoding]) # In case document has no body, such as from empty string or redirect @html = Nokogiri::HTML('', nil, @options[:encoding]) if @html.css('body').length == 0 # Remove html comment tags @html.xpath('//comment()').each { |i| i.remove } end def images(content=nil, reload=false) begin require 'fastimage' rescue LoadError raise "Please install fastimage in order to use the #images feature." end @best_candidate_has_image = false if reload prepare_candidates list_images = [] tested_images = [] content = @best_candidate[:elem] unless reload return list_images if content.nil? elements = content.css("img").map(&:attributes) elements.each do |element| next unless element["src"] url = element["src"].value height = element["height"].nil? ? 0 : element["height"].value.to_i width = element["width"].nil? ? 0 : element["width"].value.to_i if url =~ /\Ahttps?:\/\//i && (height.zero? || width.zero?) image = get_image_size(url) next unless image else image = {:width => width, :height => height} end image[:format] = File.extname(url).gsub(".", "") if tested_images.include?(url) debug("Image was tested: #{url}") next end tested_images.push(url) if image_meets_criteria?(image) list_images << url else debug("Image discarded: #{url} - height: #{image[:height]} - width: #{image[:width]} - format: #{image[:format]}") end end (list_images.empty? and content != @html) ? images(@html, true) : list_images end def images_with_fqdn_uris!(source_uri) images_with_fqdn_uris(@html, source_uri) end def images_with_fqdn_uris(document = @html.dup, source_uri) uri = URI.parse(source_uri) host = uri.host scheme = uri.scheme port = uri.port # defaults to 80 base = "#{scheme}://#{host}:#{port}/" images = [] document.css("img").each do |elem| begin elem['src'] = URI.join(base,elem['src']).to_s if URI.parse(elem['src']).host == nil images << elem['src'].to_s rescue URI::InvalidURIError => exc elem.remove end end images(document,true) end def get_image_size(url) w, h = FastImage.size(url) raise "Couldn't get size." if w.nil? || h.nil? {:width => w, :height => h} rescue => e debug("Image error: #{e}") nil end def image_meets_criteria?(image) return false if options[:ignore_image_format].include?(image[:format].downcase) image[:width] >= (options[:min_image_width] || 0) && image[:height] >= (options[:min_image_height] || 0) end def title title = @html.css("title").first title ? title.text : nil end # Look through the @html document looking for the author # Precedence Information here on the wiki: (TODO attach wiki URL if it is accepted) # Returns nil if no author is detected def author # Let's grab this author: # author_elements = @html.xpath('//meta[@name = "dc.creator"]') unless author_elements.empty? author_elements.each do |element| return element['content'].strip if element['content'] end end # Now let's try to grab this # ByAustin Fonacier #

s that do not contain other block elements into

s if elem.inner_html !~ REGEXES[:divToPElementsRe] debug("Altering div(##{elem[:id]}.#{elem[:class]}) to p"); elem.name = "p" end else # wrap text nodes in p tags # elem.children.each do |child| # if child.text? # debug("wrapping text node with a p") # child.swap("

#{child.text}

") # end # end end end end def sanitize(node, candidates, options = {}) node.css("h1, h2, h3, h4, h5, h6").each do |header| header.remove if class_weight(header) < 0 || get_link_density(header) > 0.33 end node.css("form, object, iframe, embed").each do |elem| elem.remove end if @options[:remove_empty_nodes] # remove

tags that have no text content - this will also remove p tags that contain only images. node.css("p").each do |elem| elem.remove if elem.content.strip.empty? end end # Conditionally clean s,
s, and
s clean_conditionally(node, candidates, "table, ul, div") # We'll sanitize all elements using a whitelist base_whitelist = @options[:tags] || %w[div p] all_whitelisted = base_whitelist.include?("*") # We'll add whitespace instead of block elements, # so a
b will have a nice space between them base_replace_with_whitespace = %w[br hr h1 h2 h3 h4 h5 h6 dl dd ol li ul address blockquote center] # Use a hash for speed (don't want to make a million calls to include?) whitelist = Hash.new base_whitelist.each {|tag| whitelist[tag] = true } replace_with_whitespace = Hash.new base_replace_with_whitespace.each { |tag| replace_with_whitespace[tag] = true } ([node] + node.css("*")).each do |el| # If element is in whitelist, delete all its attributes if all_whitelisted || whitelist[el.node_name] el.attributes.each { |a, x| el.delete(a) unless @options[:attributes] && @options[:attributes].include?(a.to_s) } # Otherwise, replace the element with its contents else # If element is root, replace the node as a text node if el.parent.nil? node = Nokogiri::XML::Text.new(el.text, el.document) break else if replace_with_whitespace[el.node_name] el.swap(Nokogiri::XML::Text.new(' ' << el.text << ' ', el.document)) else el.swap(Nokogiri::XML::Text.new(el.text, el.document)) end end end end s = Nokogiri::XML::Node::SaveOptions save_opts = s::NO_DECLARATION | s::NO_EMPTY_TAGS | s::AS_XHTML html = node.serialize(:save_with => save_opts) # Get rid of duplicate whitespace return html.gsub(/[\r\n\f]+/, "\n" ) end def clean_conditionally(node, candidates, selector) return unless @clean_conditionally node.css(selector).each do |el| weight = class_weight(el) content_score = candidates[el] ? candidates[el][:content_score] : 0 name = el.name.downcase if weight + content_score < 0 el.remove debug("Conditionally cleaned #{name}##{el[:id]}.#{el[:class]} with weight #{weight} and content score #{content_score} because score + content score was less than zero.") elsif el.text.count(",") < 10 counts = %w[p img li a embed input].inject({}) { |m, kind| m[kind] = el.css(kind).length; m } counts["li"] -= 100 # For every img under a noscript tag discount one from the count to avoid double counting counts["img"] -= el.css("noscript").css("img").length content_length = el.text.strip.length # Count the text length excluding any surrounding whitespace link_density = get_link_density(el) reason = clean_conditionally_reason?(name, counts, content_length, options, weight, link_density) if reason debug("Conditionally cleaned #{name}##{el[:id]}.#{el[:class]} with weight #{weight} and content score #{content_score} because it has #{reason}.") el.remove end end end end def clean_conditionally_reason?(name, counts, content_length, options, weight, link_density) if (counts["img"] > counts["p"]) && (counts["img"] > 1) "too many images" elsif counts["li"] > counts["p"] && name != "ul" && name != "ol" "more
s than
s" elsif counts["input"] > (counts["p"] / 3).to_i "less than 3x
s than s" elsif (content_length < options[:min_text_length]) && (counts["img"] != 1) "too short a content length without a single image" elsif weight < 25 && link_density > 0.2 "too many links for its weight (#{weight})" elsif weight >= 25 && link_density > 0.5 "too many links for its weight (#{weight})" elsif (counts["embed"] == 1 && content_length < 75) || counts["embed"] > 1 "s with too short a content length, or too many s" else nil end end end end