lib/boilerpipe/sax/boilerpipe_html_parser.rb in boilerpipe-ruby-0.4.0 vs lib/boilerpipe/sax/boilerpipe_html_parser.rb in boilerpipe-ruby-0.4.1

- old
+ new

@@ -1,20 +1,16 @@ -require 'nokogiri' module Boilerpipe::SAX class BoilerpipeHTMLParser def self.parse(text) - - #script bug - delete script tags + # script bug - delete script tags text.gsub!(/\<script>.+?<\/script>/i, '') # nokogiri uses libxml for mri and nekohtml for jruby # mri doesn't remove &nbsp; when missing the semicolon text.gsub!(/(&nbsp) /, '\1; ') - # use nokogiri to fix any bad tags, errors - keep experimenting with this text = Nokogiri::HTML(text).to_html - handler = HTMLContentHandler.new noko_parser = Nokogiri::HTML::SAX::Parser.new(handler) noko_parser.parse(text) handler.text_document