lib/boilerpipe/sax/boilerpipe_html_parser.rb in boilerpipe-ruby-0.4.0 vs lib/boilerpipe/sax/boilerpipe_html_parser.rb in boilerpipe-ruby-0.4.1
- old
+ new
@@ -1,20 +1,16 @@
-require 'nokogiri'
module Boilerpipe::SAX
class BoilerpipeHTMLParser
def self.parse(text)
-
- #script bug - delete script tags
+ # script bug - delete script tags
text.gsub!(/\<script>.+?<\/script>/i, '')
# nokogiri uses libxml for mri and nekohtml for jruby
# mri doesn't remove when missing the semicolon
text.gsub!(/( ) /, '\1; ')
-
# use nokogiri to fix any bad tags, errors - keep experimenting with this
text = Nokogiri::HTML(text).to_html
-
handler = HTMLContentHandler.new
noko_parser = Nokogiri::HTML::SAX::Parser.new(handler)
noko_parser.parse(text)
handler.text_document