Contents

module Boilerpipe::Extractors
  class DefaultExtractor
    def self.text(contents)
      doc = ::Boilerpipe::SAX::BoilerpipeHTMLParser.parse(contents)
      ::Boilerpipe::Extractors::DefaultExtractor.process doc
      doc.content
    end

    def self.process(doc)
      filters = ::Boilerpipe::Filters
      # merge adjacent blocks with equal text_density
      filters::SimpleBlockFusionProcessor.process doc

      # merge text blocks next to each other
      filters::BlockProximityFusion::MAX_DISTANCE_1.process doc

      # marks text blocks as content / non-content using boilerpipe alg
      filters::DensityRulesClassifier.process doc

      doc
    end
  end
end

Version data entries

5 entries across 5 versions & 1 rubygems

Version	Path
boilerpipe-ruby-0.5.0	lib/boilerpipe/extractors/default_extractor.rb
boilerpipe-ruby-0.4.4	lib/boilerpipe/extractors/default_extractor.rb
boilerpipe-ruby-0.4.3	lib/boilerpipe/extractors/default_extractor.rb
boilerpipe-ruby-0.4.2	lib/boilerpipe/extractors/default_extractor.rb
boilerpipe-ruby-0.4.1	lib/boilerpipe/extractors/default_extractor.rb