lib/fushin/posts/post.rb in fushin-0.3.1 vs lib/fushin/posts/post.rb in fushin-0.3.2

@@ -26,37 +26,30 @@
 
           out << body
         end.first
       end
 
-      def main_text
-        @main_text ||= [].tap do |out|
-          detection = CharlockHolmes::EncodingDetector.detect(main.text)
-          out << CharlockHolmes::Converter.convert(main.text, detection[:encoding], "UTF-8")
-        end.first
-      end
-
       def btcs
-        @btcs ||= main_text.scan(/\b[13][a-km-zA-HJ-NP-Z0-9]{26,33}\b/).uniq.map do |address|
+        @btcs ||= main.text.scan(/\b[13][a-km-zA-HJ-NP-Z0-9]{26,33}\b/).uniq.map do |address|
           Models::BTC.new(address)
         end
       end
 
       def urls
-        @urls ||= main_text.scan(UrlRegex.get(scheme_required: true, mode: :parsing)).uniq.map do |url|
+        @urls ||= (urls_in_text + links).uniq.map do |url|
           next if whitelisted_domain?(url)
 
           Models::Website.new(url)
-        end.compact
+        end.compact.uniq(&:normalized_url)
       end
 
-      def links
-        @links ||= main.css("a").map { |a| a.get("href") }.compact.uniq.map do |url|
-          next if whitelisted_domain?(url)
+      def urls_in_text
+        @urls_in_text ||= main.text.scan(UrlRegex.get(scheme_required: true, mode: :parsing))
+      end
 
-          Models::Website.new(url)
-        end.compact
+      def links
+        @links ||= main.css("a").map { |a| a.get("href") }.compact
       end
 
       def attachements
         []
       end
@@ -88,10 +81,11 @@
 
       def body
         res = HTTP.get(url)
         return nil unless res.code == 200
 
-        res.body.to_s
+        detection = CharlockHolmes::EncodingDetector.detect(res.body.to_s)
+        CharlockHolmes::Converter.convert(res.body.to_s, detection[:encoding], "UTF-8")
       end
     end
   end
 end