lib/twexicon/scraper.rb in twexicon-0.1.7.1 vs lib/twexicon/scraper.rb in twexicon-0.1.8

@@ -12,15 +12,15 @@
 
   def refine_tweets
     tweets.each do |num, tweet|
       t = tweet.keys[0].dup
       t.scan(/pic.twitter.com\/\w{10}/){|p| tweet.values[0][:pix] << p.strip}.gsub!(/pic.twitter.com\/\w{10}/, " ")
-      t.scan(/https?:\/\/[\w\.\?\=\&\-\/\#]+/){|w| tweet.values[0][:links] << w.strip}.gsub!(/https?:\/\/[\w\.\?\=\&\-\/\#]+/, " ")
+      t.scan(/https?:\/\/[\w\.\?\=\&\-\/\#\:]+/){|w| tweet.values[0][:links] << w.strip}.gsub!(/https?:\/\/[\w\.\?\=\&\-\/\#\:]+/, " ")
       t.scan(/#\w+/){|h| tweet.values[0][:hashtags] << h.gsub(/\W/, "").prepend("#")}.gsub!(/#\w+/, " ")
       t.scan(/@\w+/){|u| tweet.values[0][:usernames] << u.gsub(/\W/, "").prepend("@")}.gsub!(/@\w+/, " ")
-      t.scan(/(\d+[:\.\b]?\d*)+/){|n| tweet.values[0][:numbers] << n.first.gsub(/(^\W+|\W+$)/, "")}.gsub!(/(\d+[:\.\b]?\d*)+/, " ")
-      t.scan(/(\b[A-Z][\.\b][A-Z][\.\b][A-Z][\.\b]|\b[A-Z][\.\b][A-Z][\.\b])/){|a| tweet.values[0][:acronyms] << a.first.strip}.gsub!(/(\b[A-Z][\.\b][A-Z][\.\b][A-Z][\.\b]|\b[A-Z][\.\b][A-Z][\.\b])/, " ")
-      t.scan(/(([A-Z]+\W){2,}|[A-Z]{4,}\W)/){|s| tweet.values[0][:shouts] << s.first.gsub(/\W/, " ").strip}.gsub!(/(([A-Z]+\W){2,}|[A-Z]{4,}\W)/, " ")
+      t.scan(/(\d+[:\.\ ]?\d*)+/){|n| tweet.values[0][:numbers] << n.first.gsub(/(^\W+|\W+$)/, "")}.gsub!(/(\d+[:\.\ ]?\d*)+/, " ")
+      t.scan(/(\b[A-Z][\.\ ][A-Z][\.\ ][A-Z][\.\ ]|\b[A-Z][\.\ ][A-Z][\.\ ])/){|a| tweet.values[0][:acronyms] << a.first.strip}.gsub!(/(\b[A-Z][\.\ ][A-Z][\.\ ][A-Z][\.\ ]|\b[A-Z][\.\ ][A-Z][\.\ ])/, " ")
+      t.scan(/(([A-Z]+[\s\,\&\:\-]+){2,}|[A-Z]{4,}\W)/){|s| tweet.values[0][:shouts] << s.first.gsub(/\W/, " ").strip}.gsub!(/(([A-Z]+[\s\,\&\:\-]+){2,}|[A-Z]{4,}\W)/, " ")
       t.scan(/\b[A-Z]{2,3}\b/){|a| tweet.values[0][:acronyms] << a.strip}.gsub!(/\b[A-Z]{2,3}\b/, " ")
       t.scan(/\w+['\/]?\w*/){|w| tweet.values[0][:words] << w.strip}.gsub!(/\w+['\/]?\w*/, " ")
     end
   end
 end