file_presence.rb in yawast-0.5.0.beta2

- old
+ new

@@ -23,11 +23,11 @@
 
               puts ''
             end
           end
 
-          def self.check_all(uri)
+          def self.check_all(uri, common_files)
             #first, we need to see if the site responds to 404 in a reasonable way
             fake_uri = uri.copy
             fake_uri.path = "/#{SecureRandom.hex}/"
             if Yawast::Shared::Http.get_status_code(fake_uri) != '404'
               #crazy 404 handling
@@ -42,17 +42,26 @@
             check_wsftp_log uri
             check_trace_axd uri
             check_elmah_axd uri
             check_readme_html uri
             check_release_notes_txt uri
+
+            if common_files
+              puts ''
+              puts 'Checking for common files (this will take a few minutes)...'
+              check_common uri
+            end
+
+            puts ''
           end
 
           def self.check_source_control(uri)
             check_path(uri, '/.git/', true)
             check_path(uri, '/.hg/', true)
             check_path(uri, '/.svn/', true)
             check_path(uri, '/.bzr/', true)
+            check_path(uri, '/.csv/', true)
           end
 
           def self.check_cross_domain(uri)
             check_path(uri, '/crossdomain.xml', false)
             check_path(uri, '/clientaccesspolicy.xml', false)
@@ -80,9 +89,88 @@
             check_path(uri, '/readme.html', false)
           end
 
           def self.check_release_notes_txt(uri)
             check_path(uri, '/RELEASE-NOTES.txt', false)
+            check_path(uri, '/docs/RELEASE-NOTES.txt', false)
+          end
+
+          def self.check_common(uri)
+            begin
+              @search_list = []
+
+              File.open(File.dirname(__FILE__) + '/../../../resources/common_file.txt', 'r') do |f|
+                f.each_line do |line|
+                  @search_list.push line.strip
+                end
+              end
+
+              pool_size = 16
+              @jobs = Queue.new
+              @results = Queue.new
+
+              #load the queue, starting at /
+              base = uri.copy
+              base.path = '/'
+              load_queue base
+
+              workers = (pool_size).times.map do
+                Thread.new do
+                  begin
+                    while (check = @jobs.pop(true))
+                      process check
+                    end
+                  rescue ThreadError
+                    #do nothing
+                  end
+                end
+              end
+
+              results = Thread.new do
+                begin
+                  while true
+                    if @results.length > 0
+                      out = @results.pop(true)
+                      Yawast::Utilities.puts_info out
+                    end
+                  end
+                rescue ThreadError
+                  #do nothing
+                end
+              end
+
+              workers.map(&:join)
+              results.terminate
+            rescue => e
+              Yawast::Utilities.puts_error "Error searching for files (#{e.message})"
+            end
+          end
+
+          def self.load_queue(uri)
+            @search_list.each do |line|
+              check = uri.copy
+
+              begin
+                check.path = "/#{line}"
+
+                #add the job to the queue
+                @jobs.push check
+              rescue
+                #who cares
+              end
+            end
+          end
+
+          def self.process(uri)
+            begin
+              res = Yawast::Shared::Http.head uri
+
+              if res.code == '200'
+                @results.push "'#{uri.path}' found: #{uri}"
+              end
+            rescue => e
+              Yawast::Utilities.puts_error "Error searching for file '#{uri.path}' (#{e.message})"
+            end
           end
         end
       end
     end
   end