Sha256: cf675a46eeaecc90b79593a27754953a28f956e59fd2666adb536e3c09d6a6c8
Contents?: true
Size: 560 Bytes
Versions: 15
Compression:
Stored size: 560 Bytes
Contents
module CMSScanner
module Finders
module InterestingFile
# Robots.txt finder
class RobotsTxt < Finder
# @return [ String ] The url of the robots.txt file
def url
target.url('robots.txt')
end
# @return [ InterestingFile ]
def aggressive(_opts = {})
res = NS::Browser.get(url)
return unless res && res.code == 200 && res.body =~ /(?:user-agent|(?:dis)?allow):/i
NS::RobotsTxt.new(url, confidence: 100, found_by: found_by)
end
end
end
end
end
Version data entries
15 entries across 15 versions & 1 rubygems