# It is a sample how to analyze response_time, response_code and user_agent for each virtual domain websites.

# appearing plugins:
#  rewrite_tag_filter: http://rubygems.org/gems/fluent-plugin-rewrite-tag-filter
#  datacounter: http://rubygems.org/gems/fluent-plugin-datacounter
#  growthforecast: http://rubygems.org/gems/fluent-plugin-growthforecast

<source>
  @type tail
  path /var/log/httpd/access_log
  <parse>
    @type regexp
    expression /^(?<domain>[^ ]*) (?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<status>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)" (?<response_time>[^ ]*))?$/
    time_format %d/%b/%Y:%H:%M:%S %z
  </parse>
  tag td.apache.access
  pos_file /var/log/td-agent/apache_access.pos
</source>


# Extract specified virtual domain
<match td.apache.access>
  @type copy
  <store>
    @type rewrite_tag_filter
    capitalize_regex_backreference yes
    <rule>
      key domain
      pattern /^(maps|news|mail)\.google\.com$/
      tag site.Google$1
    </rule>
  </store>
  <store>
    @type rewrite_tag_filter
    capitalize_regex_backreference yes
    <rule>
      key domain
      pattern /^(maps)\.google\.com$/
      tag sitepath.Google$1
    </rule>
  </store>
</match>


# Second level analyzing
<match sitepath.GoogleMaps>
  @type copy
  <store>
    @type rewrite_tag_filter
    <rule>
      key path
      pattern /^\/labs/
      tag site.GoogleMaps.Labs
    </rule>
    <rule>
      key path
      pattern /^\/static/\d+/
      tag site.GoogleMaps.Static
    </rule>
    <rule>
      key path
      pattern /^\/help\/maps\/(streetview|getmaps)/
      tag site.GoogleMaps.Help
    </rule>
  </store>
  <store>
    @type rewrite_tag_filter
    <rule>
      key referer
      pattern /headlines\.yahoo\.co\.jp/
      tag site.GoogleMaps.referer_YahooHeadlines
    </rule>
    <rule>
      key referer
      pattern /news\.livedoor\.com/
      tag site.GoogleMaps.referer_LivedoorNews
    </rule>
  </store>
  <store>
    @type rewrite_tag_filter
    <rule>
      key agent
      pattern /Googlebot\//
      tag site.GoogleMaps.agent_Googlebot
    </rule>
    <rule>
      key agent
      pattern /^.* iPhone .+Googlebot-Mobile\/.*$/
      tag site.GoogleMaps.agent_GooglebotSmartphone
    </rule>
    <rule>
      key agent
      pattern /Googlebot-Mobile\//
      tag site.GoogleMaps.agent_GooglebotMobile
    </rule>
    <rule>
      key agent
      pattern /bingbot/
      tag site.GoogleMaps.agent_Bingbot
    </rule>
    <rule>
      key agent
      pattern /Baiduspider/
      tag site.GoogleMaps.agent_Baiduspider
    </rule>
  </store>
</match>


# Summarize
<match site.**>
  @type copy
  <store>
    @type datacounter
    unit minute
    count_key response_time
    outcast_unmatched false
    aggregate tag
    tag_prefix gf.responsetime
    output_per_tag yes
    pattern1 0-100msec ^\d{1,5}$
    pattern2 100-300msec ^[1-2]\d{5}$
    pattern3 300-600msec ^[3-5]\d{5}$
    pattern4 600msec-1sec ^[6-9]\d{5}$
    pattern5 1-2sec ^1\d{6}$
    pattern6 2-10sec ^[2-9]\d{6}$
    pattern7 10sec_over ^\d{8,}$
  </store>
  <store>
    @type datacounter
    unit minute
    outcast_unmatched false
    aggregate tag
    tag_prefix gf.responsecode
    output_per_tag yes
    count_key status
    pattern1 200 ^200$
    pattern2 2xx ^2\d\d$
    pattern3 301 ^301$
    pattern4 302 ^302$
    pattern5 3xx ^3\d\d$
    pattern6 403 ^403$
    pattern7 404 ^404$
    pattern8 410 ^410$
    pattern9 4xx ^4\d\d$
    pattern10 500 ^5\d\d$
  </store>
  <store>
    @type datacounter
    unit minute
    count_key agent
    outcast_unmatched false
    aggregate tag
    tag_prefix gf.useragent
    output_per_tag yes
    pattern1 api HttpRequest
    pattern2 robot (spider|bot|crawler|\+http\:)
    pattern3 smartphone (iPhone|iPod|Android|dream|CUPCAKE|blackberry|webOS|incognito|webmate|IEMobile)
    pattern4 mobile (^KDDI|UP.Browser|DoCoMo|Vodafone|SoftBank|WILLCOM)
    pattern5 pc .+
  </store>
</match>


# Graph
<match gf.responsetime.**>
  @type growthforecast
  remove_prefix gf.responsetime.site
  gfapi_url http://localhost:5125/api/
  tag_for   service
  section   response_time
  name_keys 0-100msec_percentage,100-300msec_percentage,300-600msec_percentage,600msec-1sec_percentage,1-2sec_percentage,2-10sec_percentage,10sec_over_percentage
</match>

<match gf.responsecode.**>
  @type growthforecast
  remove_prefix gf.responsecode.site
  gfapi_url http://localhost:5125/api/
  tag_for   service
  section   response_code
  name_keys 301_count,302_count,3xx_count,403_count,404_count,410_count,4xx_count,500_count
</match>

<match gf.useragent.**>
  @type growthforecast
  remove_prefix gf.useragent.site
  gfapi_url http://localhost:5125/api/
  tag_for   service
  section   useragent
  name_keys pc_count,mobile_count,smartphone_count,robot_count,api_count
</match>