# It is a sample how to analyze response_time, response_code and user_agent for each virtual domain websites.

# appearing plugins:
#  rewrite_tag_filter: http://rubygems.org/gems/fluent-plugin-rewrite-tag-filter
#  forest: http://rubygems.org/gems/fluent-plugin-forest
#  datacounter: http://rubygems.org/gems/fluent-plugin-datacounter
#  growthforecast: http://rubygems.org/gems/fluent-plugin-growthforecast

<source>
  type tail
  path /var/log/httpd/access_log
  format /^(?<domain>[^ ]*) (?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<status>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)" (?<response_time>[^ ]*))?$/
  time_format %d/%b/%Y:%H:%M:%S %z
  tag td.apache.access
  pos_file /var/log/td-agent/apache_access.pos
</source>


# Extract specified virtual domain
<match td.apache.access>
  type copy
  <store>
    type rewrite_tag_filter
    capitalize_regex_backreference yes
    rewriterule1 domain ^(maps|news|mail)\.google\.com$ site.Google$1
  </store>
  <store>
    type rewrite_tag_filter
    capitalize_regex_backreference yes
    rewriterule1 domain ^(maps)\.google\.com$ sitepath.Google$1
  </store>
</match>


# Second level analyzing
<match sitepath.GoogleMaps>
  <store>
    type rewrite_tag_filter
    rewriterule1 path ^/labs site.GoogleMaps.Labs
    rewriterule2 path ^/static/\d+ site.GoogleMaps.Static
    rewriterule3 path ^/help/maps/(streetview|getmaps) site.GoogleMaps.Help
  </store>
  <store>
    type rewrite_tag_filter
    rewriterule1 referer headlines\.yahoo\.co\.jp site.GoogleMaps.referer_YahooHeadlines
    rewriterule2 referer news\.livedoor\.com site.GoogleMaps.referer_LivedoorNews
  </store>
  <store>
    type rewrite_tag_filter
    rewriterule1 agent Googlebot/ site.GoogleMaps.agent_Googlebot
    rewriterule2 agent ^.* iPhone .+Googlebot-Mobile/.*$ site.GoogleMaps.agent_GooglebotSmartphone
    rewriterule3 agent Googlebot-Mobile/ site.GoogleMaps.agent_GooglebotMobile
    rewriterule4 agent bingbot site.GoogleMaps.agent_Bingbot
    rewriterule5 agent Baiduspider site.GoogleMaps.agent_Baiduspider
  </store>
</match>


# Summarize
<match site.**>
  type copy
  <store>
    type forest
    subtype datacounter
    <template>
      unit minute
      count_key response_time
      outcast_unmatched false
      aggregate all
      tag gf.responsetime.__TAG__
      pattern1 0-100msec ^\d{1,5}$
      pattern2 100-300msec ^[1-2]\d{5}$
      pattern3 300-600msec ^[3-5]\d{5}$
      pattern4 600msec-1sec ^[6-9]\d{5}$
      pattern5 1-2sec ^1\d{6}$
      pattern6 2-10sec ^[2-9]\d{6}$
      pattern7 10sec_over ^\d{8,}$
    </template>
  </store>
  <store>
    type forest
    subtype datacounter
    <template>
      unit minute
      outcast_unmatched false
      aggregate all
      tag gf.responsecode.__TAG__
      count_key status
      pattern1 200 ^200$
      pattern2 2xx ^2\d\d$
      pattern3 301 ^301$
      pattern4 302 ^302$
      pattern5 3xx ^3\d\d$
      pattern6 403 ^403$
      pattern7 404 ^404$
      pattern8 410 ^410$
      pattern9 4xx ^4\d\d$
      pattern10 500 ^5\d\d$
    </template>
  </store>
  <store>
    type forest
    subtype datacounter
    <template>
      unit minute
      count_key agent
      outcast_unmatched false
      aggregate all
      tag gf.useragent.__TAG__
      pattern1 api HttpRequest
      pattern2 robot (spider|bot|crawler|\+http\:)
      pattern3 smartphone (iPhone|iPod|Android|dream|CUPCAKE|blackberry|webOS|incognito|webmate|IEMobile)
      pattern4 mobile (^KDDI|UP.Browser|DoCoMo|Vodafone|SoftBank|WILLCOM)
      pattern5 pc .+
    </template>
  </store>
</match>


# Graph
<match gf.responsetime.**>
  type forest
  subtype growthforecast
  remove_prefix gf.responsetime.site
  <template>
    gfapi_url http://localhost:5125/api/
    service   __TAG__
    section   response_time
    name_keys 0-100msec_percentage,100-300msec_percentage,300-600msec_percentage,600msec-1sec_percentage,1-2sec_percentage,2-10sec_percentage,10sec_over_percentage
  </template>
</match>

<match gf.responsecode.**>
  type forest
  subtype growthforecast
  remove_prefix gf.responsecode.site
  <template>
    gfapi_url http://localhost:5125/api/
    service   __TAG__
    section   response_code
    name_keys 301_count,302_count,3xx_count,403_count,404_count,410_count,4xx_count,500_count
  </template>
</match>

<match gf.useragent.**>
  type forest
  subtype growthforecast
  remove_prefix gf.useragent.site
  <template>
    gfapi_url http://localhost:5125/api/
    service   __TAG__
    section   useragent
    name_keys pc_count,mobile_count,smartphone_count,robot_count,api_count
  </template>
</match>