Contents

require 'oas/log_collector'
require 'oas/log_collector/log_file'
require 'oas/log_collector/source'

module OAS
  module LogCollector
    class Manager

      def initialize(opts = {})
        sources = opts.delete(:sources) || []
        sources.each { |host, user, port| (opts[:sources] ||= []) << OAS::LogCollector::Source.new(host, user, port || 22) }

        options.merge!(opts)
      end

      def start!
        loop do
          threads = []
          options[:sources].each do |source|
            threads << Thread.new do
              collect(source)
            end
          end
          threads.each(&:join)

          sleep options[:sleep_seconds]
        end
      end

      def options
        OAS::LogCollector.options
      end

      def collect(source)
        hostname = source.host

        result = source.find(options[:source_logs_path], {
          :mtime => "-#{options[:days_to_search]}",
          :type  => 'f',
          :regex => '.*\.log\.[0-9]+\.bz2'
        })

        files = Hash.new
        total = 0
        last_timestamp = last_run = get_last_run(hostname)
        result.sort!.each do |path|
          log_file  = OAS::LogCollector::LogFile.new(path)
          if log_file.timestamp > Integer(last_run) && total < options[:files_to_copy]
            files[path] = File.join(options[:logs_path], hostname, log_file.year, log_file.month, log_file.day, log_file.name)
            last_timestamp = log_file.timestamp
            total += 1
          end
        end

        if files.any?
          source.download(files)
          set_last_run(hostname, last_timestamp)
        end
      end

      def get_last_run(hostname)
        OAS::LogCollector.redis do |conn|
          conn.hget("timestamps", hostname) || 0
        end
      end

      def set_last_run(hostname, timestamp)
        OAS::LogCollector.redis do |conn|
          conn.hset("timestamps", hostname, timestamp)
        end
      end
    end
  end
end

Version data entries

2 entries across 2 versions & 1 rubygems

Version	Path
oas-log-collector-0.1.6	lib/oas/log_collector/manager.rb
oas-log-collector-0.1.5	lib/oas/log_collector/manager.rb