module StreamWrapper class GzipMultiFile def initialize (paths = [], filters = {}) @paths = paths.is_a?(Array) ? paths.dup : [paths] @filters = filters if @paths.empty? @current_line = 0 @current_file = $stdin @current_path = '' end end def gets if file = current_stream @current_line += 1 file.gets else nil end end def eof? ! current_stream end def close @current_file.close unless @current_file.nil? end def current_position "#{@current_path}:#{@current_line}" end def debug= (bool) @debug = bool end def debug? @debug end protected def log (message) $stderr.puts message if debug? end def current_stream if @current_file.nil? || @current_file.eof? close @current_file = next_available_file end @current_file end def next_available_file if @current_path log "Closed #{@current_path} after reading #{@current_line} lines" end @current_line = 0 @current_path = @paths.shift or return nil log "Switching to #{@current_path}" file = IO.popen(shell_command(@current_path)) file.eof? ? next_available_file : file end # Use whitelist to be sure. Observation filters is applied after anyways. SHOP_AND_ACCOUNT_ID_FILTER = /^[a-zA-Z0-9._]+$/ def shell_command (path) if path =~ /\.gz/ command = "gunzip -c #{path.gsub(' ','\\ ')}" else command = "cat #{path.gsub(' ','\\ ')}" end if @filters[:shop_id] && @filters[:shop_id] =~ SHOP_AND_ACCOUNT_ID_FILTER command += " | grep -F sid=#{@filters[:shop_id]}" end if @filters[:account_id] && @filters[:account_id] =~ SHOP_AND_ACCOUNT_ID_FILTER command += " | grep -F aid=#{@filters[:account_id]}" end if @filters[:observation_type] command += " | grep -F o=#{@filters[:observation_type]}" end command end end end