#!/usr/bin/env ruby

require 'tokenizer'

tokenizer = Tokenizer::Tokenizer.new

while (line = gets)
  puts tokenizer.tokenize(line).join("\n")
end