#!/usr/bin/env ruby

require ‘tokenizer’

tokenizer = Tokenizer::Tokenizer.new

while (line = gets)

puts tokenizer.tokenize(line).join("\n")

end