#!/usr/bin/env ruby

require 'tokenizer'

tokenizer = Tokenizer::Tokenizer.new

while (line = gets) puts tokenizer.tokenize(line).join("\n") end