class Syntax::Tokenizer

exhausted.
Begins tokenizing the given text, calling #step until the text has been

def tokenize( text, &block )
  start text, &block
  step until @text.eos?
  finish
end